xnu-1504.15.3.tar.gz

[apple/xnu.git] / osfmk / default_pager / dp_backing_store.c

diff --git a/osfmk/default_pager/dp_backing_store.c b/osfmk/default_pager/dp_backing_store.c
index 0c3ae6af88abeaec7336e2fb553f765d9e9065fc..ceda0a9024544d67ac92375ac1c9e97487e1dbc0 100644 (file)
--- a/osfmk/default_pager/dp_backing_store.c
+++ b/osfmk/default_pager/dp_backing_store.c
@@ -1,23 +1,29 @@
 /*
 /*

- * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2008 Apple Inc. All rights reserved.

  *
  *

- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@

  * 
  * 

- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.

  * 
  * 

- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER

  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,

- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.

  * 
  * 

- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@

  */
 /*
  * @OSF_COPYRIGHT@
  */
 /*
  * @OSF_COPYRIGHT@


@@ -53,19 +59,32 @@
  *             Paging File Management.
  */
 
  *             Paging File Management.
  */
 

+#include <mach/host_priv.h>

 #include <mach/memory_object_control.h>
 #include <mach/memory_object_server.h>
 #include <mach/memory_object_control.h>
 #include <mach/memory_object_server.h>

-#include "default_pager_internal.h"
+#include <mach/upl.h>
+#include <default_pager/default_pager_internal.h>

 #include <default_pager/default_pager_alerts.h>
 #include <default_pager/default_pager_alerts.h>

+#include <default_pager/default_pager_object_server.h>
+
+#include <ipc/ipc_types.h>

 #include <ipc/ipc_port.h>
 #include <ipc/ipc_space.h>
 #include <ipc/ipc_port.h>
 #include <ipc/ipc_space.h>

+
+#include <kern/kern_types.h>
+#include <kern/host.h>

 #include <kern/queue.h>
 #include <kern/counters.h>
 #include <kern/sched_prim.h>
 #include <kern/queue.h>
 #include <kern/counters.h>
 #include <kern/sched_prim.h>

+

 #include <vm/vm_kern.h> 
 #include <vm/vm_pageout.h>
 #include <vm/vm_kern.h> 
 #include <vm/vm_pageout.h>

-/* CDY CDY */

 #include <vm/vm_map.h>
 #include <vm/vm_map.h>

+#include <vm/vm_object.h>
+#include <vm/vm_protos.h>
+
+
+/* todo - need large internal object support */

 
 /*
  * ALLOC_STRIDE... the maximum number of bytes allocated from
 
 /*
  * ALLOC_STRIDE... the maximum number of bytes allocated from


@@ -82,16 +101,19 @@
 int physical_transfer_cluster_count = 0;
 
 #define VM_SUPER_CLUSTER       0x40000
 int physical_transfer_cluster_count = 0;
 
 #define VM_SUPER_CLUSTER       0x40000

-#define VM_SUPER_PAGES          64
+#define VM_SUPER_PAGES          (VM_SUPER_CLUSTER / PAGE_SIZE)

 
 /*
  * 0 means no shift to pages, so == 1 page/cluster. 1 would mean
  * 2 pages/cluster, 2 means 4 pages/cluster, and so on.
  */
 
 /*
  * 0 means no shift to pages, so == 1 page/cluster. 1 would mean
  * 2 pages/cluster, 2 means 4 pages/cluster, and so on.
  */

+#define VSTRUCT_MIN_CLSHIFT    0
+

 #define VSTRUCT_DEF_CLSHIFT    2
 #define VSTRUCT_DEF_CLSHIFT    2

-int vstruct_def_clshift = VSTRUCT_DEF_CLSHIFT;

 int default_pager_clsize = 0;
 
 int default_pager_clsize = 0;
 

+int vstruct_def_clshift = VSTRUCT_DEF_CLSHIFT;
+

 /* statistics */
 unsigned int clustered_writes[VM_SUPER_PAGES+1];
 unsigned int clustered_reads[VM_SUPER_PAGES+1];
 /* statistics */
 unsigned int clustered_writes[VM_SUPER_PAGES+1];
 unsigned int clustered_reads[VM_SUPER_PAGES+1];


@@ -114,7 +136,7 @@ int async_requests_out;
 #define VS_ASYNC_REUSE 1
 struct vs_async *vs_async_free_list;
 
 #define VS_ASYNC_REUSE 1
 struct vs_async *vs_async_free_list;
 

-mutex_t default_pager_async_lock;      /* Protects globals above */
+lck_mtx_t      default_pager_async_lock;       /* Protects globals above */

 
 
 int vs_alloc_async_failed = 0;                 /* statistics */
 
 
 int vs_alloc_async_failed = 0;                 /* statistics */


@@ -126,24 +148,33 @@ void vs_free_async(struct vs_async *vsa); /* forward */
 #define VS_ALLOC_ASYNC()       vs_alloc_async()
 #define VS_FREE_ASYNC(vsa)     vs_free_async(vsa)
 
 #define VS_ALLOC_ASYNC()       vs_alloc_async()
 #define VS_FREE_ASYNC(vsa)     vs_free_async(vsa)
 

-#define VS_ASYNC_LOCK()                mutex_lock(&default_pager_async_lock)
-#define VS_ASYNC_UNLOCK()      mutex_unlock(&default_pager_async_lock)
-#define VS_ASYNC_LOCK_INIT()   mutex_init(&default_pager_async_lock,  \
-                                               ETAP_IO_DEV_PAGEH)
+#define VS_ASYNC_LOCK()                lck_mtx_lock(&default_pager_async_lock)
+#define VS_ASYNC_UNLOCK()      lck_mtx_unlock(&default_pager_async_lock)
+#define VS_ASYNC_LOCK_INIT()   lck_mtx_init(&default_pager_async_lock, &default_pager_lck_grp, &default_pager_lck_attr)

 #define VS_ASYNC_LOCK_ADDR()   (&default_pager_async_lock)
 /*
  *  Paging Space Hysteresis triggers and the target notification port
  *
  */ 
 #define VS_ASYNC_LOCK_ADDR()   (&default_pager_async_lock)
 /*
  *  Paging Space Hysteresis triggers and the target notification port
  *
  */ 

-
+unsigned int   dp_pages_free_drift_count = 0;
+unsigned int   dp_pages_free_drifted_max = 0;

 unsigned int   minimum_pages_remaining = 0;
 unsigned int   maximum_pages_free = 0;
 ipc_port_t     min_pages_trigger_port = NULL;
 ipc_port_t     max_pages_trigger_port = NULL;
 
 unsigned int   minimum_pages_remaining = 0;
 unsigned int   maximum_pages_free = 0;
 ipc_port_t     min_pages_trigger_port = NULL;
 ipc_port_t     max_pages_trigger_port = NULL;
 

+boolean_t      use_emergency_swap_file_first = FALSE;

 boolean_t      bs_low = FALSE;
 int            backing_store_release_trigger_disable = 0;
 boolean_t      bs_low = FALSE;
 int            backing_store_release_trigger_disable = 0;

- 
+boolean_t      backing_store_stop_compaction = FALSE;
+
+
+/* Have we decided if swap needs to be encrypted yet ? */
+boolean_t      dp_encryption_inited = FALSE;
+/* Should we encrypt swap ? */
+boolean_t      dp_encryption = FALSE;
+
+boolean_t      dp_isssd = FALSE;

 
 
 /*
 
 
 /*


@@ -155,9 +186,10 @@ vm_size_t  max_doubled_size = 4 * 1024 * 1024;     /* 4 meg */
 /*
  * List of all backing store and segments.
  */
 /*
  * List of all backing store and segments.
  */

+MACH_PORT_FACE         emergency_segment_backing_store;

 struct backing_store_list_head backing_store_list;
 paging_segment_t       paging_segments[MAX_NUM_PAGING_SEGMENTS];
 struct backing_store_list_head backing_store_list;
 paging_segment_t       paging_segments[MAX_NUM_PAGING_SEGMENTS];

-mutex_t                        paging_segments_lock;
+lck_mtx_t                      paging_segments_lock;

 int                    paging_segment_max = 0;
 int                    paging_segment_count = 0;
 int ps_select_array[BS_MAXPRI+1] = { -1,-1,-1,-1,-1 };
 int                    paging_segment_max = 0;
 int                    paging_segment_count = 0;
 int ps_select_array[BS_MAXPRI+1] = { -1,-1,-1,-1,-1 };


@@ -170,14 +202,33 @@ int ps_select_array[BS_MAXPRI+1] = { -1,-1,-1,-1,-1 };
  * likely to be deprecated.
  */
 unsigned  int  dp_pages_free = 0;
  * likely to be deprecated.
  */
 unsigned  int  dp_pages_free = 0;

+unsigned  int  dp_pages_reserve = 0;

 unsigned  int  cluster_transfer_minimum = 100;
 
 unsigned  int  cluster_transfer_minimum = 100;
 

-kern_return_t ps_write_file(paging_segment_t, upl_t, vm_offset_t, vm_offset_t, unsigned int, int);     /* forward */
-kern_return_t ps_read_file (paging_segment_t, upl_t, vm_offset_t, vm_offset_t, unsigned int, unsigned int *, int);     /* forward */
+/* forward declarations */
+kern_return_t ps_write_file(paging_segment_t, upl_t, upl_offset_t, dp_offset_t, unsigned int, int);    /* forward */
+kern_return_t ps_read_file (paging_segment_t, upl_t, upl_offset_t, dp_offset_t, unsigned int, unsigned int *, int);    /* forward */
+default_pager_thread_t *get_read_buffer( void );
+kern_return_t ps_vstruct_transfer_from_segment(
+       vstruct_t        vs,
+       paging_segment_t segment,
+       upl_t            upl);
+kern_return_t ps_read_device(paging_segment_t, dp_offset_t, vm_offset_t *, unsigned int, unsigned int *, int); /* forward */
+kern_return_t ps_write_device(paging_segment_t, dp_offset_t, vm_offset_t, unsigned int, struct vs_async *);    /* forward */
+kern_return_t vs_cluster_transfer(
+       vstruct_t       vs,
+       dp_offset_t     offset,
+       dp_size_t       cnt,
+       upl_t           upl);
+vs_map_t vs_get_map_entry(
+       vstruct_t       vs, 
+       dp_offset_t     offset);

 
 

+kern_return_t
+default_pager_backing_store_delete_internal( MACH_PORT_FACE );

 
 default_pager_thread_t *
 
 default_pager_thread_t *

-get_read_buffer()
+get_read_buffer( void )

 {
        int     i;
 
 {
        int     i;
 


@@ -310,10 +361,10 @@ int default_pager_info_verbose = 1;
 
 void
 bs_global_info(
 
 void
 bs_global_info(

-       vm_size_t       *totalp,
-       vm_size_t       *freep)
+       uint64_t        *totalp,
+       uint64_t        *freep)

 {
 {

-       vm_size_t               pages_total, pages_free;
+       uint64_t                pages_total, pages_free;

        paging_segment_t        ps;
        int                     i;
 
        paging_segment_t        ps;
        int                     i;
 


@@ -331,9 +382,9 @@ bs_global_info(
                 */
                pages_total += ps->ps_pgnum;
                pages_free += ps->ps_clcount << ps->ps_clshift;
                 */
                pages_total += ps->ps_pgnum;
                pages_free += ps->ps_clcount << ps->ps_clshift;

-               DEBUG(DEBUG_BS_INTERNAL,
-                     ("segment #%d: %d total, %d free\n",
-                      i, ps->ps_pgnum, ps->ps_clcount << ps->ps_clshift));
+               DP_DEBUG(DEBUG_BS_INTERNAL,
+                        ("segment #%d: %d total, %d free\n",
+                         i, ps->ps_pgnum, ps->ps_clcount << ps->ps_clshift));

        }
        *totalp = pages_total;
        *freep = pages_free;
        }
        *totalp = pages_total;
        *freep = pages_free;


@@ -412,10 +463,10 @@ void backing_store_add(backing_store_t);  /* forward */
 
 void
 backing_store_add(
 
 void
 backing_store_add(

-       backing_store_t bs)
+       __unused backing_store_t bs)

 {
 {

-       MACH_PORT_FACE          port = bs->bs_port;
-       MACH_PORT_FACE          pset = default_pager_default_set;
+//     MACH_PORT_FACE          port = bs->bs_port;
+//     MACH_PORT_FACE          pset = default_pager_default_set;

        kern_return_t           kr = KERN_SUCCESS;
 
        if (kr != KERN_SUCCESS)
        kern_return_t           kr = KERN_SUCCESS;
 
        if (kr != KERN_SUCCESS)


@@ -512,7 +563,7 @@ default_pager_backing_store_create(
 {
        backing_store_t bs;
        MACH_PORT_FACE  port;
 {
        backing_store_t bs;
        MACH_PORT_FACE  port;

-       kern_return_t   kr;
+//     kern_return_t   kr;

        struct vstruct_alias *alias_struct;
 
        if (pager != default_pager_object)
        struct vstruct_alias *alias_struct;
 
        if (pager != default_pager_object)


@@ -523,20 +574,20 @@ default_pager_backing_store_create(
        ipc_port_make_send(port);
        assert (port != IP_NULL);
 
        ipc_port_make_send(port);
        assert (port != IP_NULL);
 

-       DEBUG(DEBUG_BS_EXTERNAL,
-             ("priority=%d clsize=%d bs_port=0x%x\n",
-              priority, clsize, (int) backing_store));
+       DP_DEBUG(DEBUG_BS_EXTERNAL,
+                ("priority=%d clsize=%d bs_port=0x%x\n",
+                 priority, clsize, (int) backing_store));

 
        alias_struct = (struct vstruct_alias *) 
                                kalloc(sizeof (struct vstruct_alias));
        if(alias_struct != NULL) {
                alias_struct->vs = (struct vstruct *)bs;
 
        alias_struct = (struct vstruct_alias *) 
                                kalloc(sizeof (struct vstruct_alias));
        if(alias_struct != NULL) {
                alias_struct->vs = (struct vstruct *)bs;

-               alias_struct->name = ISVS;
-               port->alias = (int) alias_struct;
+               alias_struct->name = &default_pager_ops;
+               port->alias = (uintptr_t) alias_struct;

        }
        else {
                ipc_port_dealloc_kernel((MACH_PORT_FACE)(port));
        }
        else {
                ipc_port_dealloc_kernel((MACH_PORT_FACE)(port));

-               kfree((vm_offset_t)bs, sizeof (struct backing_store));
+               kfree(bs, sizeof (struct backing_store));

                return KERN_RESOURCE_SHORTAGE;
        }
 
                return KERN_RESOURCE_SHORTAGE;
        }
 


@@ -621,6 +672,7 @@ default_pager_backing_store_info(
 }
 
 int ps_delete(paging_segment_t);       /* forward */
 }
 
 int ps_delete(paging_segment_t);       /* forward */

+boolean_t current_thread_aborted(void);

 
 int
 ps_delete(
 
 int
 ps_delete(


@@ -682,27 +734,27 @@ ps_delete(
                        error = KERN_FAILURE;
                else {
                        vm_object_t     transfer_object;
                        error = KERN_FAILURE;
                else {
                        vm_object_t     transfer_object;

-                       int             count;
+                       unsigned int    count;

                        upl_t           upl;
 
                        upl_t           upl;
 

-                       transfer_object = vm_object_allocate(VM_SUPER_CLUSTER);
+                       transfer_object = vm_object_allocate((vm_object_size_t)VM_SUPER_CLUSTER);

                        count = 0;
                        error = vm_object_upl_request(transfer_object, 
                                (vm_object_offset_t)0, VM_SUPER_CLUSTER,
                                &upl, NULL, &count,
                        count = 0;
                        error = vm_object_upl_request(transfer_object, 
                                (vm_object_offset_t)0, VM_SUPER_CLUSTER,
                                &upl, NULL, &count,

-                               UPL_NO_SYNC | UPL_CLEAN_IN_PLACE
-                                           | UPL_SET_INTERNAL);
+                               UPL_NO_SYNC | UPL_CLEAN_IN_PLACE | UPL_SET_LITE | UPL_SET_INTERNAL);
+

                        if(error == KERN_SUCCESS) {
                                error = ps_vstruct_transfer_from_segment(
                                                        vs, ps, upl);
                        if(error == KERN_SUCCESS) {
                                error = ps_vstruct_transfer_from_segment(
                                                        vs, ps, upl);

-                               upl_commit(upl, NULL);
+                               upl_commit(upl, NULL, 0);

                                upl_deallocate(upl);
                        } else {
                                error = KERN_FAILURE;
                        }
                        vm_object_deallocate(transfer_object);
                }
                                upl_deallocate(upl);
                        } else {
                                error = KERN_FAILURE;
                        }
                        vm_object_deallocate(transfer_object);
                }

-               if(error) {
+               if(error || current_thread_aborted() || backing_store_stop_compaction) {

                        VS_LOCK(vs);
                        vs->vs_async_pending -= 1;  /* release vs_async_wait */
                        if (vs->vs_async_pending == 0 && vs->vs_waiting_async) {
                        VS_LOCK(vs);
                        vs->vs_async_pending -= 1;  /* release vs_async_wait */
                        if (vs->vs_async_pending == 0 && vs->vs_waiting_async) {


@@ -753,7 +805,7 @@ ps_delete(
 
 
 kern_return_t
 
 
 kern_return_t

-default_pager_backing_store_delete(
+default_pager_backing_store_delete_internal(

        MACH_PORT_FACE backing_store)
 {
        backing_store_t         bs;
        MACH_PORT_FACE backing_store)
 {
        backing_store_t         bs;


@@ -761,28 +813,35 @@ default_pager_backing_store_delete(
        paging_segment_t        ps;
        int                     error;
        int                     interim_pages_removed = 0;
        paging_segment_t        ps;
        int                     error;
        int                     interim_pages_removed = 0;

-       kern_return_t           kr;
+       boolean_t               dealing_with_emergency_segment = ( backing_store == emergency_segment_backing_store );

 
        if ((bs = backing_store_lookup(backing_store)) == BACKING_STORE_NULL)
                return KERN_INVALID_ARGUMENT;
 
 
        if ((bs = backing_store_lookup(backing_store)) == BACKING_STORE_NULL)
                return KERN_INVALID_ARGUMENT;
 

-#if 0
-       /* not implemented */
-       BS_UNLOCK(bs);
-       return KERN_FAILURE;
-#endif
-
-    restart:
+restart:

        PSL_LOCK();
        error = KERN_SUCCESS;
        for (i = 0; i <= paging_segment_max; i++) {
                ps = paging_segments[i];
                if (ps != PAGING_SEGMENT_NULL &&
                    ps->ps_bs == bs &&
        PSL_LOCK();
        error = KERN_SUCCESS;
        for (i = 0; i <= paging_segment_max; i++) {
                ps = paging_segments[i];
                if (ps != PAGING_SEGMENT_NULL &&
                    ps->ps_bs == bs &&

-                   ! ps->ps_going_away) {
+                   ! IS_PS_GOING_AWAY(ps)) {

                        PS_LOCK(ps);
                        PS_LOCK(ps);

+                       
+                       if( IS_PS_GOING_AWAY(ps) || !IS_PS_OK_TO_USE(ps)) {
+                       /* 
+                        * Someone is already busy reclamining this paging segment.
+                        * If it's the emergency segment we are looking at then check
+                        * that someone has not already recovered it and set the right
+                        * state i.e. online but not activated.
+                        */
+                               PS_UNLOCK(ps);
+                               continue;
+                       }
+

                        /* disable access to this segment */
                        /* disable access to this segment */

-                       ps->ps_going_away = TRUE;
+                       ps->ps_state &= ~PS_CAN_USE;
+                       ps->ps_state |= PS_GOING_AWAY;

                        PS_UNLOCK(ps);
                        /*
                         * The "ps" segment is "off-line" now,
                        PS_UNLOCK(ps);
                        /*
                         * The "ps" segment is "off-line" now,


@@ -823,10 +882,26 @@ default_pager_backing_store_delete(
                        ps = paging_segments[i];
                        if (ps != PAGING_SEGMENT_NULL &&
                            ps->ps_bs == bs &&
                        ps = paging_segments[i];
                        if (ps != PAGING_SEGMENT_NULL &&
                            ps->ps_bs == bs &&

-                           ps->ps_going_away) {
+                           IS_PS_GOING_AWAY(ps)) {

                                PS_LOCK(ps);
                                PS_LOCK(ps);

+                               
+                               if( !IS_PS_GOING_AWAY(ps)) {
+                                       PS_UNLOCK(ps);
+                                       continue;
+                               }
+                               /* Handle the special clusters that came in while we let go the lock*/  
+                               if( ps->ps_special_clusters) {
+                                       dp_pages_free += ps->ps_special_clusters << ps->ps_clshift;
+                                       ps->ps_pgcount += ps->ps_special_clusters << ps->ps_clshift;
+                                       ps->ps_clcount += ps->ps_special_clusters;
+                                       if ( ps_select_array[ps->ps_bs->bs_priority] == BS_FULLPRI) {
+                                               ps_select_array[ps->ps_bs->bs_priority] = 0;
+                                       }
+                                       ps->ps_special_clusters = 0;
+                               }

                                /* re-enable access to this segment */
                                /* re-enable access to this segment */

-                               ps->ps_going_away = FALSE;
+                               ps->ps_state &= ~PS_GOING_AWAY;
+                               ps->ps_state |= PS_CAN_USE;

                                PS_UNLOCK(ps);
                        }
                }
                                PS_UNLOCK(ps);
                        }
                }


@@ -840,13 +915,22 @@ default_pager_backing_store_delete(
                ps = paging_segments[i];
                if (ps != PAGING_SEGMENT_NULL &&
                    ps->ps_bs == bs) { 
                ps = paging_segments[i];
                if (ps != PAGING_SEGMENT_NULL &&
                    ps->ps_bs == bs) { 

-                       if(ps->ps_going_away) {
-                               paging_segments[i] = PAGING_SEGMENT_NULL;
-                               paging_segment_count--;
-                               PS_LOCK(ps);
-                               kfree((vm_offset_t)ps->ps_bmap, 
-                                               RMAPSIZE(ps->ps_ncls));
-                               kfree((vm_offset_t)ps, sizeof *ps);
+                       if(IS_PS_GOING_AWAY(ps)) {
+                               if(IS_PS_EMERGENCY_SEGMENT(ps)) {
+                                       PS_LOCK(ps);
+                                       ps->ps_state &= ~PS_GOING_AWAY;
+                                       ps->ps_special_clusters = 0;
+                                       ps->ps_pgcount = ps->ps_pgnum;
+                                       ps->ps_clcount = ps->ps_ncls = ps->ps_pgcount >> ps->ps_clshift;
+                                       dp_pages_reserve += ps->ps_pgcount;
+                                       PS_UNLOCK(ps);
+                               } else {
+                                       paging_segments[i] = PAGING_SEGMENT_NULL;
+                                       paging_segment_count--;
+                                       PS_LOCK(ps);
+                                       kfree(ps->ps_bmap, RMAPSIZE(ps->ps_ncls));
+                                       kfree(ps, sizeof *ps);
+                               }

                        }
                }
        }
                        }
                }
        }


@@ -860,6 +944,11 @@ default_pager_backing_store_delete(
 
        PSL_UNLOCK();
 
 
        PSL_UNLOCK();
 

+       if( dealing_with_emergency_segment ) {
+               BS_UNLOCK(bs);
+               return KERN_SUCCESS;
+       }
+

        /*
         * All the segments have been deleted.
         * We can remove the backing store.
        /*
         * All the segments have been deleted.
         * We can remove the backing store.


@@ -869,8 +958,8 @@ default_pager_backing_store_delete(
         * Disable lookups of this backing store.
         */
        if((void *)bs->bs_port->alias != NULL)
         * Disable lookups of this backing store.
         */
        if((void *)bs->bs_port->alias != NULL)

-               kfree((vm_offset_t) bs->bs_port->alias, 
-                               sizeof (struct vstruct_alias));
+               kfree((void *) bs->bs_port->alias,
+                     sizeof (struct vstruct_alias));

        ipc_port_dealloc_kernel((ipc_port_t) (bs->bs_port));
        bs->bs_port = MACH_PORT_NULL;
        BS_UNLOCK(bs);
        ipc_port_dealloc_kernel((ipc_port_t) (bs->bs_port));
        bs->bs_port = MACH_PORT_NULL;
        BS_UNLOCK(bs);


@@ -886,11 +975,21 @@ default_pager_backing_store_delete(
        /*
         * Free the backing store structure.
         */
        /*
         * Free the backing store structure.
         */

-       kfree((vm_offset_t)bs, sizeof *bs);
+       kfree(bs, sizeof *bs);

 
        return KERN_SUCCESS;
 }
 
 
        return KERN_SUCCESS;
 }
 

+kern_return_t
+default_pager_backing_store_delete(
+       MACH_PORT_FACE backing_store) 
+{
+       if( backing_store != emergency_segment_backing_store ) {
+               default_pager_backing_store_delete_internal(emergency_segment_backing_store);
+       }
+       return(default_pager_backing_store_delete_internal(backing_store));
+}
+

 int    ps_enter(paging_segment_t);     /* forward */
 
 int
 int    ps_enter(paging_segment_t);     /* forward */
 
 int


@@ -985,7 +1084,7 @@ default_pager_add_segment(
        PS_LOCK_INIT(ps);
        ps->ps_bmap = (unsigned char *) kalloc(RMAPSIZE(ps->ps_ncls));
        if (!ps->ps_bmap) {
        PS_LOCK_INIT(ps);
        ps->ps_bmap = (unsigned char *) kalloc(RMAPSIZE(ps->ps_ncls));
        if (!ps->ps_bmap) {

-               kfree((vm_offset_t)ps, sizeof *ps);
+               kfree(ps, sizeof *ps);

                BS_UNLOCK(bs);
                return KERN_RESOURCE_SHORTAGE;
        }
                BS_UNLOCK(bs);
                return KERN_RESOURCE_SHORTAGE;
        }


@@ -993,12 +1092,20 @@ default_pager_add_segment(
                clrbit(ps->ps_bmap, i);
        }
 
                clrbit(ps->ps_bmap, i);
        }
 

-       ps->ps_going_away = FALSE;
+       if(paging_segment_count == 0) {
+               ps->ps_state = PS_EMERGENCY_SEGMENT;
+               if(use_emergency_swap_file_first) {
+                       ps->ps_state |= PS_CAN_USE;
+               }
+       } else {
+               ps->ps_state = PS_CAN_USE;
+       }
+

        ps->ps_bs = bs;
 
        if ((error = ps_enter(ps)) != 0) {
        ps->ps_bs = bs;
 
        if ((error = ps_enter(ps)) != 0) {

-               kfree((vm_offset_t)ps->ps_bmap, RMAPSIZE(ps->ps_ncls));
-               kfree((vm_offset_t)ps, sizeof *ps);
+               kfree(ps->ps_bmap, RMAPSIZE(ps->ps_ncls));
+               kfree(ps, sizeof *ps);

                BS_UNLOCK(bs);
                return KERN_RESOURCE_SHORTAGE;
        }
                BS_UNLOCK(bs);
                return KERN_RESOURCE_SHORTAGE;
        }


@@ -1008,15 +1115,19 @@ default_pager_add_segment(
        BS_UNLOCK(bs);
 
        PSL_LOCK();
        BS_UNLOCK(bs);
 
        PSL_LOCK();

-       dp_pages_free += ps->ps_pgcount;
+       if(IS_PS_OK_TO_USE(ps)) {
+               dp_pages_free += ps->ps_pgcount;
+       } else {
+               dp_pages_reserve += ps->ps_pgcount;
+       }

        PSL_UNLOCK();
 
        bs_more_space(ps->ps_clcount);
 
        PSL_UNLOCK();
 
        bs_more_space(ps->ps_clcount);
 

-       DEBUG(DEBUG_BS_INTERNAL,
-             ("device=0x%x,offset=0x%x,count=0x%x,record_size=0x%x,shift=%d,total_size=0x%x\n",
-              device, offset, count, record_size,
-              ps->ps_record_shift, ps->ps_pgnum));
+       DP_DEBUG(DEBUG_BS_INTERNAL,
+                ("device=0x%x,offset=0x%x,count=0x%x,record_size=0x%x,shift=%d,total_size=0x%x\n",
+                 device, offset, count, record_size,
+                 ps->ps_record_shift, ps->ps_pgnum));

 
        return KERN_SUCCESS;
 }
 
        return KERN_SUCCESS;
 }


@@ -1072,7 +1183,7 @@ vs_alloc_async(void)
 {
        struct vs_async *vsa;
        MACH_PORT_FACE  reply_port;
 {
        struct vs_async *vsa;
        MACH_PORT_FACE  reply_port;

-       kern_return_t   kr;
+//     kern_return_t   kr;

 
        VS_ASYNC_LOCK();
        if (vs_async_free_list == NULL) {
 
        VS_ASYNC_LOCK();
        if (vs_async_free_list == NULL) {


@@ -1090,8 +1201,8 @@ vs_alloc_async(void)
                                kalloc(sizeof (struct vstruct_alias));
                        if(alias_struct != NULL) {
                                alias_struct->vs = (struct vstruct *)vsa;
                                kalloc(sizeof (struct vstruct_alias));
                        if(alias_struct != NULL) {
                                alias_struct->vs = (struct vstruct *)vsa;

-                               alias_struct->name = ISVS;
-                               reply_port->alias = (int) alias_struct;
+                               alias_struct->name = &default_pager_ops;
+                               reply_port->alias = (uintptr_t) alias_struct;

                                vsa->reply_port = reply_port;
                                vs_alloc_async_count++;
                        }
                                vsa->reply_port = reply_port;
                                vs_alloc_async_count++;
                        }


@@ -1099,8 +1210,7 @@ vs_alloc_async(void)
                                vs_alloc_async_failed++;
                                ipc_port_dealloc_kernel((MACH_PORT_FACE) 
                                                                (reply_port));
                                vs_alloc_async_failed++;
                                ipc_port_dealloc_kernel((MACH_PORT_FACE) 
                                                                (reply_port));

-                               kfree((vm_offset_t)vsa, 
-                                               sizeof (struct vs_async));
+                               kfree(vsa, sizeof (struct vs_async));

                                vsa = NULL;
                        }
                }
                                vsa = NULL;
                        }
                }


@@ -1143,7 +1253,7 @@ vs_alloc_async(void)
                                kalloc(sizeof (struct vstruct_alias));
                        if(alias_struct != NULL) {
                                alias_struct->vs = reply_port;
                                kalloc(sizeof (struct vstruct_alias));
                        if(alias_struct != NULL) {
                                alias_struct->vs = reply_port;

-                               alias_struct->name = ISVS;
+                               alias_struct->name = &default_pager_ops;

                                reply_port->alias = (int) vsa;
                                vsa->reply_port = reply_port;
                                vs_alloc_async_count++;
                                reply_port->alias = (int) vsa;
                                vsa->reply_port = reply_port;
                                vs_alloc_async_count++;


@@ -1152,8 +1262,7 @@ vs_alloc_async(void)
                                vs_alloc_async_failed++;
                                ipc_port_dealloc_kernel((MACH_PORT_FACE) 
                                                                (reply_port));
                                vs_alloc_async_failed++;
                                ipc_port_dealloc_kernel((MACH_PORT_FACE) 
                                                                (reply_port));

-                               kfree((vm_offset_t) vsa, 
-                                               sizeof (struct vs_async));
+                               kfree(vsa, sizeof (struct vs_async));

                                vsa = NULL;
                        }
        }
                                vsa = NULL;
                        }
        }


@@ -1169,8 +1278,8 @@ vs_free_async(
        kern_return_t   kr;
 
        reply_port = vsa->reply_port;
        kern_return_t   kr;
 
        reply_port = vsa->reply_port;

-       kfree((vm_offset_t) reply_port->alias, sizeof (struct vstuct_alias));
-       kfree((vm_offset_t) vsa, sizeof (struct vs_async));
+       kfree(reply_port->alias, sizeof (struct vstuct_alias));
+       kfree(vsa, sizeof (struct vs_async));

        ipc_port_dealloc_kernel((MACH_PORT_FACE) (reply_port));
 #if 0
        VS_ASYNC_LOCK();
        ipc_port_dealloc_kernel((MACH_PORT_FACE) (reply_port));
 #if 0
        VS_ASYNC_LOCK();


@@ -1185,10 +1294,10 @@ zone_t  vstruct_zone;
 
 vstruct_t
 ps_vstruct_create(
 
 vstruct_t
 ps_vstruct_create(

-       vm_size_t size)
+       dp_size_t size)

 {
        vstruct_t       vs;
 {
        vstruct_t       vs;

-       int             i;
+       unsigned int    i;

 
        vs = (vstruct_t) zalloc(vstruct_zone);
        if (vs == VSTRUCT_NULL) {
 
        vs = (vstruct_t) zalloc(vstruct_zone);
        if (vs == VSTRUCT_NULL) {


@@ -1200,23 +1309,15 @@ ps_vstruct_create(
        /*
         * The following fields will be provided later.
         */
        /*
         * The following fields will be provided later.
         */

-       vs->vs_mem_obj = NULL;
+       vs->vs_pager_ops = NULL;

        vs->vs_control = MEMORY_OBJECT_CONTROL_NULL;
        vs->vs_references = 1;
        vs->vs_seqno = 0;
 
        vs->vs_control = MEMORY_OBJECT_CONTROL_NULL;
        vs->vs_references = 1;
        vs->vs_seqno = 0;
 

-#ifdef MACH_KERNEL

        vs->vs_waiting_seqno = FALSE;
        vs->vs_waiting_read = FALSE;
        vs->vs_waiting_write = FALSE;
        vs->vs_waiting_async = FALSE;
        vs->vs_waiting_seqno = FALSE;
        vs->vs_waiting_read = FALSE;
        vs->vs_waiting_write = FALSE;
        vs->vs_waiting_async = FALSE;

-#else
-       mutex_init(&vs->vs_waiting_seqno, ETAP_DPAGE_VSSEQNO);
-       mutex_init(&vs->vs_waiting_read, ETAP_DPAGE_VSREAD);
-       mutex_init(&vs->vs_waiting_write, ETAP_DPAGE_VSWRITE);
-       mutex_init(&vs->vs_waiting_refs, ETAP_DPAGE_VSREFS);
-       mutex_init(&vs->vs_waiting_async, ETAP_DPAGE_VSASYNC);
-#endif

 
        vs->vs_readers = 0;
        vs->vs_writers = 0;
 
        vs->vs_readers = 0;
        vs->vs_writers = 0;


@@ -1241,14 +1342,14 @@ ps_vstruct_create(
                vs->vs_indirect = FALSE;
        }
        vs->vs_xfer_pending = FALSE;
                vs->vs_indirect = FALSE;
        }
        vs->vs_xfer_pending = FALSE;

-       DEBUG(DEBUG_VS_INTERNAL,
-             ("map=0x%x, indirect=%d\n", (int) vs->vs_dmap, vs->vs_indirect));
+       DP_DEBUG(DEBUG_VS_INTERNAL,
+                ("map=0x%x, indirect=%d\n", (int) vs->vs_dmap, vs->vs_indirect));

 
        /*
         * Check to see that we got the space.
         */
        if (!vs->vs_dmap) {
 
        /*
         * Check to see that we got the space.
         */
        if (!vs->vs_dmap) {

-               kfree((vm_offset_t)vs, sizeof *vs);
+               kfree(vs, sizeof *vs);

                return VSTRUCT_NULL;
        }
 
                return VSTRUCT_NULL;
        }
 


@@ -1269,12 +1370,12 @@ ps_vstruct_create(
        return vs;
 }
 
        return vs;
 }
 

-paging_segment_t ps_select_segment(int, int *);        /* forward */
+paging_segment_t ps_select_segment(unsigned int, int *);       /* forward */

 
 paging_segment_t
 ps_select_segment(
 
 paging_segment_t
 ps_select_segment(

-       int     shift,
-       int     *psindex)
+       unsigned int    shift,
+       int             *psindex)

 {
        paging_segment_t        ps;
        int                     i;
 {
        paging_segment_t        ps;
        int                     i;


@@ -1287,20 +1388,21 @@ ps_select_segment(
 
        PSL_LOCK();
        if (paging_segment_count == 1) {
 
        PSL_LOCK();
        if (paging_segment_count == 1) {

-               paging_segment_t lps;   /* used to avoid extra PS_UNLOCK */
+               paging_segment_t lps = PAGING_SEGMENT_NULL;     /* used to avoid extra PS_UNLOCK */

                ipc_port_t trigger = IP_NULL;
 
                ps = paging_segments[paging_segment_max];
                *psindex = paging_segment_max;
                PS_LOCK(ps);
                ipc_port_t trigger = IP_NULL;
 
                ps = paging_segments[paging_segment_max];
                *psindex = paging_segment_max;
                PS_LOCK(ps);

-               if (ps->ps_going_away) {
-                       /* this segment is being turned off */
-                       lps = PAGING_SEGMENT_NULL;
-               } else {
-                       ASSERT(ps->ps_clshift >= shift);
+               if( !IS_PS_EMERGENCY_SEGMENT(ps) ) {
+                       panic("Emergency paging segment missing\n");
+               }
+               ASSERT(ps->ps_clshift >= shift);
+               if(IS_PS_OK_TO_USE(ps)) {

                        if (ps->ps_clcount) {
                                ps->ps_clcount--;
                                dp_pages_free -=  1 << ps->ps_clshift;
                        if (ps->ps_clcount) {
                                ps->ps_clcount--;
                                dp_pages_free -=  1 << ps->ps_clshift;

+                               ps->ps_pgcount -=  1 << ps->ps_clshift;

                                if(min_pages_trigger_port && 
                                  (dp_pages_free < minimum_pages_remaining)) {
                                        trigger = min_pages_trigger_port;
                                if(min_pages_trigger_port && 
                                  (dp_pages_free < minimum_pages_remaining)) {
                                        trigger = min_pages_trigger_port;


@@ -1308,10 +1410,21 @@ ps_select_segment(
                                        bs_low = TRUE;
                                }
                                lps = ps;
                                        bs_low = TRUE;
                                }
                                lps = ps;

-                       } else
-                               lps = PAGING_SEGMENT_NULL;
-               }
+                       } 
+               } 

                PS_UNLOCK(ps);
                PS_UNLOCK(ps);

+               
+               if( lps == PAGING_SEGMENT_NULL ) {
+                       if(dp_pages_free) {
+                               dp_pages_free_drift_count++;
+                               if(dp_pages_free > dp_pages_free_drifted_max) {
+                                       dp_pages_free_drifted_max = dp_pages_free;
+                               }
+                               dprintf(("Emergency swap segment:dp_pages_free before zeroing out: %d\n",dp_pages_free));
+                       }
+                       dp_pages_free = 0;
+               }
+

                PSL_UNLOCK();
 
                if (trigger != IP_NULL) {
                PSL_UNLOCK();
 
                if (trigger != IP_NULL) {


@@ -1322,6 +1435,14 @@ ps_select_segment(
        }
 
        if (paging_segment_count == 0) {
        }
 
        if (paging_segment_count == 0) {

+               if(dp_pages_free) {
+                       dp_pages_free_drift_count++;
+                       if(dp_pages_free > dp_pages_free_drifted_max) {
+                               dp_pages_free_drifted_max = dp_pages_free;
+                       }
+                       dprintf(("No paging segments:dp_pages_free before zeroing out: %d\n",dp_pages_free));
+               }
+               dp_pages_free = 0;

                PSL_UNLOCK();
                return PAGING_SEGMENT_NULL;
        }
                PSL_UNLOCK();
                return PAGING_SEGMENT_NULL;
        }


@@ -1363,35 +1484,36 @@ ps_select_segment(
                                 * >= that of the vstruct.
                                 */
                                PS_LOCK(ps);
                                 * >= that of the vstruct.
                                 */
                                PS_LOCK(ps);

-                               if (ps->ps_going_away) {
-                                       /* this segment is being turned off */
-                               } else if ((ps->ps_clcount) &&
-                                          (ps->ps_clshift >= shift)) {
-                                       ipc_port_t trigger = IP_NULL;
-
-                                       ps->ps_clcount--;
-                                       dp_pages_free -=  1 << ps->ps_clshift;
-                                       if(min_pages_trigger_port && 
-                                               (dp_pages_free < 
-                                               minimum_pages_remaining)) {
-                                               trigger = min_pages_trigger_port;
-                                               min_pages_trigger_port = NULL;
-                                       }
-                                       PS_UNLOCK(ps);
-                                       /*
-                                        * found one, quit looking.
-                                        */
-                                       ps_select_array[i] = j;
-                                       PSL_UNLOCK();
-                                       
-                                       if (trigger != IP_NULL) {
-                                               default_pager_space_alert(
-                                                       trigger,
-                                                       HI_WAT_ALERT);
-                                               ipc_port_release_send(trigger);
+                               if (IS_PS_OK_TO_USE(ps)) {
+                                       if ((ps->ps_clcount) &&
+                                                  (ps->ps_clshift >= shift)) {
+                                               ipc_port_t trigger = IP_NULL;
+
+                                               ps->ps_clcount--;
+                                               dp_pages_free -=  1 << ps->ps_clshift;
+                                               ps->ps_pgcount -=  1 << ps->ps_clshift;
+                                               if(min_pages_trigger_port && 
+                                                       (dp_pages_free < 
+                                                       minimum_pages_remaining)) {
+                                                       trigger = min_pages_trigger_port;
+                                                       min_pages_trigger_port = NULL;
+                                               }
+                                               PS_UNLOCK(ps);
+                                               /*
+                                                * found one, quit looking.
+                                                */
+                                               ps_select_array[i] = j;
+                                               PSL_UNLOCK();
+                                               
+                                               if (trigger != IP_NULL) {
+                                                       default_pager_space_alert(
+                                                               trigger,
+                                                               HI_WAT_ALERT);
+                                                       ipc_port_release_send(trigger);
+                                               }
+                                               *psindex = j;
+                                               return ps;

                                        }
                                        }

-                                       *psindex = j;
-                                       return ps;

                                }
                                PS_UNLOCK(ps);
                        }
                                }
                                PS_UNLOCK(ps);
                        }


@@ -1405,22 +1527,31 @@ ps_select_segment(
                        j++;
                }
        }
                        j++;
                }
        }

+       
+       if(dp_pages_free) {
+               dp_pages_free_drift_count++;
+               if(dp_pages_free > dp_pages_free_drifted_max) {
+                       dp_pages_free_drifted_max = dp_pages_free;
+               }
+               dprintf(("%d Paging Segments: dp_pages_free before zeroing out: %d\n",paging_segment_count,dp_pages_free));
+       }
+       dp_pages_free = 0;

        PSL_UNLOCK();
        return PAGING_SEGMENT_NULL;
 }
 
        PSL_UNLOCK();
        return PAGING_SEGMENT_NULL;
 }
 

-vm_offset_t ps_allocate_cluster(vstruct_t, int *, paging_segment_t); /*forward*/
+dp_offset_t ps_allocate_cluster(vstruct_t, int *, paging_segment_t); /*forward*/

 
 

-vm_offset_t
+dp_offset_t

 ps_allocate_cluster(
        vstruct_t               vs,
        int                     *psindex,
        paging_segment_t        use_ps)
 {
 ps_allocate_cluster(
        vstruct_t               vs,
        int                     *psindex,
        paging_segment_t        use_ps)
 {

-       int                     byte_num;
+       unsigned int            byte_num;

        int                     bit_num = 0;
        paging_segment_t        ps;
        int                     bit_num = 0;
        paging_segment_t        ps;

-       vm_offset_t             cluster;
+       dp_offset_t             cluster;

        ipc_port_t              trigger = IP_NULL;
 
        /*
        ipc_port_t              trigger = IP_NULL;
 
        /*


@@ -1446,6 +1577,7 @@ ps_allocate_cluster(
         * This and the ordering of the paging segment "going_away" bit setting
         * protects us.
         */
         * This and the ordering of the paging segment "going_away" bit setting
         * protects us.
         */

+retry:

        if (use_ps != PAGING_SEGMENT_NULL) {
                ps = use_ps;
                PSL_LOCK();
        if (use_ps != PAGING_SEGMENT_NULL) {
                ps = use_ps;
                PSL_LOCK();


@@ -1455,6 +1587,7 @@ ps_allocate_cluster(
 
                ps->ps_clcount--;
                dp_pages_free -=  1 << ps->ps_clshift;
 
                ps->ps_clcount--;
                dp_pages_free -=  1 << ps->ps_clshift;

+               ps->ps_pgcount -=  1 << ps->ps_clshift;

                if(min_pages_trigger_port && 
                                (dp_pages_free < minimum_pages_remaining)) {
                        trigger = min_pages_trigger_port;
                if(min_pages_trigger_port && 
                                (dp_pages_free < minimum_pages_remaining)) {
                        trigger = min_pages_trigger_port;


@@ -1469,17 +1602,88 @@ ps_allocate_cluster(
 
        } else if ((ps = ps_select_segment(vs->vs_clshift, psindex)) ==
                   PAGING_SEGMENT_NULL) {
 
        } else if ((ps = ps_select_segment(vs->vs_clshift, psindex)) ==
                   PAGING_SEGMENT_NULL) {

-#if 0
-               bs_no_paging_space(TRUE);
-#endif
-#if 0
-               if (verbose)
-#endif
-                       dprintf(("no space in available paging segments; "
-                                "swapon suggested\n"));
-               /* the count got off maybe, reset to zero */
+               static clock_sec_t lastnotify = 0;
+               clock_sec_t now;
+               clock_nsec_t nanoseconds_dummy;
+               
+               /* 
+                * Don't immediately jump to the emergency segment. Give the
+                * dynamic pager a chance to create it's first normal swap file.
+                * Unless, of course the very first normal swap file can't be 
+                * created due to some problem and we didn't expect that problem
+                * i.e. use_emergency_swap_file_first was never set to true initially.
+                * It then gets set in the swap file creation error handling.
+                */
+               if(paging_segment_count > 1 || use_emergency_swap_file_first == TRUE) {
+                       
+                       ps = paging_segments[EMERGENCY_PSEG_INDEX];
+                       if(IS_PS_EMERGENCY_SEGMENT(ps) && !IS_PS_GOING_AWAY(ps)) {
+                               PSL_LOCK();
+                               PS_LOCK(ps);
+                               
+                               if(IS_PS_GOING_AWAY(ps)) {
+                                       /* Someone de-activated the emergency paging segment*/
+                                       PS_UNLOCK(ps);
+                                       PSL_UNLOCK();
+
+                               } else if(dp_pages_free) {
+                                       /* 
+                                        * Someone has already activated the emergency paging segment 
+                                        * OR
+                                        * Between us having rec'd a NULL segment from ps_select_segment
+                                        * and reaching here a new normal segment could have been added.
+                                        * E.g. we get NULL segment and another thread just added the
+                                        * new swap file. Hence check to see if we have more dp_pages_free
+                                        * before activating the emergency segment.
+                                        */
+                                       PS_UNLOCK(ps);
+                                       PSL_UNLOCK();
+                                       goto retry;
+                               
+                               } else if(!IS_PS_OK_TO_USE(ps) && ps->ps_clcount) {
+                                       /*
+                                        * PS_CAN_USE is only reset from the emergency segment when it's
+                                        * been successfully recovered. So it's legal to have an emergency
+                                        * segment that has PS_CAN_USE but no clusters because it's recovery
+                                        * failed.
+                                        */
+                                       backing_store_t bs = ps->ps_bs;
+                                       ps->ps_state |= PS_CAN_USE;
+                                       if(ps_select_array[bs->bs_priority] == BS_FULLPRI ||
+                                               ps_select_array[bs->bs_priority] == BS_NOPRI) {
+                                               ps_select_array[bs->bs_priority] = 0;
+                                       }
+                                       dp_pages_free += ps->ps_pgcount;
+                                       dp_pages_reserve -= ps->ps_pgcount;
+                                       PS_UNLOCK(ps);
+                                       PSL_UNLOCK();
+                                       dprintf(("Switching ON Emergency paging segment\n"));
+                                       goto retry;
+                               }
+
+                               PS_UNLOCK(ps);
+                               PSL_UNLOCK();
+                       }
+               }
+               
+               /*
+                * Emit a notification of the low-paging resource condition
+                * but don't issue it more than once every five seconds.  This
+                * prevents us from overflowing logs with thousands of
+                * repetitions of the message.
+                */
+               clock_get_system_nanotime(&now, &nanoseconds_dummy);
+               if (paging_segment_count > 1 && (now > lastnotify + 5)) {
+                       /* With an activated emergency paging segment we still
+                        * didn't get any clusters. This could mean that the 
+                        * emergency paging segment is exhausted.
+                        */
+                       dprintf(("System is out of paging space.\n"));
+                       lastnotify = now;
+               }
+

                PSL_LOCK();
                PSL_LOCK();

-               dp_pages_free = 0;
+               

                if(min_pages_trigger_port) {
                        trigger = min_pages_trigger_port;
                        min_pages_trigger_port = NULL;
                if(min_pages_trigger_port) {
                        trigger = min_pages_trigger_port;
                        min_pages_trigger_port = NULL;


@@ -1490,7 +1694,7 @@ ps_allocate_cluster(
                        default_pager_space_alert(trigger, HI_WAT_ALERT);
                        ipc_port_release_send(trigger);
                }
                        default_pager_space_alert(trigger, HI_WAT_ALERT);
                        ipc_port_release_send(trigger);
                }

-               return (vm_offset_t) -1;
+               return (dp_offset_t) -1;

        }
 
        /*
        }
 
        /*


@@ -1522,15 +1726,15 @@ ps_allocate_cluster(
        return cluster;
 }
 
        return cluster;
 }
 

-void ps_deallocate_cluster(paging_segment_t, vm_offset_t);     /* forward */
+void ps_deallocate_cluster(paging_segment_t, dp_offset_t);     /* forward */

 
 void
 ps_deallocate_cluster(
        paging_segment_t        ps,
 
 void
 ps_deallocate_cluster(
        paging_segment_t        ps,

-       vm_offset_t             cluster)
+       dp_offset_t             cluster)

 {
 
 {
 

-       if (cluster >= (vm_offset_t) ps->ps_ncls)
+       if (cluster >= ps->ps_ncls)

                panic("ps_deallocate_cluster: Invalid cluster number");
 
        /*
                panic("ps_deallocate_cluster: Invalid cluster number");
 
        /*


@@ -1540,9 +1744,13 @@ ps_deallocate_cluster(
        PSL_LOCK();
        PS_LOCK(ps);
        clrbit(ps->ps_bmap, cluster);
        PSL_LOCK();
        PS_LOCK(ps);
        clrbit(ps->ps_bmap, cluster);

-       ++ps->ps_clcount;
-       dp_pages_free +=  1 << ps->ps_clshift;
-       PSL_UNLOCK();
+       if( IS_PS_OK_TO_USE(ps)) {
+               ++ps->ps_clcount;
+               ps->ps_pgcount +=  1 << ps->ps_clshift;
+               dp_pages_free +=  1 << ps->ps_clshift;
+       } else {
+               ps->ps_special_clusters += 1;
+       }

 
        /*
         * Move the hint down to the freed cluster if it is
 
        /*
         * Move the hint down to the freed cluster if it is


@@ -1552,27 +1760,26 @@ ps_deallocate_cluster(
                ps->ps_hint = (cluster/NBBY);
        }
 
                ps->ps_hint = (cluster/NBBY);
        }
 

-       PS_UNLOCK(ps);

 
        /*
         * If we're freeing space on a full priority, reset the array.
         */
 
        /*
         * If we're freeing space on a full priority, reset the array.
         */

-       PSL_LOCK();
-       if (ps_select_array[ps->ps_bs->bs_priority] == BS_FULLPRI)
+       if ( IS_PS_OK_TO_USE(ps) && ps_select_array[ps->ps_bs->bs_priority] == BS_FULLPRI)

                ps_select_array[ps->ps_bs->bs_priority] = 0;
                ps_select_array[ps->ps_bs->bs_priority] = 0;

+       PS_UNLOCK(ps);

        PSL_UNLOCK();
 
        return;
 }
 
        PSL_UNLOCK();
 
        return;
 }
 

-void ps_dealloc_vsmap(struct vs_map *, vm_size_t);     /* forward */
+void ps_dealloc_vsmap(struct vs_map *, dp_size_t);     /* forward */

 
 void
 ps_dealloc_vsmap(
        struct vs_map   *vsmap,
 
 void
 ps_dealloc_vsmap(
        struct vs_map   *vsmap,

-       vm_size_t       size)
+       dp_size_t       size)

 {
 {

-       int i;
+       unsigned int i;

        for (i = 0; i < size; i++)
                if (!VSM_ISCLR(vsmap[i]) && !VSM_ISERR(vsmap[i]))
                        ps_deallocate_cluster(VSM_PS(vsmap[i]),
        for (i = 0; i < size; i++)
                if (!VSM_ISCLR(vsmap[i]) && !VSM_ISERR(vsmap[i]))
                        ps_deallocate_cluster(VSM_PS(vsmap[i]),


@@ -1583,8 +1790,8 @@ void
 ps_vstruct_dealloc(
        vstruct_t vs)
 {
 ps_vstruct_dealloc(
        vstruct_t vs)
 {

-       int     i;
-       spl_t   s;
+       unsigned int    i;
+//     spl_t   s;

 
        VS_MAP_LOCK(vs);
 
 
        VS_MAP_LOCK(vs);
 


@@ -1601,31 +1808,29 @@ ps_vstruct_dealloc(
                for (i = 0; i < INDIRECT_CLMAP_ENTRIES(vs->vs_size); i++) {
                        if (vs->vs_imap[i] != NULL) {
                                ps_dealloc_vsmap(vs->vs_imap[i], CLMAP_ENTRIES);
                for (i = 0; i < INDIRECT_CLMAP_ENTRIES(vs->vs_size); i++) {
                        if (vs->vs_imap[i] != NULL) {
                                ps_dealloc_vsmap(vs->vs_imap[i], CLMAP_ENTRIES);

-                               kfree((vm_offset_t)vs->vs_imap[i], 
-                                                       CLMAP_THRESHOLD);
+                               kfree(vs->vs_imap[i], CLMAP_THRESHOLD);

                        }
                }
                        }
                }

-               kfree((vm_offset_t)vs->vs_imap, 
-                                       INDIRECT_CLMAP_SIZE(vs->vs_size));
+               kfree(vs->vs_imap, INDIRECT_CLMAP_SIZE(vs->vs_size));

        } else {
                /*
                 * Direct map.  Free used clusters, then memory.
                 */
                ps_dealloc_vsmap(vs->vs_dmap, vs->vs_size);
        } else {
                /*
                 * Direct map.  Free used clusters, then memory.
                 */
                ps_dealloc_vsmap(vs->vs_dmap, vs->vs_size);

-               kfree((vm_offset_t)vs->vs_dmap, CLMAP_SIZE(vs->vs_size));
+               kfree(vs->vs_dmap, CLMAP_SIZE(vs->vs_size));

        }
        VS_MAP_UNLOCK(vs);
 
        bs_commit(- vs->vs_size);
 
        }
        VS_MAP_UNLOCK(vs);
 
        bs_commit(- vs->vs_size);
 

-       zfree(vstruct_zone, (vm_offset_t)vs);
+       zfree(vstruct_zone, vs);

 }
 
 }
 

-int ps_map_extend(vstruct_t, int);     /* forward */
+int ps_map_extend(vstruct_t, unsigned int);    /* forward */

 
 int ps_map_extend(
        vstruct_t       vs,
 
 int ps_map_extend(
        vstruct_t       vs,

-       int             new_size)
+       unsigned int    new_size)

 {
        struct vs_map   **new_imap;
        struct vs_map   *new_dmap = NULL;
 {
        struct vs_map   **new_imap;
        struct vs_map   *new_dmap = NULL;


@@ -1687,7 +1892,7 @@ int ps_map_extend(
                        /* Allocate an indirect page */
                        if ((new_imap[0] = (struct vs_map *)
                             kalloc(CLMAP_THRESHOLD)) == NULL) {
                        /* Allocate an indirect page */
                        if ((new_imap[0] = (struct vs_map *)
                             kalloc(CLMAP_THRESHOLD)) == NULL) {

-                               kfree((vm_offset_t)new_imap, new_map_size);
+                               kfree(new_imap, new_map_size);

                                return -1;
                        }
                        new_dmap = new_imap[0];
                                return -1;
                        }
                        new_dmap = new_imap[0];


@@ -1728,23 +1933,23 @@ int ps_map_extend(
        bs_commit(new_size - vs->vs_size);
        vs->vs_size = new_size;
        if (old_map)
        bs_commit(new_size - vs->vs_size);
        vs->vs_size = new_size;
        if (old_map)

-               kfree((vm_offset_t)old_map, old_map_size);
+               kfree(old_map, old_map_size);

        return 0;
 }
 
        return 0;
 }
 

-vm_offset_t
+dp_offset_t

 ps_clmap(
        vstruct_t       vs,
 ps_clmap(
        vstruct_t       vs,

-       vm_offset_t     offset,
+       dp_offset_t     offset,

        struct clmap    *clmap,
        int             flag,
        struct clmap    *clmap,
        int             flag,

-       vm_size_t       size,
+       dp_size_t       size,

        int             error)
 {
        int             error)
 {

-       vm_offset_t     cluster;        /* The cluster of offset.       */
-       vm_offset_t     newcl;          /* The new cluster allocated.   */
-       vm_offset_t     newoff;
-       int             i;
+       dp_offset_t     cluster;        /* The cluster of offset.       */
+       dp_offset_t     newcl;          /* The new cluster allocated.   */
+       dp_offset_t     newoff;
+       unsigned int    i;

        struct vs_map   *vsmap;
 
        VS_MAP_LOCK(vs);
        struct vs_map   *vsmap;
 
        VS_MAP_LOCK(vs);


@@ -1764,11 +1969,11 @@ ps_clmap(
                if (flag == CL_FIND) {
                        /* Do not allocate if just doing a lookup */
                        VS_MAP_UNLOCK(vs);
                if (flag == CL_FIND) {
                        /* Do not allocate if just doing a lookup */
                        VS_MAP_UNLOCK(vs);

-                       return (vm_offset_t) -1;
+                       return (dp_offset_t) -1;

                }
                if (ps_map_extend(vs, cluster + 1)) {
                        VS_MAP_UNLOCK(vs);
                }
                if (ps_map_extend(vs, cluster + 1)) {
                        VS_MAP_UNLOCK(vs);

-                       return (vm_offset_t) -1;
+                       return (dp_offset_t) -1;

                }
        }
 
                }
        }
 


@@ -1790,14 +1995,14 @@ ps_clmap(
                if (vsmap == NULL) {
                        if (flag == CL_FIND) {
                                VS_MAP_UNLOCK(vs);
                if (vsmap == NULL) {
                        if (flag == CL_FIND) {
                                VS_MAP_UNLOCK(vs);

-                               return (vm_offset_t) -1;
+                               return (dp_offset_t) -1;

                        }
 
                        /* Allocate the indirect block */
                        vsmap = (struct vs_map *) kalloc(CLMAP_THRESHOLD);
                        if (vsmap == NULL) {
                                VS_MAP_UNLOCK(vs);
                        }
 
                        /* Allocate the indirect block */
                        vsmap = (struct vs_map *) kalloc(CLMAP_THRESHOLD);
                        if (vsmap == NULL) {
                                VS_MAP_UNLOCK(vs);

-                               return (vm_offset_t) -1;
+                               return (dp_offset_t) -1;

                        }
                        /* Initialize the cluster offsets */
                        for (i = 0; i < CLMAP_ENTRIES; i++)
                        }
                        /* Initialize the cluster offsets */
                        for (i = 0; i < CLMAP_ENTRIES; i++)


@@ -1821,7 +2026,7 @@ ps_clmap(
        if (VSM_ISERR(*vsmap)) {
                clmap->cl_error = VSM_GETERR(*vsmap);
                VS_MAP_UNLOCK(vs);
        if (VSM_ISERR(*vsmap)) {
                clmap->cl_error = VSM_GETERR(*vsmap);
                VS_MAP_UNLOCK(vs);

-               return (vm_offset_t) -1;
+               return (dp_offset_t) -1;

        } else if (VSM_ISCLR(*vsmap)) {
                int psindex;
 
        } else if (VSM_ISCLR(*vsmap)) {
                int psindex;
 


@@ -1835,16 +2040,16 @@ ps_clmap(
                                VSM_SETERR(*vsmap, error);
                        }
                        VS_MAP_UNLOCK(vs);
                                VSM_SETERR(*vsmap, error);
                        }
                        VS_MAP_UNLOCK(vs);

-                       return (vm_offset_t) -1;
+                       return (dp_offset_t) -1;

                } else {
                        /*
                         * Attempt to allocate a cluster from the paging segment
                         */
                        newcl = ps_allocate_cluster(vs, &psindex,
                                                    PAGING_SEGMENT_NULL);
                } else {
                        /*
                         * Attempt to allocate a cluster from the paging segment
                         */
                        newcl = ps_allocate_cluster(vs, &psindex,
                                                    PAGING_SEGMENT_NULL);

-                       if (newcl == -1) {
+                       if (newcl == (dp_offset_t) -1) {

                                VS_MAP_UNLOCK(vs);
                                VS_MAP_UNLOCK(vs);

-                               return (vm_offset_t) -1;
+                               return (dp_offset_t) -1;

                        }
                        VSM_CLR(*vsmap);
                        VSM_SETCLOFF(*vsmap, newcl);
                        }
                        VSM_CLR(*vsmap);
                        VSM_SETCLOFF(*vsmap, newcl);


@@ -1903,7 +2108,7 @@ ps_clmap(
         * entire cluster is in error.
         */
        if (size && flag == CL_FIND) {
         * entire cluster is in error.
         */
        if (size && flag == CL_FIND) {

-               vm_offset_t off = (vm_offset_t) 0;
+               dp_offset_t off = (dp_offset_t) 0;

 
                if (!error) {
                        for (i = VSCLSIZE(vs) - clmap->cl_numpages; size > 0;
 
                if (!error) {
                        for (i = VSCLSIZE(vs) - clmap->cl_numpages; size > 0;


@@ -1923,33 +2128,33 @@ ps_clmap(
                 * Deallocate cluster if error, and no valid pages
                 * already present.
                 */
                 * Deallocate cluster if error, and no valid pages
                 * already present.
                 */

-               if (off != (vm_offset_t) 0)
+               if (off != (dp_offset_t) 0)

                        ps_deallocate_cluster(clmap->cl_ps, off);
                VS_MAP_UNLOCK(vs);
                        ps_deallocate_cluster(clmap->cl_ps, off);
                VS_MAP_UNLOCK(vs);

-               return (vm_offset_t) 0;
+               return (dp_offset_t) 0;

        } else
                VS_MAP_UNLOCK(vs);
 
        } else
                VS_MAP_UNLOCK(vs);
 

-       DEBUG(DEBUG_VS_INTERNAL,
-             ("returning 0x%X,vs=0x%X,vsmap=0x%X,flag=%d\n",
-              newcl+newoff, (int) vs, (int) vsmap, flag));
-       DEBUG(DEBUG_VS_INTERNAL,
-             ("        clmap->cl_ps=0x%X,cl_numpages=%d,clbmap=0x%x,cl_alloc=%x\n",
-              (int) clmap->cl_ps, clmap->cl_numpages,
-              (int) clmap->cl_bmap.clb_map, (int) clmap->cl_alloc.clb_map));
+       DP_DEBUG(DEBUG_VS_INTERNAL,
+                ("returning 0x%X,vs=0x%X,vsmap=0x%X,flag=%d\n",
+                 newcl+newoff, (int) vs, (int) vsmap, flag));
+       DP_DEBUG(DEBUG_VS_INTERNAL,
+                ("     clmap->cl_ps=0x%X,cl_numpages=%d,clbmap=0x%x,cl_alloc=%x\n",
+                 (int) clmap->cl_ps, clmap->cl_numpages,
+                 (int) clmap->cl_bmap.clb_map, (int) clmap->cl_alloc.clb_map));

 
        return (newcl + newoff);
 }
 
 
        return (newcl + newoff);
 }
 

-void ps_clunmap(vstruct_t, vm_offset_t, vm_size_t);    /* forward */
+void ps_clunmap(vstruct_t, dp_offset_t, dp_size_t);    /* forward */

 
 void
 ps_clunmap(
        vstruct_t       vs,
 
 void
 ps_clunmap(
        vstruct_t       vs,

-       vm_offset_t     offset,
-       vm_size_t       length)
+       dp_offset_t     offset,
+       dp_size_t       length)

 {
 {

-       vm_offset_t             cluster; /* The cluster number of offset */
+       dp_offset_t             cluster; /* The cluster number of offset */

        struct vs_map           *vsmap;
 
        VS_MAP_LOCK(vs);
        struct vs_map           *vsmap;
 
        VS_MAP_LOCK(vs);


@@ -1959,8 +2164,8 @@ ps_clunmap(
         * clusters and map entries as encountered.
         */
        while (length > 0) {
         * clusters and map entries as encountered.
         */
        while (length > 0) {

-               vm_offset_t     newoff;
-               int             i;
+               dp_offset_t     newoff;
+               unsigned int    i;

 
                cluster = atop_32(offset) >> vs->vs_clshift;
                if (vs->vs_indirect)    /* indirect map */
 
                cluster = atop_32(offset) >> vs->vs_clshift;
                if (vs->vs_indirect)    /* indirect map */


@@ -1982,7 +2187,7 @@ ps_clunmap(
                 * paging segment cluster pages.
                 * Optimize for entire cluster cleraing.
                 */
                 * paging segment cluster pages.
                 * Optimize for entire cluster cleraing.
                 */

-               if (newoff = (offset&((1<<(vm_page_shift+vs->vs_clshift))-1))) {
+               if ( (newoff = (offset&((1<<(vm_page_shift+vs->vs_clshift))-1))) ) {

                        /*
                         * Not cluster aligned.
                         */
                        /*
                         * Not cluster aligned.
                         */


@@ -2011,13 +2216,13 @@ ps_clunmap(
        VS_MAP_UNLOCK(vs);
 }
 
        VS_MAP_UNLOCK(vs);
 }
 

-void ps_vs_write_complete(vstruct_t, vm_offset_t, vm_size_t, int); /* forward */
+void ps_vs_write_complete(vstruct_t, dp_offset_t, dp_size_t, int); /* forward */

 
 void
 ps_vs_write_complete(
        vstruct_t       vs,
 
 void
 ps_vs_write_complete(
        vstruct_t       vs,

-       vm_offset_t     offset,
-       vm_size_t       size,
+       dp_offset_t     offset,
+       dp_size_t       size,

        int             error)
 {
        struct clmap    clmap;
        int             error)
 {
        struct clmap    clmap;


@@ -2035,19 +2240,19 @@ ps_vs_write_complete(
        (void) ps_clmap(vs, offset, &clmap, CL_FIND, size, error);
 }
 
        (void) ps_clmap(vs, offset, &clmap, CL_FIND, size, error);
 }
 

-void vs_cl_write_complete(vstruct_t, paging_segment_t, vm_offset_t, vm_offset_t, vm_size_t, boolean_t, int);   /* forward */
+void vs_cl_write_complete(vstruct_t, paging_segment_t, dp_offset_t, vm_offset_t, dp_size_t, boolean_t, int);   /* forward */

 
 void
 vs_cl_write_complete(
 
 void
 vs_cl_write_complete(

-       vstruct_t               vs,
-       paging_segment_t        ps,
-       vm_offset_t             offset,
-       vm_offset_t             addr,
-       vm_size_t               size,
-       boolean_t               async,
-       int                     error)
+       vstruct_t                       vs,
+       __unused paging_segment_t       ps,
+       dp_offset_t                     offset,
+       __unused vm_offset_t            addr,
+       dp_size_t                       size,
+       boolean_t                       async,
+       int                             error)

 {
 {

-       kern_return_t   kr;
+//     kern_return_t   kr;

 
        if (error) {
                /*
 
        if (error) {
                /*


@@ -2071,7 +2276,6 @@ vs_cl_write_complete(
                if (vs->vs_async_pending == 0 && vs->vs_waiting_async) {
                        vs->vs_waiting_async = FALSE;
                        VS_UNLOCK(vs);
                if (vs->vs_async_pending == 0 && vs->vs_waiting_async) {
                        vs->vs_waiting_async = FALSE;
                        VS_UNLOCK(vs);

-                       /* mutex_unlock(&vs->vs_waiting_async); */

                        thread_wakeup(&vs->vs_async_pending);
                } else {
                        VS_UNLOCK(vs);
                        thread_wakeup(&vs->vs_async_pending);
                } else {
                        VS_UNLOCK(vs);


@@ -2106,11 +2310,7 @@ device_write_reply(
                if(vsa->vsa_error) {
                   /* need to consider error condition.  re-write data or */
                   /* throw it away here. */
                if(vsa->vsa_error) {
                   /* need to consider error condition.  re-write data or */
                   /* throw it away here. */

-                  vm_offset_t  ioaddr;
-                  if(vm_map_copyout(kernel_map, &ioaddr,
-                                (vm_map_copy_t)vsa->vsa_addr) != KERN_SUCCESS)
-                  panic("vs_cluster_write: unable to copy source list\n");
-                  vm_deallocate(kernel_map, ioaddr, vsa->vsa_size);
+                  vm_map_copy_discard((vm_map_copy_t)vsa->vsa_addr);

                }
                ps_vs_write_complete(vsa->vsa_vs, vsa->vsa_offset, 
                                                vsa->vsa_size, vsa->vsa_error);
                }
                ps_vs_write_complete(vsa->vsa_vs, vsa->vsa_offset, 
                                                vsa->vsa_size, vsa->vsa_error);


@@ -2149,7 +2349,7 @@ device_read_reply(
        vsa->vsa_addr = (vm_offset_t)data;
        vsa->vsa_size = (vm_size_t)dataCnt;
        vsa->vsa_error = return_code;
        vsa->vsa_addr = (vm_offset_t)data;
        vsa->vsa_size = (vm_size_t)dataCnt;
        vsa->vsa_error = return_code;

-       thread_wakeup(&vsa->vsa_lock);
+       thread_wakeup(&vsa);

        return KERN_SUCCESS;
 }
 
        return KERN_SUCCESS;
 }
 


@@ -2187,12 +2387,10 @@ device_open_reply(
        return KERN_SUCCESS;
 }
 
        return KERN_SUCCESS;
 }
 

-kern_return_t ps_read_device(paging_segment_t, vm_offset_t, vm_offset_t *, unsigned int, unsigned int *, int); /* forward */
-

 kern_return_t
 ps_read_device(
        paging_segment_t        ps,
 kern_return_t
 ps_read_device(
        paging_segment_t        ps,

-       vm_offset_t             offset,
+       dp_offset_t             offset,

        vm_offset_t             *bufferp,
        unsigned int            size,
        unsigned int            *residualp,
        vm_offset_t             *bufferp,
        unsigned int            size,
        unsigned int            *residualp,


@@ -2207,7 +2405,6 @@ ps_read_device(
        vm_offset_t     buf_ptr;
        unsigned int    records_read;
        struct vs_async *vsa;   
        vm_offset_t     buf_ptr;
        unsigned int    records_read;
        struct vs_async *vsa;   

-       mutex_t vs_waiting_read_reply;

 
        device_t        device;
        vm_map_copy_t   device_data = NULL;
 
        device_t        device;
        vm_map_copy_t   device_data = NULL;


@@ -2231,7 +2428,6 @@ ps_read_device(
                        vsa->vsa_size = 0;
                        vsa->vsa_ps = NULL;
                }
                        vsa->vsa_size = 0;
                        vsa->vsa_ps = NULL;
                }

-               mutex_init(&vsa->vsa_lock, ETAP_DPAGE_VSSEQNO);

                ip_lock(vsa->reply_port);
                vsa->reply_port->ip_sorights++;
                ip_reference(vsa->reply_port);
                ip_lock(vsa->reply_port);
                vsa->reply_port->ip_sorights++;
                ip_reference(vsa->reply_port);


@@ -2247,7 +2443,7 @@ ps_read_device(
                                 (io_buf_ptr_t *) &dev_buffer,
                                 (mach_msg_type_number_t *) &bytes_read);
                if(kr == MIG_NO_REPLY) { 
                                 (io_buf_ptr_t *) &dev_buffer,
                                 (mach_msg_type_number_t *) &bytes_read);
                if(kr == MIG_NO_REPLY) { 

-                       assert_wait(&vsa->vsa_lock, THREAD_UNINT);
+                       assert_wait(&vsa, THREAD_UNINT);

                        thread_block(THREAD_CONTINUE_NULL);
 
                        dev_buffer = vsa->vsa_addr;
                        thread_block(THREAD_CONTINUE_NULL);
 
                        dev_buffer = vsa->vsa_addr;


@@ -2287,9 +2483,9 @@ ps_read_device(
                records_read = (bytes_read >>
                                (vm_page_shift - ps->ps_record_shift));
                dev_offset += records_read;
                records_read = (bytes_read >>
                                (vm_page_shift - ps->ps_record_shift));
                dev_offset += records_read;

-               DEBUG(DEBUG_VS_INTERNAL,
-                     ("calling vm_deallocate(addr=0x%X,size=0x%X)\n",
-                      dev_buffer, bytes_read));
+               DP_DEBUG(DEBUG_VS_INTERNAL,
+                        ("calling vm_deallocate(addr=0x%X,size=0x%X)\n",
+                         dev_buffer, bytes_read));

                if (vm_deallocate(kernel_map, dev_buffer, bytes_read)
                    != KERN_SUCCESS)
                        Panic("dealloc buf");
                if (vm_deallocate(kernel_map, dev_buffer, bytes_read)
                    != KERN_SUCCESS)
                        Panic("dealloc buf");


@@ -2298,7 +2494,7 @@ ps_read_device(
        *residualp = size - total_read;
        if((dev_buffer != *bufferp) && (total_read != 0)) {
                vm_offset_t temp_buffer;
        *residualp = size - total_read;
        if((dev_buffer != *bufferp) && (total_read != 0)) {
                vm_offset_t temp_buffer;

-               vm_allocate(kernel_map, &temp_buffer, total_read, TRUE);
+               vm_allocate(kernel_map, &temp_buffer, total_read, VM_FLAGS_ANYWHERE);

                memcpy((void *) temp_buffer, (void *) *bufferp, total_read);
                if(vm_map_copyin_page_list(kernel_map, temp_buffer, total_read, 
                        VM_MAP_COPYIN_OPT_SRC_DESTROY | 
                memcpy((void *) temp_buffer, (void *) *bufferp, total_read);
                if(vm_map_copyin_page_list(kernel_map, temp_buffer, total_read, 
                        VM_MAP_COPYIN_OPT_SRC_DESTROY | 


@@ -2328,12 +2524,10 @@ ps_read_device(
        return KERN_SUCCESS;
 }
 
        return KERN_SUCCESS;
 }
 

-kern_return_t ps_write_device(paging_segment_t, vm_offset_t, vm_offset_t, unsigned int, struct vs_async *);    /* forward */
-

 kern_return_t
 ps_write_device(
        paging_segment_t        ps,
 kern_return_t
 ps_write_device(
        paging_segment_t        ps,

-       vm_offset_t             offset,
+       dp_offset_t             offset,

        vm_offset_t             addr,
        unsigned int            size,
        struct vs_async         *vsa)
        vm_offset_t             addr,
        unsigned int            size,
        struct vs_async         *vsa)


@@ -2435,40 +2629,43 @@ ps_write_device(
 
 kern_return_t
 ps_read_device(
 
 kern_return_t
 ps_read_device(

-       paging_segment_t        ps,
-       vm_offset_t             offset,
-       vm_offset_t             *bufferp,
-       unsigned int            size,
-       unsigned int            *residualp,
-       int                     flags)
+       __unused paging_segment_t       ps,
+       __unused dp_offset_t            offset,
+       __unused vm_offset_t            *bufferp,
+       __unused unsigned int           size,
+       __unused unsigned int           *residualp,
+       __unused int                            flags)

 {
   panic("ps_read_device not supported");
 {
   panic("ps_read_device not supported");

+  return KERN_FAILURE;

 }
 
 }
 

+kern_return_t

 ps_write_device(
 ps_write_device(

-       paging_segment_t        ps,
-       vm_offset_t             offset,
-       vm_offset_t             addr,
-       unsigned int            size,
-       struct vs_async         *vsa)
+       __unused paging_segment_t       ps,
+       __unused dp_offset_t            offset,
+       __unused vm_offset_t            addr,
+       __unused unsigned int           size,
+       __unused struct vs_async        *vsa)

 {
   panic("ps_write_device not supported");
 {
   panic("ps_write_device not supported");

+  return KERN_FAILURE;

 }
 
 #endif /* DEVICE_PAGING */
 }
 
 #endif /* DEVICE_PAGING */

-void pvs_object_data_provided(vstruct_t, upl_t, vm_offset_t, vm_size_t);       /* forward */
+void pvs_object_data_provided(vstruct_t, upl_t, upl_offset_t, upl_size_t);     /* forward */

 
 void
 pvs_object_data_provided(
 
 void
 pvs_object_data_provided(

-       vstruct_t       vs,
-       upl_t           upl,
-       vm_offset_t     offset,
-       vm_size_t       size)
+       __unused vstruct_t              vs,
+       __unused upl_t                  upl,
+       __unused upl_offset_t   offset,
+       upl_size_t                              size)

 {
 
 {
 

-       DEBUG(DEBUG_VS_INTERNAL,
-             ("buffer=0x%x,offset=0x%x,size=0x%x\n",
-              upl, offset, size));
+       DP_DEBUG(DEBUG_VS_INTERNAL,
+                ("buffer=0x%x,offset=0x%x,size=0x%x\n",
+                 upl, offset, size));

 
        ASSERT(size > 0);
        GSTAT(global_stats.gs_pages_in += atop_32(size));
 
        ASSERT(size > 0);
        GSTAT(global_stats.gs_pages_in += atop_32(size));


@@ -2480,77 +2677,156 @@ pvs_object_data_provided(
 
 }
 
 
 }
 

+static memory_object_offset_t   last_start;
+static vm_size_t               last_length;
+

 kern_return_t
 pvs_cluster_read(
        vstruct_t       vs,
 kern_return_t
 pvs_cluster_read(
        vstruct_t       vs,

-       vm_offset_t     vs_offset,
-       vm_size_t       cnt)
+       dp_offset_t     vs_offset,
+       dp_size_t       cnt,
+        void           *fault_info)

 {
 {

-       upl_t                   upl;

        kern_return_t           error = KERN_SUCCESS;
        kern_return_t           error = KERN_SUCCESS;

-       int                     size;
+       unsigned int            size;

        unsigned int            residual;
        unsigned int            request_flags;
        unsigned int            residual;
        unsigned int            request_flags;

-       int                     seg_index;
-       int                     pages_in_cl;
+       int                     io_flags = 0;
+       int                     seg_index;
+       int                     pages_in_cl;

        int                     cl_size;
        int                     cl_mask;
        int                     cl_size;
        int                     cl_mask;

-       int                     cl_index;
-       int                     xfer_size;
-       vm_offset_t       ps_offset[(VM_SUPER_CLUSTER / PAGE_SIZE) >> VSTRUCT_DEF_CLSHIFT];
-       paging_segment_t        psp[(VM_SUPER_CLUSTER / PAGE_SIZE) >> VSTRUCT_DEF_CLSHIFT];
+       int                     cl_index;
+       unsigned int            xfer_size;
+       dp_offset_t             orig_vs_offset;
+       dp_offset_t       ps_offset[(VM_SUPER_CLUSTER / PAGE_SIZE) >> VSTRUCT_MIN_CLSHIFT];
+       paging_segment_t        psp[(VM_SUPER_CLUSTER / PAGE_SIZE) >> VSTRUCT_MIN_CLSHIFT];

        struct clmap            clmap;
        struct clmap            clmap;

+       upl_t                   upl;
+       unsigned int            page_list_count;
+       memory_object_offset_t  cluster_start;
+       vm_size_t               cluster_length;
+       uint32_t                io_streaming;

 
        pages_in_cl = 1 << vs->vs_clshift;
        cl_size = pages_in_cl * vm_page_size;
        cl_mask = cl_size - 1;
 
 
        pages_in_cl = 1 << vs->vs_clshift;
        cl_size = pages_in_cl * vm_page_size;
        cl_mask = cl_size - 1;
 

+#if    USE_PRECIOUS
+       request_flags = UPL_NO_SYNC |  UPL_CLEAN_IN_PLACE | UPL_PRECIOUS | UPL_RET_ONLY_ABSENT | UPL_SET_LITE;
+#else
+       request_flags = UPL_NO_SYNC |  UPL_CLEAN_IN_PLACE | UPL_RET_ONLY_ABSENT | UPL_SET_LITE;
+#endif
+       cl_index = (vs_offset & cl_mask) / vm_page_size;
+
+        if ((ps_clmap(vs, vs_offset & ~cl_mask, &clmap, CL_FIND, 0, 0) == (dp_offset_t)-1) ||
+           !CLMAP_ISSET(clmap, cl_index)) {
+               /*
+                * the needed page doesn't exist in the backing store...
+                * we don't want to try to do any I/O, just abort the
+                * page and let the fault handler provide a zero-fill
+                */
+               if (cnt == 0) {
+                       /*
+                        * The caller was just poking at us to see if
+                        * the page has been paged out.  No need to 
+                        * mess with the page at all.
+                        * Just let the caller know we don't have that page.
+                        */
+                       return KERN_FAILURE;
+               }
+
+               page_list_count = 0;
+
+               memory_object_super_upl_request(vs->vs_control, (memory_object_offset_t)vs_offset,
+                                               PAGE_SIZE, PAGE_SIZE, 
+                                               &upl, NULL, &page_list_count,
+                                               request_flags);
+
+               if (clmap.cl_error)
+                       upl_abort(upl, UPL_ABORT_ERROR);
+               else
+                       upl_abort(upl, UPL_ABORT_UNAVAILABLE);
+               upl_deallocate(upl);
+
+               return KERN_SUCCESS;
+       }
+
+       if (cnt == 0) {
+               /*
+                * The caller was just poking at us to see if
+                * the page has been paged out.  No need to 
+                * mess with the page at all.
+                * Just let the caller know we do have that page.
+                */
+               return KERN_SUCCESS;
+       }
+               
+       assert(dp_encryption_inited);
+       if (dp_encryption) {
+               /*
+                * ENCRYPTED SWAP:
+                * request that the UPL be prepared for
+                * decryption.
+                */
+               request_flags |= UPL_ENCRYPT;
+       }
+       orig_vs_offset = vs_offset;
+
+       assert(cnt != 0);
+       cnt = VM_SUPER_CLUSTER;
+       cluster_start = (memory_object_offset_t) vs_offset;
+       cluster_length = (vm_size_t) cnt;
+       io_streaming = 0;
+
+       /*
+        * determine how big a speculative I/O we should try for...
+        */
+       if (memory_object_cluster_size(vs->vs_control, &cluster_start, &cluster_length, &io_streaming, (memory_object_fault_info_t)fault_info) == KERN_SUCCESS) {
+               assert(vs_offset >= (dp_offset_t) cluster_start &&
+                      vs_offset < (dp_offset_t) (cluster_start + cluster_length));
+               vs_offset = (dp_offset_t) cluster_start;
+               cnt = (dp_size_t) cluster_length;
+       } else {
+               cluster_length = PAGE_SIZE;
+               cnt = PAGE_SIZE;
+       }
+
+       if (io_streaming)
+                io_flags |= UPL_IOSTREAMING;
+
+       last_start = cluster_start;
+       last_length = cluster_length;
+

        /*
         * This loop will be executed multiple times until the entire
        /*
         * This loop will be executed multiple times until the entire

-        * request has been satisfied... if the request spans cluster
+        * range has been looked at or we issue an I/O... if the request spans cluster

         * boundaries, the clusters will be checked for logical continunity,
         * boundaries, the clusters will be checked for logical continunity,

-        * if contiguous the I/O request will span multiple clusters, otherwise
-        * it will be broken up into the minimal set of I/O's
-        *
-        * If there are holes in a request (either unallocated pages in a paging
-        * segment or an unallocated paging segment), we stop
-        * reading at the hole, inform the VM of any data read, inform
-        * the VM of an unavailable range, then loop again, hoping to
-        * find valid pages later in the requested range.  This continues until
-        * the entire range has been examined, and read, if present.
+        * if contiguous the I/O request will span multiple clusters...
+        * at most only 1 I/O will be issued... it will encompass the original offset

         */
         */

-
-#if    USE_PRECIOUS
-       request_flags = UPL_NO_SYNC |  UPL_CLEAN_IN_PLACE | UPL_PRECIOUS | UPL_RET_ONLY_ABSENT;
-#else
-       request_flags = UPL_NO_SYNC |  UPL_CLEAN_IN_PLACE | UPL_RET_ONLY_ABSENT;
-#endif
-       while (cnt && (error == KERN_SUCCESS)) {
+       while (cnt && error == KERN_SUCCESS) {

                int     ps_info_valid;
                int     ps_info_valid;

-               int     page_list_count;

 
 

-               if((vs_offset & cl_mask) && 
-                       (cnt > (VM_SUPER_CLUSTER - 
-                               (vs_offset & cl_mask)))) {
+               if ((vs_offset & cl_mask) && (cnt > (VM_SUPER_CLUSTER - (vs_offset & cl_mask)))) {

                        size = VM_SUPER_CLUSTER;
                        size -= vs_offset & cl_mask;
                        size = VM_SUPER_CLUSTER;
                        size -= vs_offset & cl_mask;

-               } else if (cnt > VM_SUPER_CLUSTER) {
+               } else if (cnt > VM_SUPER_CLUSTER)

                        size = VM_SUPER_CLUSTER;
                        size = VM_SUPER_CLUSTER;

-               } else {
+               else

                        size = cnt;
                        size = cnt;

-               }
+

                cnt -= size;
 
                ps_info_valid = 0;
                seg_index     = 0;
 
                while (size > 0 && error == KERN_SUCCESS) {
                cnt -= size;
 
                ps_info_valid = 0;
                seg_index     = 0;
 
                while (size > 0 && error == KERN_SUCCESS) {

-                       int           abort_size;
+                       unsigned int  abort_size;

                        int           failed_size;
                        int           beg_pseg;
                        int           beg_indx;
                        int           failed_size;
                        int           beg_pseg;
                        int           beg_indx;

-                       vm_offset_t   cur_offset;
-
+                       dp_offset_t   cur_offset;

 
                        if ( !ps_info_valid) {
                                ps_offset[seg_index] = ps_clmap(vs, vs_offset & ~cl_mask, &clmap, CL_FIND, 0, 0);
 
                        if ( !ps_info_valid) {
                                ps_offset[seg_index] = ps_clmap(vs, vs_offset & ~cl_mask, &clmap, CL_FIND, 0, 0);


@@ -2560,30 +2836,16 @@ pvs_cluster_read(
                        /*
                         * skip over unallocated physical segments 
                         */
                        /*
                         * skip over unallocated physical segments 
                         */

-                       if (ps_offset[seg_index] == (vm_offset_t) -1) {
+                       if (ps_offset[seg_index] == (dp_offset_t) -1) {

                                abort_size = cl_size - (vs_offset & cl_mask);
                                abort_size = MIN(abort_size, size);
 
                                abort_size = cl_size - (vs_offset & cl_mask);
                                abort_size = MIN(abort_size, size);
 

-                               page_list_count = 0;
-                               memory_object_super_upl_request(
-                                       vs->vs_control,
-                                       (memory_object_offset_t)vs_offset,
-                                       abort_size, abort_size, 
-                                       &upl, NULL, &page_list_count,
-                                       request_flags);
-
-                               if (clmap.cl_error) {
-                                       upl_abort(upl, UPL_ABORT_ERROR);
-                               } else {
-                                       upl_abort(upl, UPL_ABORT_UNAVAILABLE);
-                               }
-                               upl_deallocate(upl);
-
-                               size       -= abort_size;
-                               vs_offset  += abort_size;
+                               size      -= abort_size;
+                               vs_offset += abort_size;

 
                                seg_index++;
                                ps_info_valid = 0;
 
                                seg_index++;
                                ps_info_valid = 0;

+

                                continue;
                        }
                        cl_index = (vs_offset & cl_mask) / vm_page_size;
                                continue;
                        }
                        cl_index = (vs_offset & cl_mask) / vm_page_size;


@@ -2597,24 +2859,8 @@ pvs_cluster_read(
                                abort_size += vm_page_size;
                        }
                        if (abort_size) {
                                abort_size += vm_page_size;
                        }
                        if (abort_size) {

-                               /*
-                                * Let VM system know about holes in clusters.
-                                */
-                               GSTAT(global_stats.gs_pages_unavail += atop_32(abort_size));
-
-                               page_list_count = 0;
-                               memory_object_super_upl_request(
-                                       vs->vs_control,
-                                       (memory_object_offset_t)vs_offset,
-                                       abort_size, abort_size, 
-                                       &upl, NULL, &page_list_count,
-                                       request_flags);
-
-                               upl_abort(upl, UPL_ABORT_UNAVAILABLE);
-                               upl_deallocate(upl);
-
-                               size       -= abort_size;
-                               vs_offset  += abort_size;
+                               size      -= abort_size;
+                               vs_offset += abort_size;

 
                                if (cl_index == pages_in_cl) {
                                        /*
 
                                if (cl_index == pages_in_cl) {
                                        /*


@@ -2623,6 +2869,7 @@ pvs_cluster_read(
                                         */
                                        seg_index++;
                                        ps_info_valid = 0;
                                         */
                                        seg_index++;
                                        ps_info_valid = 0;

+

                                        continue;
                                }
                                if (size == 0)
                                        continue;
                                }
                                if (size == 0)


@@ -2643,8 +2890,7 @@ pvs_cluster_read(
                         */
                        for (xfer_size = 0; xfer_size < size; ) {
 
                         */
                        for (xfer_size = 0; xfer_size < size; ) {
 

-                               while (cl_index < pages_in_cl 
-                                               && xfer_size < size) {
+                               while (cl_index < pages_in_cl && xfer_size < size) {

                                        /*
                                         * accumulate allocated pages within 
                                         * a physical segment
                                        /*
                                         * accumulate allocated pages within 
                                         * a physical segment


@@ -2659,12 +2905,11 @@ pvs_cluster_read(
                                        } else
                                                break;
                                }
                                        } else
                                                break;
                                }

-                               if (cl_index < pages_in_cl 
-                                               || xfer_size >= size) {
+                               if (cl_index < pages_in_cl || xfer_size >= size) {

                                        /*
                                         * we've hit an unallocated page or 
                                        /*
                                         * we've hit an unallocated page or 

-                                        * the end of this request... go fire
-                                        * the I/O
+                                        * the end of this request... see if
+                                        * it's time to fire the I/O

                                         */
                                        break;
                                }
                                         */
                                        break;
                                }


@@ -2675,10 +2920,7 @@ pvs_cluster_read(
                                 */
                                seg_index++;
                                  
                                 */
                                seg_index++;
                                  

-                               ps_offset[seg_index] = 
-                                       ps_clmap(vs, 
-                                               cur_offset & ~cl_mask, 
-                                               &clmap, CL_FIND, 0, 0);
+                               ps_offset[seg_index] = ps_clmap(vs, cur_offset & ~cl_mask, &clmap, CL_FIND, 0, 0);

                                psp[seg_index] = CLMAP_PS(clmap);
                                ps_info_valid = 1;
 
                                psp[seg_index] = CLMAP_PS(clmap);
                                ps_info_valid = 1;
 


@@ -2689,36 +2931,52 @@ pvs_cluster_read(
                                         * the one we're currently in, or it's 
                                         * in a different paging file, or
                                         * it hasn't been allocated....
                                         * the one we're currently in, or it's 
                                         * in a different paging file, or
                                         * it hasn't been allocated....

-                                        * we stop here and generate the I/O
+                                        * we stop this run and go check
+                                        * to see if it's time to fire the I/O

                                         */
                                        break;
                                }
                                /*
                                 * start with first page of the next physical
                                         */
                                        break;
                                }
                                /*
                                 * start with first page of the next physical

-                                *  segment
+                                * segment

                                 */
                                cl_index = 0;
                        }
                                 */
                                cl_index = 0;
                        }

-                       if (xfer_size) {
+                       if (xfer_size == 0) {

                                /*
                                /*

-                                * we have a contiguous range of allocated pages
-                                * to read from
+                                * no I/O to generate for this segment

                                 */
                                 */

-                               page_list_count = 0;
-                               memory_object_super_upl_request(vs->vs_control,
-                                       (memory_object_offset_t)vs_offset,
-                                       xfer_size, xfer_size, 
-                                       &upl, NULL, &page_list_count,
-                                       request_flags | UPL_SET_INTERNAL);
-
-                               error = ps_read_file(psp[beg_pseg], 
-                                       upl, (vm_offset_t) 0, 
-                                       ps_offset[beg_pseg] + 
-                                               (beg_indx * vm_page_size), 
-                                       xfer_size, &residual, 0);
-                       } else

                                continue;
                                continue;

+                       }
+                       if (cur_offset <= orig_vs_offset) {
+                               /*
+                                * we've hit a hole in our speculative cluster
+                                * before the offset that we're really after...
+                                * don't issue the I/O since it doesn't encompass
+                                * the original offset and we're looking to only
+                                * pull in the speculative pages if they can be
+                                * made part of a single I/O
+                                */
+                               size      -= xfer_size;
+                               vs_offset += xfer_size;

 
 

+                               continue;
+                       }
+                       /*
+                        * we have a contiguous range of allocated pages
+                        * to read from that encompasses the original offset
+                        */
+                       page_list_count = 0;
+                       memory_object_super_upl_request(vs->vs_control, (memory_object_offset_t)vs_offset,
+                                                       xfer_size, xfer_size, 
+                                                       &upl, NULL, &page_list_count,
+                                                       request_flags | UPL_SET_INTERNAL | UPL_NOBLOCK);
+
+                       error = ps_read_file(psp[beg_pseg], 
+                                            upl, (upl_offset_t) 0, 
+                                            ps_offset[beg_pseg] + (beg_indx * vm_page_size), 
+                                            xfer_size, &residual, io_flags);
+                       

                        failed_size = 0;
 
                        /*
                        failed_size = 0;
 
                        /*


@@ -2739,65 +2997,64 @@ pvs_cluster_read(
                                 * supplied data is deallocated from the pager's
                                 *  address space.
                                 */
                                 * supplied data is deallocated from the pager's
                                 *  address space.
                                 */

-                               pvs_object_data_provided(
-                                       vs, upl, vs_offset, xfer_size);
+                               pvs_object_data_provided(vs, upl, vs_offset, xfer_size);

                        } else {
                                failed_size = xfer_size;
 
                                if (error == KERN_SUCCESS) {
                                        if (residual == xfer_size) {
                        } else {
                                failed_size = xfer_size;
 
                                if (error == KERN_SUCCESS) {
                                        if (residual == xfer_size) {

-                                       /*
-                                        * If a read operation returns no error
-                                        * and no data moved, we turn it into
-                                        * an error, assuming we're reading at
-                                        * or beyong EOF.
-                                        * Fall through and error the entire
-                                        * range.
-                                        */
+                                               /*
+                                                * If a read operation returns no error
+                                                * and no data moved, we turn it into
+                                                * an error, assuming we're reading at
+                                                * or beyong EOF.
+                                                * Fall through and error the entire range.
+                                                */

                                                error = KERN_FAILURE;
                                        } else {
                                                error = KERN_FAILURE;
                                        } else {

-                                       /*
-                                        * Otherwise, we have partial read. If
-                                        * the part read is a integral number
-                                        * of pages supply it. Otherwise round
-                                        * it up to a page boundary, zero fill
-                                        * the unread part, and supply it.
-                                        * Fall through and error the remainder
-                                        * of the range, if any.
-                                        */
-                                               int fill, lsize;
+                                               /*
+                                                * Otherwise, we have partial read. If
+                                                * the part read is a integral number
+                                                * of pages supply it. Otherwise round
+                                                * it up to a page boundary, zero fill
+                                                * the unread part, and supply it.
+                                                * Fall through and error the remainder
+                                                * of the range, if any.
+                                                */
+                                               int fill;
+                                               unsigned int lsize;
+
+                                               fill = residual & ~vm_page_size;
+                                               lsize = (xfer_size - residual) + fill;

 
 

-                                               fill = residual 
-                                                       & ~vm_page_size;
-                                               lsize = (xfer_size - residual)
-                                                                        + fill;
-                                               pvs_object_data_provided(
-                                                       vs, upl, 
-                                                       vs_offset, lsize);
+                                               pvs_object_data_provided(vs, upl, vs_offset, lsize);

 
                                                if (lsize < xfer_size) {
 
                                                if (lsize < xfer_size) {

-                                                       failed_size = 
-                                                           xfer_size - lsize;
+                                                       failed_size = xfer_size - lsize;

                                                        error = KERN_FAILURE;
                                                }
                                        }
                                } 
                        }
                                                        error = KERN_FAILURE;
                                                }
                                        }
                                } 
                        }

-                       /*
-                        * If there was an error in any part of the range, tell
-                        * the VM. Note that error is explicitly checked again
-                        *  since it can be modified above.
-                        */

                        if (error != KERN_SUCCESS) {
                        if (error != KERN_SUCCESS) {

+                               /*
+                                * There was an error in some part of the range, tell
+                                * the VM. Note that error is explicitly checked again
+                                * since it can be modified above.
+                                */

                                BS_STAT(psp[beg_pseg]->ps_bs,
                                BS_STAT(psp[beg_pseg]->ps_bs,

-                                       psp[beg_pseg]->ps_bs->bs_pages_in_fail 
-                                               += atop_32(failed_size));
+                                       psp[beg_pseg]->ps_bs->bs_pages_in_fail += atop_32(failed_size));

                        }
                        }

-                       size       -= xfer_size;
-                       vs_offset  += xfer_size;
+                       /*
+                        * we've issued a single I/O that encompassed the original offset
+                        * at this point we either met our speculative request length or 
+                        * we ran into a 'hole' (i.e. page not present in the cluster, cluster
+                        * not present or not physically contiguous to the previous one), so
+                        * we're done issuing I/O at this point
+                        */
+                       return (error);

                }
                }

-
-       } /* END while (cnt && (error == 0)) */
+       }

        return error;
 }
 
        return error;
 }
 


@@ -2807,66 +3064,85 @@ kern_return_t
 vs_cluster_write(
        vstruct_t       vs,
        upl_t           internal_upl,
 vs_cluster_write(
        vstruct_t       vs,
        upl_t           internal_upl,

-       vm_offset_t     offset,
-       vm_size_t       cnt,
+       upl_offset_t    offset,
+       upl_size_t      cnt,

        boolean_t       dp_internal,
        int             flags)
 {
        boolean_t       dp_internal,
        int             flags)
 {

-       vm_offset_t     size;
-       vm_offset_t     transfer_size;
+       upl_size_t      transfer_size;

        int             error = 0;
        struct clmap    clmap;
 
        int             error = 0;
        struct clmap    clmap;
 

-       vm_offset_t     actual_offset;  /* Offset within paging segment */
+       dp_offset_t     actual_offset;  /* Offset within paging segment */

        paging_segment_t ps;
        paging_segment_t ps;

-       vm_offset_t     subx_size;
-       vm_offset_t     mobj_base_addr;
-       vm_offset_t     mobj_target_addr;
-       int             mobj_size;
-
-       struct vs_async *vsa;
-       vm_map_copy_t   copy;
+       dp_offset_t     mobj_base_addr;
+       dp_offset_t     mobj_target_addr;

 
        upl_t           upl;
        upl_page_info_t *pl;
        int             page_index;
 
        upl_t           upl;
        upl_page_info_t *pl;
        int             page_index;

+       unsigned int    page_max_index;

        int             list_size;
        int             pages_in_cl;
        int             list_size;
        int             pages_in_cl;

-       int             cl_size;
+       unsigned int    cl_size;

        int             base_index;
        int             base_index;

-       int             seg_size;
+       unsigned int    seg_size;
+       unsigned int    upl_offset_in_object;
+       boolean_t       minimal_clustering = FALSE;
+       boolean_t       found_dirty;

 
        pages_in_cl = 1 << vs->vs_clshift;
        cl_size = pages_in_cl * vm_page_size;
        
 
        pages_in_cl = 1 << vs->vs_clshift;
        cl_size = pages_in_cl * vm_page_size;
        

+#if CONFIG_FREEZE
+       minimal_clustering = TRUE;
+#endif
+       if (dp_isssd == TRUE)
+               minimal_clustering = TRUE;
+

        if (!dp_internal) {
        if (!dp_internal) {

-               int          page_list_count;
+               unsigned int page_list_count;

                int          request_flags;
                int          request_flags;

-               int          super_size;
+               unsigned int super_size;

                int          first_dirty;
                int          num_dirty;
                int          num_of_pages;
                int          seg_index;
                int          first_dirty;
                int          num_dirty;
                int          num_of_pages;
                int          seg_index;

-               vm_offset_t  upl_offset;
-               vm_offset_t  seg_offset;
-               vm_offset_t  ps_offset[((VM_SUPER_CLUSTER / PAGE_SIZE) >> VSTRUCT_DEF_CLSHIFT) + 1];
-               paging_segment_t   psp[((VM_SUPER_CLUSTER / PAGE_SIZE) >> VSTRUCT_DEF_CLSHIFT) + 1];
+               upl_offset_t  upl_offset;
+               upl_offset_t  upl_offset_aligned;
+               dp_offset_t  seg_offset;
+               dp_offset_t  ps_offset[((VM_SUPER_CLUSTER / PAGE_SIZE) >> VSTRUCT_MIN_CLSHIFT) + 1];
+               paging_segment_t   psp[((VM_SUPER_CLUSTER / PAGE_SIZE) >> VSTRUCT_MIN_CLSHIFT) + 1];

 
 
 
 

-               if (bs_low) {
+               if (bs_low)

                        super_size = cl_size;
                        super_size = cl_size;

-
-                       request_flags = UPL_NOBLOCK |
-                               UPL_RET_ONLY_DIRTY | UPL_COPYOUT_FROM | 
-                               UPL_NO_SYNC | UPL_SET_INTERNAL;
-               } else {
+               else

                        super_size = VM_SUPER_CLUSTER;
 
                        super_size = VM_SUPER_CLUSTER;
 

-                       request_flags = UPL_NOBLOCK | UPL_CLEAN_IN_PLACE |
-                               UPL_RET_ONLY_DIRTY | UPL_COPYOUT_FROM | 
-                               UPL_NO_SYNC | UPL_SET_INTERNAL;
-               }
+               request_flags = UPL_NOBLOCK | UPL_CLEAN_IN_PLACE |
+                               UPL_RET_ONLY_DIRTY | UPL_COPYOUT_FROM | 
+                               UPL_NO_SYNC | UPL_SET_INTERNAL | UPL_SET_LITE;

 
 

+               if (!dp_encryption_inited) {
+                       /*
+                        * ENCRYPTED SWAP:
+                        * Once we've started using swap, we
+                        * can't change our mind on whether
+                        * it needs to be encrypted or
+                        * not.
+                        */
+                       dp_encryption_inited = TRUE;
+               }
+               if (dp_encryption) {
+                       /*
+                        * ENCRYPTED SWAP:
+                        * request that the UPL be prepared for
+                        * encryption.
+                        */
+                       request_flags |= UPL_ENCRYPT;
+                       flags |= UPL_PAGING_ENCRYPTED;
+               }

                page_list_count = 0;
                memory_object_super_upl_request(vs->vs_control,
                                (memory_object_offset_t)offset,
                page_list_count = 0;
                memory_object_super_upl_request(vs->vs_control,
                                (memory_object_offset_t)offset,


@@ -2874,31 +3150,64 @@ vs_cluster_write(
                                &upl, NULL, &page_list_count,
                                request_flags | UPL_FOR_PAGEOUT);
 
                                &upl, NULL, &page_list_count,
                                request_flags | UPL_FOR_PAGEOUT);
 

+               /*
+                * The default pager does not handle objects larger than
+                * 4GB, so it does not deal with offset that don't fit in
+                * 32-bit.  Cast down upl->offset now and make sure we
+                * did not lose any valuable bits.
+                */
+               upl_offset_in_object = (unsigned int) upl->offset;
+               assert(upl->offset == upl_offset_in_object);
+

                pl = UPL_GET_INTERNAL_PAGE_LIST(upl);
 
                pl = UPL_GET_INTERNAL_PAGE_LIST(upl);
 

-               seg_size = cl_size - (upl->offset % cl_size);
-               upl_offset = upl->offset & ~(cl_size - 1);
+               seg_size = cl_size - (upl_offset_in_object % cl_size);
+               upl_offset_aligned = upl_offset_in_object & ~(cl_size - 1);
+               page_index = 0;
+               page_max_index = upl->size / PAGE_SIZE;
+               found_dirty = TRUE;

 
 

-               for (seg_index = 0, transfer_size = upl->size; 
-                                               transfer_size > 0; ) {
-                       ps_offset[seg_index] = 
-                               ps_clmap(vs, 
-                                       upl_offset,
-                                       &clmap, CL_ALLOC, 
-                                       cl_size, 0); 
+               for (seg_index = 0, transfer_size = upl->size; transfer_size > 0; ) {
+                       unsigned int    seg_pgcnt;

 
 

-                       if (ps_offset[seg_index] == (vm_offset_t) -1) {
-                               upl_abort(upl, 0);
-                               upl_deallocate(upl);
-                               
-                               return KERN_FAILURE;
+                       seg_pgcnt = seg_size / PAGE_SIZE;

 
 

+                       if (minimal_clustering == TRUE) {
+                               unsigned int    non_dirty;
+
+                               non_dirty = 0;
+                               found_dirty = FALSE;
+
+                               for (; non_dirty < seg_pgcnt; non_dirty++) {
+                                       if ((page_index + non_dirty) >= page_max_index)
+                                               break;
+
+                                       if (UPL_DIRTY_PAGE(pl, page_index + non_dirty) ||
+                                           UPL_PRECIOUS_PAGE(pl, page_index + non_dirty)) {
+                                               found_dirty = TRUE;
+                                               break;
+                                       }
+                               }

                        }
                        }

-                       psp[seg_index] = CLMAP_PS(clmap);
+                       if (found_dirty == TRUE) {
+                               ps_offset[seg_index] = 
+                                       ps_clmap(vs, 
+                                                upl_offset_aligned,
+                                                &clmap, CL_ALLOC, 
+                                                cl_size, 0); 

 
 

+                               if (ps_offset[seg_index] == (dp_offset_t) -1) {
+                                       upl_abort(upl, 0);
+                                       upl_deallocate(upl);
+                               
+                                       return KERN_FAILURE;
+                               }
+                               psp[seg_index] = CLMAP_PS(clmap);
+                       }

                        if (transfer_size > seg_size) {
                        if (transfer_size > seg_size) {

+                               page_index += seg_pgcnt;

                                transfer_size -= seg_size;
                                transfer_size -= seg_size;

-                               upl_offset += cl_size;
+                               upl_offset_aligned += cl_size;

                                seg_size    = cl_size;
                                seg_index++;
                        } else
                                seg_size    = cl_size;
                                seg_index++;
                        } else


@@ -2913,7 +3222,7 @@ vs_cluster_write(
                                break;
                num_of_pages = page_index + 1;
 
                                break;
                num_of_pages = page_index + 1;
 

-               base_index = (upl->offset % cl_size) / PAGE_SIZE;
+               base_index = (upl_offset_in_object % cl_size) / PAGE_SIZE;

 
                for (page_index = 0; page_index < num_of_pages; ) {
                        /*
 
                for (page_index = 0; page_index < num_of_pages; ) {
                        /*


@@ -3004,12 +3313,15 @@ vs_cluster_write(
                                while (transfer_size) {
 
                                        if ((seg_size = cl_size - 
                                while (transfer_size) {
 
                                        if ((seg_size = cl_size - 

-                                               ((upl->offset + upl_offset) % cl_size)) 
+                                               ((upl_offset_in_object +
+                                                 upl_offset) % cl_size)) 

                                                        > transfer_size)
                                                seg_size = transfer_size;
 
                                                        > transfer_size)
                                                seg_size = transfer_size;
 

-                                       ps_vs_write_complete(vs, 
-                                               upl->offset + upl_offset, 
+                                       ps_vs_write_complete(
+                                               vs, 
+                                               (upl_offset_in_object +
+                                                upl_offset), 

                                                seg_size, error);
 
                                        transfer_size -= seg_size;
                                                seg_size, error);
 
                                        transfer_size -= seg_size;


@@ -3019,7 +3331,7 @@ vs_cluster_write(
                                transfer_size = num_dirty * vm_page_size;
 
                                seg_index  = (base_index + first_dirty) / pages_in_cl;
                                transfer_size = num_dirty * vm_page_size;
 
                                seg_index  = (base_index + first_dirty) / pages_in_cl;

-                               seg_offset = (upl->offset + upl_offset) % cl_size;
+                               seg_offset = (upl_offset_in_object + upl_offset) % cl_size;

 
                                error = ps_write_file(psp[seg_index], 
                                                upl, upl_offset,
 
                                error = ps_write_file(psp[seg_index], 
                                                upl, upl_offset,


@@ -3041,7 +3353,7 @@ vs_cluster_write(
                }
 
        } else {
                }
 
        } else {

-               assert(cnt  <= (vm_page_size << vs->vs_clshift));
+               assert(cnt <= (unsigned) (vm_page_size << vs->vs_clshift));

                list_size = cnt;
 
                page_index = 0;
                list_size = cnt;
 
                page_index = 0;


@@ -3058,12 +3370,12 @@ vs_cluster_write(
                                &clmap, CL_ALLOC, 
                                transfer_size < cl_size ? 
                                        transfer_size : cl_size, 0);
                                &clmap, CL_ALLOC, 
                                transfer_size < cl_size ? 
                                        transfer_size : cl_size, 0);

-                       if(actual_offset == (vm_offset_t) -1) {
+                       if(actual_offset == (dp_offset_t) -1) {

                                error = 1;
                                break;
                        }
                        cnt = MIN(transfer_size, 
                                error = 1;
                                break;
                        }
                        cnt = MIN(transfer_size, 

-                               CLMAP_NPGS(clmap) * vm_page_size);
+                                 (unsigned) CLMAP_NPGS(clmap) * vm_page_size);

                        ps = CLMAP_PS(clmap);
                        /* Assume that the caller has given us contiguous */
                        /* pages */
                        ps = CLMAP_PS(clmap);
                        /* Assume that the caller has given us contiguous */
                        /* pages */


@@ -3099,7 +3411,7 @@ ps_vstruct_allocated_size(
 {
        int             num_pages;
        struct vs_map   *vsmap;
 {
        int             num_pages;
        struct vs_map   *vsmap;

-       int             i, j, k;
+       unsigned int    i, j, k;

 
        num_pages = 0;
        if (vs->vs_indirect) {
 
        num_pages = 0;
        if (vs->vs_indirect) {


@@ -3140,16 +3452,16 @@ ps_vstruct_allocated_size(
        return ptoa_32(num_pages);
 }
 
        return ptoa_32(num_pages);
 }
 

-size_t
+unsigned int

 ps_vstruct_allocated_pages(
        vstruct_t               vs,
        default_pager_page_t    *pages,
 ps_vstruct_allocated_pages(
        vstruct_t               vs,
        default_pager_page_t    *pages,

-       size_t                  pages_size)
+       unsigned int            pages_size)

 {
 {

-       int             num_pages;
+       unsigned int    num_pages;

        struct vs_map   *vsmap;
        struct vs_map   *vsmap;

-       vm_offset_t     offset;
-       int             i, j, k;
+       dp_offset_t     offset;
+       unsigned int    i, j, k;

 
        num_pages = 0;
        offset = 0;
 
        num_pages = 0;
        offset = 0;


@@ -3215,9 +3527,9 @@ ps_vstruct_transfer_from_segment(
        upl_t            upl)
 {
        struct vs_map   *vsmap;
        upl_t            upl)
 {
        struct vs_map   *vsmap;

-       struct vs_map   old_vsmap;
-       struct vs_map   new_vsmap;
-       int             i, j, k;
+//     struct vs_map   old_vsmap;
+//     struct vs_map   new_vsmap;
+       unsigned int    i, j;

 
        VS_LOCK(vs);    /* block all work on this vstruct */
                        /* can't allow the normal multiple write */
 
        VS_LOCK(vs);    /* block all work on this vstruct */
                        /* can't allow the normal multiple write */


@@ -3240,8 +3552,8 @@ ps_vstruct_transfer_from_segment(
        VS_UNLOCK(vs);
 vs_changed:
        if (vs->vs_indirect) {
        VS_UNLOCK(vs);
 vs_changed:
        if (vs->vs_indirect) {

-               int     vsmap_size;
-               int     clmap_off;
+               unsigned int    vsmap_size;
+               int             clmap_off;

                /* loop on indirect maps */
                for (i = 0; i < INDIRECT_CLMAP_ENTRIES(vs->vs_size); i++) {
                        vsmap = vs->vs_imap[i];
                /* loop on indirect maps */
                for (i = 0; i < INDIRECT_CLMAP_ENTRIES(vs->vs_size); i++) {
                        vsmap = vs->vs_imap[i];


@@ -3319,10 +3631,10 @@ vs_changed:
                        vs_finish_write(vs);
                        VS_LOCK(vs);
                        vs->vs_xfer_pending = TRUE;
                        vs_finish_write(vs);
                        VS_LOCK(vs);
                        vs->vs_xfer_pending = TRUE;

-                       VS_UNLOCK(vs);

                        vs_wait_for_sync_writers(vs);
                        vs_start_write(vs);
                        vs_wait_for_readers(vs);
                        vs_wait_for_sync_writers(vs);
                        vs_start_write(vs);
                        vs_wait_for_readers(vs);

+                       VS_UNLOCK(vs);

                        if (vs->vs_indirect) {
                                goto vs_changed;
                        }
                        if (vs->vs_indirect) {
                                goto vs_changed;
                        }


@@ -3341,10 +3653,10 @@ vs_changed:
 vs_map_t
 vs_get_map_entry(
        vstruct_t       vs, 
 vs_map_t
 vs_get_map_entry(
        vstruct_t       vs, 

-       vm_offset_t     offset)
+       dp_offset_t     offset)

 {
        struct vs_map   *vsmap;
 {
        struct vs_map   *vsmap;

-       vm_offset_t     cluster;
+       dp_offset_t     cluster;

 
        cluster = atop_32(offset) >> vs->vs_clshift;
        if (vs->vs_indirect) {
 
        cluster = atop_32(offset) >> vs->vs_clshift;
        if (vs->vs_indirect) {


@@ -3363,25 +3675,26 @@ vs_get_map_entry(
 kern_return_t
 vs_cluster_transfer(
        vstruct_t       vs,
 kern_return_t
 vs_cluster_transfer(
        vstruct_t       vs,

-       vm_offset_t     offset,
-       vm_size_t       cnt,
+       dp_offset_t     offset,
+       dp_size_t       cnt,

        upl_t           upl)
 {
        upl_t           upl)
 {

-       vm_offset_t             actual_offset;
+       dp_offset_t             actual_offset;

        paging_segment_t        ps;
        struct clmap            clmap;
        kern_return_t           error = KERN_SUCCESS;
        paging_segment_t        ps;
        struct clmap            clmap;
        kern_return_t           error = KERN_SUCCESS;

-       int                     size, size_wanted, i;
-       unsigned int            residual;
-       int                     unavail_size;
-       default_pager_thread_t  *dpt;
-       boolean_t               dealloc;
-       struct  vs_map          *vsmap_ptr;
+       unsigned int            size, size_wanted;
+       int                     i;
+       unsigned int            residual = 0;
+       unsigned int            unavail_size;
+//     default_pager_thread_t  *dpt;
+//     boolean_t               dealloc;
+       struct  vs_map          *vsmap_ptr = NULL;

        struct  vs_map          read_vsmap;
        struct  vs_map          original_read_vsmap;
        struct  vs_map          write_vsmap;
        struct  vs_map          read_vsmap;
        struct  vs_map          original_read_vsmap;
        struct  vs_map          write_vsmap;

-       upl_t                   sync_upl;
-       vm_offset_t     ioaddr;
+//     upl_t                           sync_upl;
+//     vm_offset_t                     ioaddr;

 
        /* vs_cluster_transfer reads in the pages of a cluster and
         * then writes these pages back to new backing store.  The
 
        /* vs_cluster_transfer reads in the pages of a cluster and
         * then writes these pages back to new backing store.  The


@@ -3412,7 +3725,7 @@ vs_cluster_transfer(
                vsmap_ptr = vs_get_map_entry(vs, offset);
                actual_offset = ps_clmap(vs, offset, &clmap, CL_FIND, 0, 0);
 
                vsmap_ptr = vs_get_map_entry(vs, offset);
                actual_offset = ps_clmap(vs, offset, &clmap, CL_FIND, 0, 0);
 

-               if (actual_offset == (vm_offset_t) -1) {
+               if (actual_offset == (dp_offset_t) -1) {

 
                        /*
                         * Nothing left to write in this cluster at least
 
                        /*
                         * Nothing left to write in this cluster at least


@@ -3465,6 +3778,7 @@ vs_cluster_transfer(
 
                if (size == 0) {
                        ASSERT(unavail_size);
 
                if (size == 0) {
                        ASSERT(unavail_size);

+                       ps_clunmap(vs, offset, unavail_size);

                        cnt -= unavail_size;
                        offset += unavail_size;
                        if((offset & ((vm_page_size << vs->vs_clshift) - 1)) 
                        cnt -= unavail_size;
                        offset += unavail_size;
                        if((offset & ((vm_page_size << vs->vs_clshift) - 1)) 


@@ -3483,6 +3797,10 @@ vs_cluster_transfer(
                        original_read_vsmap = *vsmap_ptr;
 
                if(ps->ps_segtype == PS_PARTITION) {
                        original_read_vsmap = *vsmap_ptr;
 
                if(ps->ps_segtype == PS_PARTITION) {

+                       panic("swap partition not supported\n");
+                       /*NOTREACHED*/
+                       error = KERN_FAILURE;
+                       residual = size;

 /*
                        NEED TO ISSUE WITH SYNC & NO COMMIT
                        error = ps_read_device(ps, actual_offset, &buffer,
 /*
                        NEED TO ISSUE WITH SYNC & NO COMMIT
                        error = ps_read_device(ps, actual_offset, &buffer,


@@ -3490,7 +3808,7 @@ vs_cluster_transfer(
 */
                } else {
                        /* NEED TO ISSUE WITH SYNC & NO COMMIT */
 */
                } else {
                        /* NEED TO ISSUE WITH SYNC & NO COMMIT */

-                       error = ps_read_file(ps, upl, (vm_offset_t) 0, actual_offset, 
+                       error = ps_read_file(ps, upl, (upl_offset_t) 0, actual_offset, 

                                        size, &residual, 
                                        (UPL_IOSYNC | UPL_NOCOMMIT));
                }
                                        size, &residual, 
                                        (UPL_IOSYNC | UPL_NOCOMMIT));
                }


@@ -3506,7 +3824,6 @@ vs_cluster_transfer(
                 * 
                 */
                if ((error == KERN_SUCCESS) && (residual == 0)) {
                 * 
                 */
                if ((error == KERN_SUCCESS) && (residual == 0)) {

-                       int page_list_count = 0;

 
                        /*
                         * Got everything we asked for, supply the data to
 
                        /*
                         * Got everything we asked for, supply the data to


@@ -3540,6 +3857,7 @@ vs_cluster_transfer(
                                        */
                                        write_vsmap = *vsmap_ptr;
                                        *vsmap_ptr = read_vsmap;
                                        */
                                        write_vsmap = *vsmap_ptr;
                                        *vsmap_ptr = read_vsmap;

+                                       ps_clunmap(vs, offset, size);

                                } else {
                                        /* discard the old backing object */
                                        write_vsmap = *vsmap_ptr;
                                } else {
                                        /* discard the old backing object */
                                        write_vsmap = *vsmap_ptr;


@@ -3601,14 +3919,16 @@ vs_cluster_transfer(
 }
 
 kern_return_t
 }
 
 kern_return_t

-default_pager_add_file(MACH_PORT_FACE backing_store,
-       int             *vp,
+default_pager_add_file(
+       MACH_PORT_FACE  backing_store,
+       vnode_ptr_t     vp,

        int             record_size,
        int             record_size,

-       long            size)
+       vm_size_t       size)

 {
        backing_store_t         bs;
        paging_segment_t        ps;
        int                     i;
 {
        backing_store_t         bs;
        paging_segment_t        ps;
        int                     i;

+       unsigned int            j;

        int                     error;
 
        if ((bs = backing_store_lookup(backing_store))
        int                     error;
 
        if ((bs = backing_store_lookup(backing_store))


@@ -3647,31 +3967,42 @@ default_pager_add_file(MACH_PORT_FACE backing_store,
        ps->ps_vnode = (struct vnode *)vp;
        ps->ps_offset = 0;
        ps->ps_record_shift = local_log2(vm_page_size / record_size);
        ps->ps_vnode = (struct vnode *)vp;
        ps->ps_offset = 0;
        ps->ps_record_shift = local_log2(vm_page_size / record_size);

-       ps->ps_recnum = size;
-       ps->ps_pgnum = size >> ps->ps_record_shift;
+       assert((dp_size_t) size == size);
+       ps->ps_recnum = (dp_size_t) size;
+       ps->ps_pgnum = ((dp_size_t) size) >> ps->ps_record_shift;

 
        ps->ps_pgcount = ps->ps_pgnum;
        ps->ps_clshift = local_log2(bs->bs_clsize);
        ps->ps_clcount = ps->ps_ncls = ps->ps_pgcount >> ps->ps_clshift;
 
        ps->ps_pgcount = ps->ps_pgnum;
        ps->ps_clshift = local_log2(bs->bs_clsize);
        ps->ps_clcount = ps->ps_ncls = ps->ps_pgcount >> ps->ps_clshift;

+       ps->ps_special_clusters = 0;

        ps->ps_hint = 0;
 
        PS_LOCK_INIT(ps);
        ps->ps_bmap = (unsigned char *) kalloc(RMAPSIZE(ps->ps_ncls));
        if (!ps->ps_bmap) {
        ps->ps_hint = 0;
 
        PS_LOCK_INIT(ps);
        ps->ps_bmap = (unsigned char *) kalloc(RMAPSIZE(ps->ps_ncls));
        if (!ps->ps_bmap) {

-               kfree((vm_offset_t)ps, sizeof *ps);
+               kfree(ps, sizeof *ps);

                BS_UNLOCK(bs);
                return KERN_RESOURCE_SHORTAGE;
        }
                BS_UNLOCK(bs);
                return KERN_RESOURCE_SHORTAGE;
        }

-       for (i = 0; i < ps->ps_ncls; i++) {
-               clrbit(ps->ps_bmap, i);
+       for (j = 0; j < ps->ps_ncls; j++) {
+               clrbit(ps->ps_bmap, j);

        }
 
        }
 

-       ps->ps_going_away = FALSE;
+       if(paging_segment_count == 0) {
+               ps->ps_state = PS_EMERGENCY_SEGMENT;
+               if(use_emergency_swap_file_first) {
+                       ps->ps_state |= PS_CAN_USE;
+               }
+               emergency_segment_backing_store = backing_store;
+       } else {
+               ps->ps_state = PS_CAN_USE;
+       }
+       

        ps->ps_bs = bs;
 
        if ((error = ps_enter(ps)) != 0) {
        ps->ps_bs = bs;
 
        if ((error = ps_enter(ps)) != 0) {

-               kfree((vm_offset_t)ps->ps_bmap, RMAPSIZE(ps->ps_ncls));
-               kfree((vm_offset_t)ps, sizeof *ps);
+               kfree(ps->ps_bmap, RMAPSIZE(ps->ps_ncls));
+               kfree(ps, sizeof *ps);

                BS_UNLOCK(bs);
                return KERN_RESOURCE_SHORTAGE;
        }
                BS_UNLOCK(bs);
                return KERN_RESOURCE_SHORTAGE;
        }


@@ -3679,17 +4010,37 @@ default_pager_add_file(MACH_PORT_FACE backing_store,
        bs->bs_pages_free += ps->ps_clcount << ps->ps_clshift;
        bs->bs_pages_total += ps->ps_clcount << ps->ps_clshift;
        PSL_LOCK();
        bs->bs_pages_free += ps->ps_clcount << ps->ps_clshift;
        bs->bs_pages_total += ps->ps_clcount << ps->ps_clshift;
        PSL_LOCK();

-       dp_pages_free += ps->ps_pgcount;
+       if(IS_PS_OK_TO_USE(ps)) {
+               dp_pages_free += ps->ps_pgcount;
+       } else {
+               dp_pages_reserve += ps->ps_pgcount;
+       }

        PSL_UNLOCK();
 
        BS_UNLOCK(bs);
 
        bs_more_space(ps->ps_clcount);
 
        PSL_UNLOCK();
 
        BS_UNLOCK(bs);
 
        bs_more_space(ps->ps_clcount);
 

-       DEBUG(DEBUG_BS_INTERNAL,
-             ("device=0x%x,offset=0x%x,count=0x%x,record_size=0x%x,shift=%d,total_size=0x%x\n",
-              device, offset, size, record_size,
-              ps->ps_record_shift, ps->ps_pgnum));
+       /*
+        * If the paging segment being activated is not the emergency
+        * segment and we notice that the emergency segment is being
+        * used then we help recover it. If all goes well, the
+        * emergency segment will be back to its original state of
+        * online but not activated (till it's needed the next time).
+        */
+       ps = paging_segments[EMERGENCY_PSEG_INDEX];
+       if(IS_PS_EMERGENCY_SEGMENT(ps) && IS_PS_OK_TO_USE(ps)) {
+               if(default_pager_backing_store_delete(emergency_segment_backing_store)) {
+                       dprintf(("Failed to recover emergency paging segment\n"));
+               } else {
+                       dprintf(("Recovered emergency paging segment\n"));
+               }
+       }
+       
+       DP_DEBUG(DEBUG_BS_INTERNAL,
+                ("device=0x%x,offset=0x%x,count=0x%x,record_size=0x%x,shift=%d,total_size=0x%x\n",
+                 device, offset, (dp_size_t) size, record_size,
+                 ps->ps_record_shift, ps->ps_pgnum));

 
        return KERN_SUCCESS;
 }
 
        return KERN_SUCCESS;
 }


@@ -3700,9 +4051,9 @@ kern_return_t
 ps_read_file(
        paging_segment_t        ps,
        upl_t                   upl,
 ps_read_file(
        paging_segment_t        ps,
        upl_t                   upl,

-       vm_offset_t             upl_offset,
-       vm_offset_t             offset,
-       unsigned int            size,
+       upl_offset_t            upl_offset,
+       dp_offset_t             offset,
+       upl_size_t              size,

        unsigned int            *residualp,
        int                     flags)
 {
        unsigned int            *residualp,
        int                     flags)
 {


@@ -3710,14 +4061,17 @@ ps_read_file(
        int                     error = 0;
        int                     result;
 
        int                     error = 0;
        int                     result;
 

+       assert(dp_encryption_inited);

 
        clustered_reads[atop_32(size)]++;
 
        f_offset = (vm_object_offset_t)(ps->ps_offset + offset);
        
 
        clustered_reads[atop_32(size)]++;
 
        f_offset = (vm_object_offset_t)(ps->ps_offset + offset);
        

-       /* for transfer case we need to pass uploffset and flags */
-       error = vnode_pagein(ps->ps_vnode, 
-                                  upl, upl_offset, f_offset, (vm_size_t)size, flags | UPL_NORDAHEAD, NULL);
+       /*
+        * for transfer case we need to pass uploffset and flags
+        */
+       assert((upl_size_t) size == size);
+       error = vnode_pagein(ps->ps_vnode, upl, upl_offset, f_offset, (upl_size_t)size, flags, NULL);

 
        /* The vnode_pagein semantic is somewhat at odds with the existing   */
        /* device_read semantic.  Partial reads are not experienced at this  */
 
        /* The vnode_pagein semantic is somewhat at odds with the existing   */
        /* device_read semantic.  Partial reads are not experienced at this  */


@@ -3739,21 +4093,29 @@ kern_return_t
 ps_write_file(
        paging_segment_t        ps,
        upl_t                   upl,
 ps_write_file(
        paging_segment_t        ps,
        upl_t                   upl,

-       vm_offset_t             upl_offset,
-       vm_offset_t             offset,
+       upl_offset_t            upl_offset,
+       dp_offset_t             offset,

        unsigned int            size,
        int                     flags)
 {
        vm_object_offset_t      f_offset;
        kern_return_t           result;
 
        unsigned int            size,
        int                     flags)
 {
        vm_object_offset_t      f_offset;
        kern_return_t           result;
 

-       int             error = 0;
+       assert(dp_encryption_inited);

 
        clustered_writes[atop_32(size)]++;
        f_offset = (vm_object_offset_t)(ps->ps_offset + offset);
 
 
        clustered_writes[atop_32(size)]++;
        f_offset = (vm_object_offset_t)(ps->ps_offset + offset);
 

-       if (vnode_pageout(ps->ps_vnode,
-                               upl, upl_offset, f_offset, (vm_size_t)size, flags, NULL))
+       if (flags & UPL_PAGING_ENCRYPTED) {
+               /*
+                * ENCRYPTED SWAP:
+                * encrypt all the pages that we're going
+                * to pageout.
+                */
+               upl_encrypt(upl, upl_offset, size);
+       }
+       assert((upl_size_t) size == size);
+       if (vnode_pageout(ps->ps_vnode, upl, upl_offset, f_offset, (upl_size_t)size, flags, NULL))

                result = KERN_FAILURE;
        else
                result = KERN_SUCCESS;
                result = KERN_FAILURE;
        else
                result = KERN_SUCCESS;


@@ -3762,7 +4124,7 @@ ps_write_file(
 }
 
 kern_return_t
 }
 
 kern_return_t

-default_pager_triggers(MACH_PORT_FACE default_pager,
+default_pager_triggers( __unused MACH_PORT_FACE default_pager,

        int             hi_wat,
        int             lo_wat,
        int             flags,
        int             hi_wat,
        int             lo_wat,
        int             flags,


@@ -3770,9 +4132,32 @@ default_pager_triggers(MACH_PORT_FACE default_pager,
 {
        MACH_PORT_FACE release;
        kern_return_t kr;
 {
        MACH_PORT_FACE release;
        kern_return_t kr;

+       clock_sec_t now;
+       clock_nsec_t nanoseconds_dummy;
+       static clock_sec_t error_notify = 0;

 
        PSL_LOCK();
 
        PSL_LOCK();

-       if (flags == HI_WAT_ALERT) {
+       if (flags == SWAP_ENCRYPT_ON) {
+               /* ENCRYPTED SWAP: turn encryption on */
+               release = trigger_port;
+               if (!dp_encryption_inited) {
+                       dp_encryption_inited = TRUE;
+                       dp_encryption = TRUE;
+                       kr = KERN_SUCCESS;
+               } else {
+                       kr = KERN_FAILURE;
+               }
+       } else if (flags == SWAP_ENCRYPT_OFF) {
+               /* ENCRYPTED SWAP: turn encryption off */
+               release = trigger_port;
+               if (!dp_encryption_inited) {
+                       dp_encryption_inited = TRUE;
+                       dp_encryption = FALSE;
+                       kr = KERN_SUCCESS;
+               } else {
+                       kr = KERN_FAILURE;
+               }
+       } else if (flags == HI_WAT_ALERT) {

                release = min_pages_trigger_port;
                min_pages_trigger_port = trigger_port;
                minimum_pages_remaining = hi_wat/vm_page_size;
                release = min_pages_trigger_port;
                min_pages_trigger_port = trigger_port;
                minimum_pages_remaining = hi_wat/vm_page_size;


@@ -3783,6 +4168,22 @@ default_pager_triggers(MACH_PORT_FACE default_pager,
                max_pages_trigger_port = trigger_port;
                maximum_pages_free = lo_wat/vm_page_size;
                kr = KERN_SUCCESS;
                max_pages_trigger_port = trigger_port;
                maximum_pages_free = lo_wat/vm_page_size;
                kr = KERN_SUCCESS;

+       } else if (flags == USE_EMERGENCY_SWAP_FILE_FIRST) {
+               use_emergency_swap_file_first = TRUE;
+               release = trigger_port;
+               kr = KERN_SUCCESS;
+       } else if (flags == SWAP_FILE_CREATION_ERROR) {
+               release = trigger_port;
+               kr = KERN_SUCCESS;
+               if( paging_segment_count == 1) {
+                       use_emergency_swap_file_first = TRUE;
+               }
+               no_paging_space_action();
+               clock_get_system_nanotime(&now, &nanoseconds_dummy);
+               if (now > error_notify + 5) {
+                       dprintf(("Swap File Error.\n"));
+                       error_notify = now;
+               }

        } else {
                release = trigger_port;
                kr =  KERN_INVALID_ARGUMENT;
        } else {
                release = trigger_port;
                kr =  KERN_INVALID_ARGUMENT;


@@ -3809,11 +4210,13 @@ default_pager_triggers(MACH_PORT_FACE default_pager,
 #define PF_LATENCY     10      /* number of intervals before release */
 
 static int dp_pages_free_low_count = 0;
 #define PF_LATENCY     10      /* number of intervals before release */
 
 static int dp_pages_free_low_count = 0;

+thread_call_t default_pager_backing_store_monitor_callout;

 
 void
 
 void

-default_pager_backing_store_monitor(thread_call_param_t p1, thread_call_param_t p2)
+default_pager_backing_store_monitor(__unused thread_call_param_t p1,
+                                                                       __unused thread_call_param_t p2)

 {
 {

-       unsigned long long      average;
+//     unsigned long long      average;

        ipc_port_t              trigger;
        uint64_t                deadline;
 
        ipc_port_t              trigger;
        uint64_t                deadline;
 


@@ -3862,5 +4265,5 @@ default_pager_backing_store_monitor(thread_call_param_t p1, thread_call_param_t
        }
 
        clock_interval_to_deadline(PF_INTERVAL, NSEC_PER_SEC, &deadline);
        }
 
        clock_interval_to_deadline(PF_INTERVAL, NSEC_PER_SEC, &deadline);

-       thread_call_func_delayed(default_pager_backing_store_monitor, NULL, deadline);
+       thread_call_enter_delayed(default_pager_backing_store_monitor_callout, deadline);

 }
 }