xnu-792.tar.gz

[apple/xnu.git] / osfmk / vm / bsd_vm.c

diff --git a/osfmk/vm/bsd_vm.c b/osfmk/vm/bsd_vm.c
index 6c4a68dc7a47392d8e39fbf2bc6d76c9f5dd3e5b..fa1cfa94b6545cad30f753ddb7175b3f00bb0728 100644 (file)
--- a/osfmk/vm/bsd_vm.c
+++ b/osfmk/vm/bsd_vm.c
@@ -1,5 +1,5 @@
 /*
 /*

- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.

  *
  * @APPLE_LICENSE_HEADER_START@
  * 
  *
  * @APPLE_LICENSE_HEADER_START@
  * 


@@ -21,20 +21,33 @@
  */
 
 #include <sys/errno.h>
  */
 
 #include <sys/errno.h>

-#include <kern/host.h>
+

 #include <mach/mach_types.h>
 #include <mach/mach_types.h>

-#include <vm/vm_map.h>
-#include <vm/vm_kern.h>
-#include <vm/vm_pageout.h>
+#include <mach/mach_traps.h>
+#include <mach/host_priv.h>

 #include <mach/kern_return.h>
 #include <mach/kern_return.h>

+#include <mach/memory_object_control.h>

 #include <mach/memory_object_types.h>
 #include <mach/port.h>
 #include <mach/policy.h>
 #include <mach/memory_object_types.h>
 #include <mach/port.h>
 #include <mach/policy.h>

+#include <mach/upl.h>
+#include <mach/thread_act.h>
+
+#include <kern/host.h>
+#include <kern/thread.h>
+

 #include <ipc/ipc_port.h>
 #include <ipc/ipc_space.h>
 #include <ipc/ipc_port.h>
 #include <ipc/ipc_space.h>

-#include <kern/thread.h>
-#include <vm/vm_pageout.h>

 
 

+#include <default_pager/default_pager_types.h>
+#include <default_pager/default_pager_object_server.h>
+
+#include <vm/vm_map.h>
+#include <vm/vm_kern.h>
+#include <vm/vm_pageout.h>
+#include <vm/memory_object.h>
+#include <vm/vm_pageout.h>
+#include <vm/vm_protos.h>

 
 /* BSD VM COMPONENT INTERFACES */
 int
 
 /* BSD VM COMPONENT INTERFACES */
 int


@@ -59,24 +72,36 @@ get_map_nentries(
        return(map->hdr.nentries);
 }
 
        return(map->hdr.nentries);
 }
 

+mach_vm_offset_t
+mach_get_vm_start(vm_map_t map)
+{
+       return( vm_map_first_entry(map)->vme_start);
+}
+
+mach_vm_offset_t
+mach_get_vm_end(vm_map_t map)
+{
+       return( vm_map_last_entry(map)->vme_end);
+}
+

 /*
 /*

- * 
+ * Legacy routines to get the start and end for a vm_map_t.  They
+ * return them in the vm_offset_t format.  So, they should only be
+ * called on maps that are the same size as the kernel map for
+ * accurate results.

  */
 vm_offset_t
  */
 vm_offset_t

-get_map_start(
+get_vm_start(

        vm_map_t map)
 {
        vm_map_t map)
 {

-       return(vm_map_first_entry(map)->vme_start);
+       return(CAST_DOWN(vm_offset_t, vm_map_first_entry(map)->vme_start));

 }
 
 }
 

-/*
- * 
- */

 vm_offset_t
 vm_offset_t

-get_map_end(
+get_vm_end(

        vm_map_t map)
 {
        vm_map_t map)
 {

-       return(vm_map_last_entry(map)->vme_end);
+       return(CAST_DOWN(vm_offset_t, vm_map_last_entry(map)->vme_end));

 }
 
 /* 
 }
 
 /* 


@@ -86,150 +111,42 @@ get_map_end(
 /* until component support available */
 int    vnode_pager_workaround;
 
 /* until component support available */
 int    vnode_pager_workaround;
 

-typedef int vnode_port_t;
-

 typedef struct vnode_pager {
 typedef struct vnode_pager {

-       ipc_port_t pager;                       /* pager */
-       ipc_port_t pager_handle;        /* pager handle */
-       ipc_port_t vm_obj_handle;       /* memory object's control handle */
-       vnode_port_t vnode_handle;      /* vnode handle */
+       int                     *pager;         /* pager workaround pointer  */
+       unsigned int            pager_ikot;     /* JMM: fake ip_kotype()     */
+       unsigned int            ref_count;      /* reference count           */
+       memory_object_control_t control_handle; /* mem object control handle */
+       struct vnode            *vnode_handle;  /* vnode handle              */

 } *vnode_pager_t;
 
 } *vnode_pager_t;
 

-typedef struct vnode_port_entry {
-       queue_chain_t   links;          /* queue links */
-       ipc_port_t      name;                   /* port name */
-       vnode_pager_t   pager_rec;      /* pager record */
-} *vnode_port_entry_t;
-
-
-#define         VNODE_PORT_HASH_COUNT   127
-#define         vnode_port_hash(name_port) \
-                        (((int)(name_port) & 0xffffff) % VNODE_PORT_HASH_COUNT)
-
-queue_head_t   vnode_port_hashtable[VNODE_PORT_HASH_COUNT];
-zone_t                 vnode_port_hash_zone;
-decl_simple_lock_data(,vnode_port_hash_lock)
-

 
 ipc_port_t
 
 ipc_port_t

-trigger_name_to_port(
+trigger_name_to_port(                  /* forward */

        mach_port_t);
 
        mach_port_t);
 

-void 
-vnode_pager_bootstrap(
-       void);
-
-void
-vnode_pager_alloc_map(
-       void);
-
-ipc_port_t
-vnode_pager_setup(
-       vnode_port_t,
-       ipc_port_t);
-
-ipc_port_t
-vnode_pager_lookup(
-       vnode_port_t,
-       ipc_port_t);
-
-kern_return_t
-vnode_pager_init(
-       ipc_port_t, 
-       ipc_port_t, 
-       vm_size_t);
-

 kern_return_t
 kern_return_t

-vnode_pager_data_request( 
-       ipc_port_t, 
-       ipc_port_t,
-       vm_object_offset_t, 
-       vm_size_t, 
-       vm_prot_t);
-
-kern_return_t
-vnode_pager_data_return(
-       ipc_port_t,
-       ipc_port_t,
-       vm_object_offset_t,
-       pointer_t,
-       vm_size_t,
-       boolean_t,
-       boolean_t);
-
-void
-vnode_pager_no_senders(
-       ipc_port_t,
-       mach_port_mscount_t);
-
-kern_return_t
-vnode_pager_terminate(
-       ipc_port_t,
-       ipc_port_t);
-
-kern_return_t
-vnode_pager_cluster_read(
+vnode_pager_cluster_read(              /* forward */

        vnode_pager_t, 
        vm_object_offset_t, 
        vm_size_t);
 
 void
        vnode_pager_t, 
        vm_object_offset_t, 
        vm_size_t);
 
 void

-vnode_pager_cluster_write(
+vnode_pager_cluster_write(             /* forward */

        vnode_pager_t,
        vm_object_offset_t,
        vnode_pager_t,
        vm_object_offset_t,

-       vm_size_t);
+       vm_size_t,
+       vm_object_offset_t *,
+       int *,
+       int);

 
 

-kern_return_t
-memory_object_change_attributes(
-        vm_object_t,
-        memory_object_flavor_t,
-        memory_object_info_t,
-        mach_msg_type_number_t,
-        ipc_port_t,
-        mach_msg_type_name_t);
-
-int    
-vnode_pagein(
-       vnode_port_t,
-       upl_t,
-       vm_offset_t,
-       vm_object_offset_t, 
-       int, 
-       int,
-       int *);
-int    
-vnode_pageout(
-       vnode_port_t,
-       upl_t,
-       vm_offset_t,
-       vm_object_offset_t, 
-       int, 
-       int,
-       int *);

 
 vnode_pager_t
 
 vnode_pager_t

-vnode_object_create(
-       vnode_port_t    vp);
-
-void
-vnode_port_hash_init(void);
-
-void
-vnode_port_hash_insert(
-       ipc_port_t,   
-       vnode_pager_t);
+vnode_object_create(                   /* forward */
+       struct vnode *);

 
 vnode_pager_t
 
 vnode_pager_t

-vnode_port_hash_lookup(
-       ipc_port_t);
-
-void
-vnode_port_hash_delete(
-       ipc_port_t);
-
-void
-vnode_pager_release_from_cache(
-       int     *cnt);
+vnode_pager_lookup(                    /* forward */
+       memory_object_t);

 
 zone_t vnode_pager_zone;
 
 
 zone_t vnode_pager_zone;
 


@@ -263,50 +180,97 @@ int pagerdebug=0;
  */
 int
 macx_triggers(
  */
 int
 macx_triggers(

-       int     hi_water,
-       int     low_water,
-       int     flags,
-       mach_port_t     trigger_name)
+       struct macx_triggers_args *args)

 {
 {

+       int     hi_water = args->hi_water;
+       int     low_water = args->low_water;
+       int     flags = args->flags;
+       mach_port_t     trigger_name = args->alert_port;

        kern_return_t kr;
        kern_return_t kr;

-       ipc_port_t              default_pager_port = MACH_PORT_NULL;
+       memory_object_default_t default_pager;

        ipc_port_t              trigger_port;
 
        ipc_port_t              trigger_port;
 

+       default_pager = MEMORY_OBJECT_DEFAULT_NULL;

        kr = host_default_memory_manager(host_priv_self(), 
        kr = host_default_memory_manager(host_priv_self(), 

-                                       &default_pager_port, 0);
+                                       &default_pager, 0);

        if(kr != KERN_SUCCESS) {
                return EINVAL;
        }
        if(kr != KERN_SUCCESS) {
                return EINVAL;
        }

-       trigger_port = trigger_name_to_port(trigger_name);
-       if(trigger_port == NULL) {
+
+       if ((flags & SWAP_ENCRYPT_ON) &&
+           (flags & SWAP_ENCRYPT_OFF)) {
+               /* can't have it both ways */

                return EINVAL;
        }
                return EINVAL;
        }

-        /* trigger_port is locked and active */
-        ip_unlock(trigger_port);
-       default_pager_triggers(default_pager_port, 
-                       hi_water, low_water, flags, trigger_port);
-       ipc_port_make_send(trigger_port); 
+
+       if (flags & SWAP_ENCRYPT_ON) {
+               /* ENCRYPTED SWAP: tell default_pager to encrypt */
+               default_pager_triggers(default_pager,
+                                      0, 0,
+                                      SWAP_ENCRYPT_ON,
+                                      IP_NULL);
+       } else if (flags & SWAP_ENCRYPT_OFF) {
+               /* ENCRYPTED SWAP: tell default_pager not to encrypt */
+               default_pager_triggers(default_pager,
+                                      0, 0,
+                                      SWAP_ENCRYPT_OFF,
+                                      IP_NULL);
+       }
+
+       if (flags & HI_WAT_ALERT) {
+               trigger_port = trigger_name_to_port(trigger_name);
+               if(trigger_port == NULL) {
+                       return EINVAL;
+               }
+               /* trigger_port is locked and active */
+               ipc_port_make_send_locked(trigger_port); 
+               /* now unlocked */
+               default_pager_triggers(default_pager, 
+                                      hi_water, low_water,
+                                      HI_WAT_ALERT, trigger_port);
+       }
+
+       if (flags & LO_WAT_ALERT) {
+               trigger_port = trigger_name_to_port(trigger_name);
+               if(trigger_port == NULL) {
+                       return EINVAL;
+               }
+               /* trigger_port is locked and active */
+               ipc_port_make_send_locked(trigger_port);
+               /* and now its unlocked */
+               default_pager_triggers(default_pager, 
+                                      hi_water, low_water,
+                                      LO_WAT_ALERT, trigger_port);
+       }

 
        /*
         * Set thread scheduling priority and policy for the current thread
         * it is assumed for the time being that the thread setting the alert
 
        /*
         * Set thread scheduling priority and policy for the current thread
         * it is assumed for the time being that the thread setting the alert

-        * is the same one which will be servicing it. 
+        * is the same one which will be servicing it.
+        *
+        * XXX This does not belong in the kernel XXX

         */
        {
         */
        {

-               struct policy_timeshare_base     fifo_base;
-               struct policy_timeshare_limit fifo_limit;
-               policy_base_t   base;
-               processor_set_t pset;
-               policy_limit_t  limit;
-
-               pset = (current_thread())->processor_set;  
-               base = (policy_base_t) &fifo_base;
-               limit = (policy_limit_t) &fifo_limit;
-               fifo_limit.max_priority = fifo_base.base_priority = MAXPRI_STANDARD;
-               thread_set_policy((current_thread())->top_act, pset, POLICY_FIFO, base, POLICY_TIMESHARE_BASE_COUNT, limit, POLICY_TIMESHARE_LIMIT_COUNT);
+               thread_precedence_policy_data_t         pre;
+               thread_extended_policy_data_t           ext;
+
+               ext.timeshare = FALSE;
+               pre.importance = INT32_MAX;
+
+               thread_policy_set(current_thread(),
+                                 THREAD_EXTENDED_POLICY,
+                                 (thread_policy_t)&ext,
+                                 THREAD_EXTENDED_POLICY_COUNT);
+
+               thread_policy_set(current_thread(),
+                                 THREAD_PRECEDENCE_POLICY,
+                                 (thread_policy_t)&pre,
+                                 THREAD_PRECEDENCE_POLICY_COUNT);

        }
  
        }
  

-       current_thread()->vm_privilege = TRUE;
+       current_thread()->options |= TH_OPT_VMPRIV;
+
+       return 0;

 }
 
 /*
 }
 
 /*


@@ -329,6 +293,126 @@ trigger_name_to_port(
        return trigger_port;
 }
 
        return trigger_port;
 }
 

+
+extern int     uiomove64(addr64_t, int, void *);
+#define        MAX_RUN 32
+
+int
+memory_object_control_uiomove(
+       memory_object_control_t control,
+       memory_object_offset_t  offset,
+       void            *       uio,
+       int                     start_offset,
+       int                     io_requested,
+       int                     mark_dirty)
+{
+       vm_object_t             object;
+       vm_page_t               dst_page;
+       int                     xsize;
+       int                     retval = 0;
+       int                     cur_run;
+       int                     cur_needed;
+       int                     i;
+       vm_page_t               page_run[MAX_RUN];
+
+
+       object = memory_object_control_to_vm_object(control);
+       if (object == VM_OBJECT_NULL) {
+               return (0);
+       }
+       assert(!object->internal);
+
+       vm_object_lock(object);
+
+       if (mark_dirty && object->copy != VM_OBJECT_NULL) {
+               /*
+                * We can't modify the pages without honoring
+                * copy-on-write obligations first, so fall off
+                * this optimized path and fall back to the regular
+                * path.
+                */
+               vm_object_unlock(object);
+               return 0;
+       }
+           
+       while (io_requested && retval == 0) {
+
+               cur_needed = (start_offset + io_requested + (PAGE_SIZE - 1)) / PAGE_SIZE;
+
+               if (cur_needed > MAX_RUN)
+                       cur_needed = MAX_RUN;
+
+               for (cur_run = 0; cur_run < cur_needed; ) {
+
+                       if ((dst_page = vm_page_lookup(object, offset)) == VM_PAGE_NULL)
+                               break;
+                       /*
+                        * Sync up on getting the busy bit
+                        */
+                       if ((dst_page->busy || dst_page->cleaning)) {
+                               /*
+                                * someone else is playing with the page... if we've
+                                * already collected pages into this run, go ahead
+                                * and process now, we can't block on this
+                                * page while holding other pages in the BUSY state
+                                * otherwise we will wait
+                                */
+                               if (cur_run)
+                                       break;
+                               PAGE_SLEEP(object, dst_page, THREAD_UNINT);
+                               continue;
+                       }
+                       /*
+                        * this routine is only called when copying
+                        * to/from real files... no need to consider
+                        * encrypted swap pages
+                        */
+                       assert(!dst_page->encrypted);
+
+                       if (mark_dirty)
+                               dst_page->dirty = TRUE;
+                       dst_page->busy = TRUE;
+
+                       page_run[cur_run++] = dst_page;
+
+                       offset += PAGE_SIZE_64;
+               }
+               if (cur_run == 0)
+                       /*
+                        * we hit a 'hole' in the cache
+                        * we bail at this point
+                        * we'll unlock the object below
+                        */
+                       break;
+               vm_object_unlock(object);
+
+               for (i = 0; i < cur_run; i++) {
+                 
+                       dst_page = page_run[i];
+
+                       if ((xsize = PAGE_SIZE - start_offset) > io_requested)
+                               xsize = io_requested;
+
+                       if ( (retval = uiomove64((addr64_t)(((addr64_t)(dst_page->phys_page) << 12) + start_offset), xsize, uio)) )
+                               break;
+
+                       io_requested -= xsize;
+                       start_offset = 0;
+               }
+               vm_object_lock(object);
+
+               for (i = 0; i < cur_run; i++) {
+                       dst_page = page_run[i];
+
+                       PAGE_WAKEUP_DONE(dst_page);
+               }
+       }
+       vm_object_unlock(object);
+
+       return (retval);
+}
+
+

 /*
  *
  */
 /*
  *
  */


@@ -340,100 +424,51 @@ vnode_pager_bootstrap(void)
        size = (vm_size_t) sizeof(struct vnode_pager);
        vnode_pager_zone = zinit(size, (vm_size_t) MAX_VNODE*size,
                                PAGE_SIZE, "vnode pager structures");
        size = (vm_size_t) sizeof(struct vnode_pager);
        vnode_pager_zone = zinit(size, (vm_size_t) MAX_VNODE*size,
                                PAGE_SIZE, "vnode pager structures");

-       vnode_port_hash_init();
-

        return;
 }
 
 /*
  *
  */
        return;
 }
 
 /*
  *
  */

-ipc_port_t
+memory_object_t

 vnode_pager_setup(
 vnode_pager_setup(

-       vnode_port_t    vp,
-       ipc_port_t      pager)
+       struct vnode    *vp,
+       __unused memory_object_t        pager)

 {
        vnode_pager_t   vnode_object;
 {
        vnode_pager_t   vnode_object;

-       kern_return_t   kr;
-       ipc_port_t      previous;
-
-       if (pager &&
-           (vnode_object = vnode_port_hash_lookup(pager))) {
-               if (vnode_object->vnode_handle == vp) 
-                       return(pager);
-       }

 
        vnode_object = vnode_object_create(vp);
        if (vnode_object == VNODE_PAGER_NULL)
                panic("vnode_pager_setup: vnode_object_create() failed");
 
        vnode_object = vnode_object_create(vp);
        if (vnode_object == VNODE_PAGER_NULL)
                panic("vnode_pager_setup: vnode_object_create() failed");

-
-       vnode_object->pager = ipc_port_alloc_kernel();
-       assert (vnode_object->pager != IP_NULL);
-       pager_mux_hash_insert(vnode_object->pager, 
-                       (rpc_subsystem_t)&vnode_pager_workaround);
-
-       vnode_object->pager_handle = ipc_port_make_send(vnode_object->pager);
-
-       vnode_port_hash_insert(vnode_object->pager_handle, vnode_object);
-
-       ipc_port_make_sonce(vnode_object->pager);
-       ip_lock(vnode_object->pager);   /* unlocked in nsrequest below */
-       ipc_port_nsrequest(vnode_object->pager, 1, vnode_object->pager, &previous);
-
-       PAGER_DEBUG(PAGER_INIT, ("vnode_pager_setup: vp %x pager %x vnode_pager %x\n", vp, vnode_object->pager_handle, vnode_object));
-
-       ubc_setpager( vp, vnode_object->pager_handle);
-       return(vnode_object->pager_handle);
-}
-
-/*
- *
- */
-ipc_port_t
-vnode_pager_lookup(
-       vnode_port_t    vp,
-       ipc_port_t      pager)
-{
-       vnode_pager_t   vnode_object;
-       kern_return_t   kr;
-
-       if (pager &&
-           (vnode_object = vnode_port_hash_lookup(pager))) {
-               if (vnode_object->vnode_handle == vp) 
-                       return(vnode_object->vm_obj_handle);
-               else 
-                       return NULL;
-       }
-       else 
-               return NULL;
+       return((memory_object_t)vnode_object);

 }
 
 /*
  *
  */
 kern_return_t
 }
 
 /*
  *
  */
 kern_return_t

-vnode_pager_init(ipc_port_t pager, 
-               ipc_port_t pager_request, 
-               vm_size_t pg_size)
+vnode_pager_init(memory_object_t mem_obj, 
+               memory_object_control_t control, 
+#if !DEBUG
+                __unused
+#endif
+                vm_size_t pg_size)

 {
        vnode_pager_t   vnode_object;
        kern_return_t   kr;
        memory_object_attr_info_data_t  attributes;
 {
        vnode_pager_t   vnode_object;
        kern_return_t   kr;
        memory_object_attr_info_data_t  attributes;

-       vm_object_t     vm_object;

 
 
 
 

-       PAGER_DEBUG(PAGER_ALL, ("vnode_pager_init: %x, %x, %x\n", pager, pager_request, pg_size));
+       PAGER_DEBUG(PAGER_ALL, ("vnode_pager_init: %p, %p, %x\n", mem_obj, control, pg_size));

 
 

-       vnode_object = vnode_port_hash_lookup(pager);
-       if (vnode_object == VNODE_PAGER_NULL)
-               panic("vnode_pager_init: lookup failed");
+       if (control == MEMORY_OBJECT_CONTROL_NULL)
+               return KERN_INVALID_ARGUMENT;

 
 

-       vnode_object->vm_obj_handle = pager_request;
+       vnode_object = vnode_pager_lookup(mem_obj);

 
 

-       vm_object = vm_object_lookup(pager_request);
+       memory_object_control_reference(control);

 
 

-       if (vm_object == VM_OBJECT_NULL)
-               panic("vnode_pager_init: vm_object_lookup() failed");
+       vnode_object->control_handle = control;

 
        attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY;
        /* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/
 
        attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY;
        /* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/


@@ -442,11 +477,10 @@ vnode_pager_init(ipc_port_t pager,
        attributes.temporary = TRUE;
 
        kr = memory_object_change_attributes(
        attributes.temporary = TRUE;
 
        kr = memory_object_change_attributes(

-                                       vm_object,
+                                       control,

                                        MEMORY_OBJECT_ATTRIBUTE_INFO,
                                        (memory_object_info_t) &attributes,
                                        MEMORY_OBJECT_ATTRIBUTE_INFO,
                                        (memory_object_info_t) &attributes,

-                                       MEMORY_OBJECT_ATTR_INFO_COUNT,
-                                       MACH_PORT_NULL, 0);
+                                       MEMORY_OBJECT_ATTR_INFO_COUNT);

        if (kr != KERN_SUCCESS)
                panic("vnode_pager_init: memory_object_change_attributes() failed");
 
        if (kr != KERN_SUCCESS)
                panic("vnode_pager_init: memory_object_change_attributes() failed");
 


@@ -458,78 +492,115 @@ vnode_pager_init(ipc_port_t pager,
  */
 kern_return_t
 vnode_pager_data_return(
  */
 kern_return_t
 vnode_pager_data_return(

-        ipc_port_t             mem_obj,
-        ipc_port_t             control_port,
-        vm_object_offset_t     offset,
-        pointer_t              addr,
+        memory_object_t                mem_obj,
+        memory_object_offset_t offset,

         vm_size_t              data_cnt,
         vm_size_t              data_cnt,

-        boolean_t              dirty,
-        boolean_t              kernel_copy)  
+        memory_object_offset_t *resid_offset,
+       int                     *io_error,
+       __unused boolean_t              dirty,
+       __unused boolean_t              kernel_copy,
+       int                     upl_flags)  

 {
        register vnode_pager_t  vnode_object;
 
 {
        register vnode_pager_t  vnode_object;
 

-       vnode_object = vnode_port_hash_lookup(mem_obj);
-       if (vnode_object == VNODE_PAGER_NULL)
-               panic("vnode_pager_data_return: lookup failed");
+       vnode_object = vnode_pager_lookup(mem_obj);

 
 

-       vnode_pager_cluster_write(vnode_object, offset, data_cnt);
+       vnode_pager_cluster_write(vnode_object, offset, data_cnt, resid_offset, io_error, upl_flags);

 
        return KERN_SUCCESS;
 }
 
 
        return KERN_SUCCESS;
 }
 

+kern_return_t
+vnode_pager_data_initialize(
+       __unused memory_object_t                mem_obj,
+       __unused memory_object_offset_t offset,
+       __unused vm_size_t              data_cnt)
+{
+       panic("vnode_pager_data_initialize");
+       return KERN_FAILURE;
+}
+
+kern_return_t
+vnode_pager_data_unlock(
+       __unused memory_object_t                mem_obj,
+       __unused memory_object_offset_t offset,
+       __unused vm_size_t              size,
+       __unused vm_prot_t              desired_access)
+{
+       return KERN_FAILURE;
+}
+
+kern_return_t
+vnode_pager_get_object_size(
+       memory_object_t         mem_obj,
+       memory_object_offset_t  *length)
+{
+       vnode_pager_t   vnode_object;
+
+       vnode_object = vnode_pager_lookup(mem_obj);
+
+       *length = vnode_pager_get_filesize(vnode_object->vnode_handle);
+       return KERN_SUCCESS;
+}
+

 /*
  *
  */
 kern_return_t  
 vnode_pager_data_request(
 /*
  *
  */
 kern_return_t  
 vnode_pager_data_request(

-       ipc_port_t              mem_obj,
-       ipc_port_t              mem_obj_control,
-       vm_object_offset_t      offset,
+       memory_object_t         mem_obj,
+       memory_object_offset_t  offset,

        vm_size_t               length,
        vm_size_t               length,

-       vm_prot_t               protection_required)
+#if !DEBUG
+       __unused
+#endif
+vm_prot_t              protection_required)

 {
        register vnode_pager_t  vnode_object;
 
 {
        register vnode_pager_t  vnode_object;
 

-       PAGER_DEBUG(PAGER_ALL, ("vnode_pager_data_request: %x, %x, %x, %x, %x\n", mem_obj, mem_obj_control, offset, length, protection_required));
+       PAGER_DEBUG(PAGER_ALL, ("vnode_pager_data_request: %x, %x, %x, %x\n", mem_obj, offset, length, protection_required));

 
 

-       vnode_object = vnode_port_hash_lookup(mem_obj);
+       vnode_object = vnode_pager_lookup(mem_obj);

 
 

-       PAGER_DEBUG(PAGER_PAGEIN, ("vnode_pager_data_request: %x, %x, %x, %x, %x, vnode_object %x\n", mem_obj, mem_obj_control, offset, length, protection_required, vnode_object));
+       PAGER_DEBUG(PAGER_PAGEIN, ("vnode_pager_data_request: %x, %x, %x, %x, vnode_object %x\n", mem_obj, offset, length, protection_required, vnode_object));

                
                

-       if (vnode_object == VNODE_PAGER_NULL)
-               panic("vnode_pager_data_request: lookup failed");
+       return vnode_pager_cluster_read(vnode_object, offset, length);
+}

 
 

-       vnode_pager_cluster_read(vnode_object, offset, length);
+/*
+ *
+ */
+void
+vnode_pager_reference(
+       memory_object_t         mem_obj)
+{      
+       register vnode_pager_t  vnode_object;
+       unsigned int            new_ref_count;

 
 

-       return KERN_SUCCESS;
+       vnode_object = vnode_pager_lookup(mem_obj);
+       new_ref_count = hw_atomic_add(&vnode_object->ref_count, 1);
+       assert(new_ref_count > 1);

 }
 
 /*
  *
  */
 void
 }
 
 /*
  *
  */
 void

-vnode_pager_no_senders(
-       ipc_port_t      mem_obj,
-       mach_port_mscount_t mscount)
+vnode_pager_deallocate(
+       memory_object_t         mem_obj)

 {
        register vnode_pager_t  vnode_object;
 
 {
        register vnode_pager_t  vnode_object;
 

-       PAGER_DEBUG(PAGER_ALL, ("vnode_pager_nosenders: %x, %x\n", mem_obj, mscount));
-
-       vnode_object = vnode_port_hash_lookup(mem_obj);
-       if (vnode_object == VNODE_PAGER_NULL)
-               panic("vnode_pager_no_senders: lookup failed");
+       PAGER_DEBUG(PAGER_ALL, ("vnode_pager_deallocate: %x\n", mem_obj));

 
 

-       assert(vnode_object->pager_handle == mem_obj);
+       vnode_object = vnode_pager_lookup(mem_obj);

 
 

-       pager_mux_hash_delete((ipc_port_t) vnode_object->pager_handle);
-       ipc_port_dealloc_kernel(vnode_object->pager);
-       vnode_port_hash_delete(vnode_object->pager_handle);
-       if (vnode_object->vnode_handle != (vnode_port_t) NULL) {
-               vnode_pager_vrele(vnode_object->vnode_handle);
+       if (hw_atomic_sub(&vnode_object->ref_count, 1) == 0) {
+               if (vnode_object->vnode_handle != NULL) {
+                       vnode_pager_vrele(vnode_object->vnode_handle);
+               }
+               zfree(vnode_pager_zone, vnode_object);

        }
        }

-       zfree(vnode_pager_zone, (vm_offset_t) vnode_object);
-

        return;
 }
 
        return;
 }
 


@@ -538,50 +609,55 @@ vnode_pager_no_senders(
  */
 kern_return_t
 vnode_pager_terminate(
  */
 kern_return_t
 vnode_pager_terminate(

-       ipc_port_t      mem_obj,
-       ipc_port_t      mem_obj_control)
+#if !DEBUG
+       __unused
+#endif
+       memory_object_t mem_obj)

 {
 {

-       register vnode_pager_t  vnode_object;
-
-       PAGER_DEBUG(PAGER_ALL, ("vnode_pager_terminate: %x, %x\n", mem_obj, mem_obj_control));
+       PAGER_DEBUG(PAGER_ALL, ("vnode_pager_terminate: %x\n", mem_obj));

 
 

-       vnode_object = vnode_port_hash_lookup(mem_obj);
-       if (vnode_object == VNODE_PAGER_NULL)
-               panic("vnode_pager_terminate: lookup failed");
+       return(KERN_SUCCESS);
+}

 
 

-       assert(vnode_object->pager_handle == mem_obj);
+/*
+ *
+ */
+kern_return_t
+vnode_pager_synchronize(
+       memory_object_t         mem_obj,
+       memory_object_offset_t  offset,
+       vm_size_t               length,
+       __unused vm_sync_t              sync_flags)
+{
+       register vnode_pager_t  vnode_object;

 
 

-       /* release extra send right created by the fact that the caller */
-       /* of vnode_pager_setup does not establish a mapping between a  */
-       /* cache object and the mem_obj (AMO).  When a subsequent vm_map */
-       /* is done, vm_map will bump the send right count */
-       ipc_port_release_send(mem_obj);
+       PAGER_DEBUG(PAGER_ALL, ("vnode_pager_synchronize: %x\n", mem_obj));

 
 

-       /* release a send right because terminate is called directly and */
-       /* not through IPC, the right won't disappear quietly */
-       ipc_port_release_send(mem_obj);
+       vnode_object = vnode_pager_lookup(mem_obj);

 
 

-       ipc_port_dealloc_kernel(mem_obj_control);
+       memory_object_synchronize_completed(vnode_object->control_handle, offset, length);

 
 

-       return(KERN_SUCCESS);
+       return (KERN_SUCCESS);

 }
 
 /*
  *
  */
 kern_return_t
 }
 
 /*
  *
  */
 kern_return_t

-vnode_pager_synchronize(
-       ipc_port_t              pager,
-       ipc_port_t              pager_request,
-       vm_object_offset_t      offset,
-       vm_offset_t             length,
-       vm_sync_t               sync_flags)
+vnode_pager_unmap(
+       memory_object_t         mem_obj)

 {
 {

-       memory_object_synchronize_completed(vm_object_lookup(pager_request), offset, length);
+       register vnode_pager_t  vnode_object;

 
 

-       return (KERN_SUCCESS);
+       PAGER_DEBUG(PAGER_ALL, ("vnode_pager_unmap: %x\n", mem_obj));
+
+       vnode_object = vnode_pager_lookup(mem_obj);
+
+       ubc_unmap(vnode_object->vnode_handle);
+       return KERN_SUCCESS;

 }
 
 }
 

+

 /*
  *
  */
 /*
  *
  */


@@ -589,35 +665,115 @@ void
 vnode_pager_cluster_write(
        vnode_pager_t           vnode_object,
        vm_object_offset_t      offset,
 vnode_pager_cluster_write(
        vnode_pager_t           vnode_object,
        vm_object_offset_t      offset,

-       vm_size_t               cnt)
+       vm_size_t               cnt,
+       vm_object_offset_t   *  resid_offset,
+       int                  *  io_error,
+       int                     upl_flags)

 {
 {

-       int             error = 0;
-       int             local_error = 0;
-       int             kret;
-       int             size;
+        vm_size_t       size;
+       upl_t           upl = NULL;
+       int             request_flags;
+       int             errno;

 
 

-       if (cnt & PAGE_MASK) {
-               panic("vs_cluster_write: cnt not a multiple of PAGE_SIZE");
-       }
-       size = (cnt < (PAGE_SIZE*32)) ? cnt : (PAGE_SIZE*32); /* effective min */
-       
-       while (cnt) {
+       if (upl_flags & UPL_MSYNC) {

 
 

-               kret = vnode_pageout(vnode_object->vnode_handle, (upl_t )NULL, (vm_offset_t)NULL, offset, size, 0, &local_error);
+               upl_flags |= UPL_VNODE_PAGER;

 
 

-               if (local_error != 0) {
-                       error = local_error;
-                       local_error = 0;
+               if ( (upl_flags & UPL_IOSYNC) && io_error)
+                       upl_flags |= UPL_KEEPCACHED;
+
+               while (cnt) {
+                       kern_return_t   kr;
+
+                       size = (cnt < (PAGE_SIZE * MAX_UPL_TRANSFER)) ? cnt : (PAGE_SIZE * MAX_UPL_TRANSFER); /* effective max */
+
+                       request_flags = UPL_RET_ONLY_DIRTY | UPL_COPYOUT_FROM | UPL_CLEAN_IN_PLACE |
+                                       UPL_SET_INTERNAL | UPL_SET_LITE;
+
+                       kr = memory_object_upl_request(vnode_object->control_handle, 
+                                                      offset, size, &upl, NULL, NULL, request_flags);
+                       if (kr != KERN_SUCCESS)
+                               panic("vnode_pager_cluster_write: upl request failed\n");
+
+                       vnode_pageout(vnode_object->vnode_handle, 
+                                     upl, (vm_offset_t)0, offset, size, upl_flags, &errno);
+
+                       if ( (upl_flags & UPL_KEEPCACHED) ) {
+                               if ( (*io_error = errno) )
+                                       break;
+                       }
+                       cnt    -= size;
+                       offset += size;
+               }
+               if (resid_offset)
+                       *resid_offset = offset;
+
+       } else {
+               vm_object_offset_t      vnode_size;
+               vm_object_offset_t      base_offset;
+               vm_object_t             object;
+               vm_page_t               target_page;
+               int                     ticket;
+
+               /*
+                * this is the pageout path
+                */
+               vnode_size = vnode_pager_get_filesize(vnode_object->vnode_handle);
+
+               if (vnode_size > (offset + PAGE_SIZE)) {
+                       /*
+                        * preset the maximum size of the cluster
+                        * and put us on a nice cluster boundary...
+                        * and then clip the size to insure we
+                        * don't request past the end of the underlying file
+                        */
+                       size = PAGE_SIZE * MAX_UPL_TRANSFER;
+                       base_offset = offset & ~((signed)(size - 1));
+
+                       if ((base_offset + size) > vnode_size)
+                               size = round_page_32(((vm_size_t)(vnode_size - base_offset)));
+               } else {
+                       /*
+                        * we've been requested to page out a page beyond the current
+                        * end of the 'file'... don't try to cluster in this case...
+                        * we still need to send this page through because it might
+                        * be marked precious and the underlying filesystem may need
+                        * to do something with it (besides page it out)...
+                        */
+                       base_offset = offset;
+                       size = PAGE_SIZE;
+               }
+               object = memory_object_control_to_vm_object(vnode_object->control_handle);
+
+               if (object == VM_OBJECT_NULL)
+                       panic("vnode_pager_cluster_write: NULL vm_object in control handle\n");
+
+               request_flags = UPL_NOBLOCK | UPL_FOR_PAGEOUT | UPL_CLEAN_IN_PLACE |
+                               UPL_RET_ONLY_DIRTY | UPL_COPYOUT_FROM |
+                               UPL_SET_INTERNAL | UPL_SET_LITE;
+
+               vm_object_lock(object);
+
+               if ((target_page = vm_page_lookup(object, offset)) != VM_PAGE_NULL) {
+                       /*
+                        * only pick up pages whose ticket number matches
+                        * the ticket number of the page orginally targeted
+                        * for pageout
+                        */
+                       ticket = target_page->page_ticket;
+
+                       request_flags |= ((ticket << UPL_PAGE_TICKET_SHIFT) & UPL_PAGE_TICKET_MASK);

                }
                }

-               cnt -= size;
-               offset += size;
+               vm_object_unlock(object);
+
+               vm_object_upl_request(object, base_offset, size,
+                                     &upl, NULL, NULL, request_flags);
+               if (upl == NULL)
+                       panic("vnode_pager_cluster_write: upl request failed\n");
+
+               vnode_pageout(vnode_object->vnode_handle,
+                              upl, (vm_offset_t)0, upl->offset, upl->size, UPL_VNODE_PAGER, NULL);

        }
        }

-#if 0
-       if (error != 0)
-               return(KERN_FAILURE);
-       
-       return(KERN_SUCCESS);
-#endif /* 0 */

 }
 
 
 }
 
 


@@ -630,32 +786,54 @@ vnode_pager_cluster_read(
        vm_object_offset_t      offset,
        vm_size_t               cnt)
 {
        vm_object_offset_t      offset,
        vm_size_t               cnt)
 {

-       int             error = 0;

        int             local_error = 0;
        int             kret;
        int             local_error = 0;
        int             kret;

-       int             size;

 
 

-       if(cnt & PAGE_MASK) {
-               panic("vs_cluster_read: cnt not a multiple of PAGE_SIZE");
-       }
-
-       size = PAGE_SIZE;
-
-       while (cnt) {
-
-               kret = vnode_pagein(vnode_object->vnode_handle, (upl_t)NULL, (vm_offset_t)NULL, offset, size, 0, &local_error);
+       assert(! (cnt & PAGE_MASK));

 
 

-               if (local_error != 0) {
-                       error = local_error;
-                       local_error = 0;
+       kret = vnode_pagein(vnode_object->vnode_handle,
+                           (upl_t) NULL,
+                           (vm_offset_t) NULL,
+                           offset,
+                           cnt,
+                           0,
+                           &local_error);
+/*
+       if(kret == PAGER_ABSENT) {
+       Need to work out the defs here, 1 corresponds to PAGER_ABSENT 
+       defined in bsd/vm/vm_pager.h  However, we should not be including 
+       that file here it is a layering violation.
+*/
+       if (kret == 1) {
+               int     uplflags;
+               upl_t   upl = NULL;
+               int     count = 0;
+               kern_return_t   kr;
+
+               uplflags = (UPL_NO_SYNC |
+                           UPL_CLEAN_IN_PLACE |
+                           UPL_SET_INTERNAL);
+               count = 0;
+               kr = memory_object_upl_request(vnode_object->control_handle,
+                                              offset, cnt,
+                                              &upl, NULL, &count, uplflags);
+               if (kr == KERN_SUCCESS) {
+                       upl_abort(upl, 0);
+                       upl_deallocate(upl);
+               } else {
+                       /*
+                        * We couldn't gather the page list, probably
+                        * because the memory object doesn't have a link
+                        * to a VM object anymore (forced unmount, for
+                        * example).  Just return an error to the vm_fault()
+                        * path and let it handle it.
+                        */

                }
                }

-               cnt -= size;
-               offset += size;
+
+               return KERN_FAILURE;

        }
        }

-       if (error != 0)
-               return(KERN_FAILURE);

 
 

-       return(KERN_SUCCESS);
+       return KERN_SUCCESS;

 
 }
 
 
 }
 


@@ -668,7 +846,7 @@ vnode_pager_release_from_cache(
                int     *cnt)
 {
        memory_object_free_from_cache(
                int     *cnt)
 {
        memory_object_free_from_cache(

-                       &realhost, (int)&vnode_pager_workaround, cnt);
+                       &realhost, &vnode_pager_workaround, cnt);

 }
 
 /*
 }
 
 /*


@@ -676,116 +854,41 @@ vnode_pager_release_from_cache(
  */
 vnode_pager_t
 vnode_object_create(
  */
 vnode_pager_t
 vnode_object_create(

-        vnode_port_t   vp)
+        struct vnode *vp)

 {
        register vnode_pager_t  vnode_object;
 
        vnode_object = (struct vnode_pager *) zalloc(vnode_pager_zone);
        if (vnode_object == VNODE_PAGER_NULL)
                return(VNODE_PAGER_NULL);
 {
        register vnode_pager_t  vnode_object;
 
        vnode_object = (struct vnode_pager *) zalloc(vnode_pager_zone);
        if (vnode_object == VNODE_PAGER_NULL)
                return(VNODE_PAGER_NULL);

-       vnode_object->pager_handle = IP_NULL;
-       vnode_object->vm_obj_handle = IP_NULL;
-       vnode_object->vnode_handle = vp;
-
-       return(vnode_object);
-}
-
-/*
- *
- */
-void
-vnode_port_hash_init(void)
-{
-       register vm_size_t      size;
-       register int            i;
-
-
-       size = (vm_size_t) sizeof(struct vnode_port_entry);
-
-       vnode_port_hash_zone = zinit(size,
-                                    (vm_size_t) MAX_VNODE * size,
-                                    PAGE_SIZE, "vnode_pager port hash");
-
-       for (i = 0; i < VNODE_PORT_HASH_COUNT; i++) 
-               queue_init(&vnode_port_hashtable[i]);
-
-       simple_lock_init(&vnode_port_hash_lock,ETAP_NO_TRACE);
-}
-
-/*
- *
- */
-void
-vnode_port_hash_insert(
-       ipc_port_t              name_port,
-       vnode_pager_t   rec)
-{
-       register vnode_port_entry_t     new_entry;

 
 

-       new_entry = (vnode_port_entry_t) zalloc(vnode_port_hash_zone);

        /*
        /*

-        * TODO: Delete the following check once MAX_VNODE is removed 
+        * The vm_map call takes both named entry ports and raw memory
+        * objects in the same parameter.  We need to make sure that
+        * vm_map does not see this object as a named entry port.  So,
+        * we reserve the second word in the object for a fake ip_kotype
+        * setting - that will tell vm_map to use it as a memory object.

         */
         */

-       if (!new_entry)
-               panic("vnode_port_hash_insert: no space");
-       new_entry->name = name_port;
-       new_entry->pager_rec = rec;
-
-       simple_lock(&vnode_port_hash_lock);
-       queue_enter(&vnode_port_hashtable[vnode_port_hash(name_port)],
-                       new_entry, vnode_port_entry_t, links);
-       simple_unlock(&vnode_port_hash_lock);
+       vnode_object->pager = &vnode_pager_workaround;
+       vnode_object->pager_ikot = IKOT_MEMORY_OBJECT;
+       vnode_object->ref_count = 1;
+       vnode_object->control_handle = MEMORY_OBJECT_CONTROL_NULL;
+       vnode_object->vnode_handle = vp;
+
+       return(vnode_object);

 }
 
 /*
  *
  */
 vnode_pager_t
 }
 
 /*
  *
  */
 vnode_pager_t

-vnode_port_hash_lookup(
-       ipc_port_t       name_port)
+vnode_pager_lookup(
+       memory_object_t  name)

 {
 {

-       register queue_t                bucket;
-       register vnode_port_entry_t     entry;
-       vnode_pager_t                   rec;
-
-       bucket = (queue_t) &vnode_port_hashtable[vnode_port_hash(name_port)];
-
-       simple_lock(&vnode_port_hash_lock);
-       entry = (vnode_port_entry_t) queue_first(bucket);
-       while (!queue_end(bucket,&entry->links)) {
-               if (entry->name == name_port) {
-                       rec = entry->pager_rec;
-                       simple_unlock(&vnode_port_hash_lock);
-                       return(rec);
-               }
-               entry = (vnode_port_entry_t)queue_next(&entry->links);
-       }
-       simple_unlock(&vnode_port_hash_lock);
-       return(VNODE_PAGER_NULL);
-}
+       vnode_pager_t   vnode_object;

 
 

-/*
- *
- */
-void
-vnode_port_hash_delete(
-       ipc_port_t name_port)
-{
-       register queue_t bucket;
-       register vnode_port_entry_t entry;
-
-       bucket = (queue_t) &vnode_port_hashtable[vnode_port_hash(name_port)];
-
-       simple_lock(&vnode_port_hash_lock);
-       entry = (vnode_port_entry_t) queue_first(bucket);
-       while (!queue_end(bucket,&entry->links)) {
-               if (entry->name == name_port) {
-                       queue_remove(bucket, entry, vnode_port_entry_t,links);
-                       simple_unlock(&vnode_port_hash_lock);
-                       zfree(vnode_port_hash_zone, (vm_offset_t) entry);
-                       return;
-               }
-               entry = (vnode_port_entry_t)queue_next(&entry->links);
-       }
-       simple_unlock(&vnode_port_hash_lock);
+       vnode_object = (vnode_pager_t)name;
+       assert(vnode_object->pager == &vnode_pager_workaround);
+       return (vnode_object);

 }
 }

+