xnu-7195.81.3.tar.gz

[apple/xnu.git] / osfmk / vm / device_vm.c

diff --git a/osfmk/vm/device_vm.c b/osfmk/vm/device_vm.c
index c7fdc88bc342b36dfd507e735d720e1c97505938..76e537501d969589080b3b88b6ec84a907888ee4 100644 (file)
--- a/osfmk/vm/device_vm.c
+++ b/osfmk/vm/device_vm.c
@@ -1,49 +1,62 @@
 /*
 /*

- * Copyright (c) 2000-2001 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2020 Apple Inc. All rights reserved.

  *
  *

- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ *
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ *
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ *
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER

  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,

- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@

  */
 
 #include <sys/errno.h>
  */
 
 #include <sys/errno.h>

-#include <kern/host.h>
+

 #include <mach/mach_types.h>
 #include <mach/mach_types.h>

-#include <vm/vm_map.h>
-#include <vm/vm_kern.h>
-#include <vm/vm_pageout.h>

 #include <mach/kern_return.h>
 #include <mach/kern_return.h>

+#include <mach/memory_object_control.h>

 #include <mach/memory_object_types.h>
 #include <mach/port.h>
 #include <mach/policy.h>
 #include <mach/memory_object_types.h>
 #include <mach/port.h>
 #include <mach/policy.h>

+#include <mach/upl.h>
+#include <kern/kern_types.h>
+#include <kern/ipc_kobject.h>
+#include <kern/host.h>
+#include <kern/thread.h>

 #include <ipc/ipc_port.h>
 #include <ipc/ipc_space.h>
 #include <ipc/ipc_port.h>
 #include <ipc/ipc_space.h>

-#include <kern/thread.h>

 #include <device/device_port.h>
 #include <device/device_port.h>

+#include <vm/memory_object.h>

 #include <vm/vm_pageout.h>
 #include <vm/vm_pageout.h>

-
-#include <libkern/OSAtomic.h>
+#include <vm/vm_map.h>
+#include <vm/vm_kern.h>
+#include <vm/vm_pageout.h>
+#include <vm/vm_protos.h>
+#include <mach/sdt.h>
+#include <os/refcnt.h>

 
 
 /* Device VM COMPONENT INTERFACES */
 
 
 
 
 /* Device VM COMPONENT INTERFACES */
 
 

-/* 
- * Device PAGER 
+/*
+ * Device PAGER

  */
 
 
  */
 
 


@@ -52,153 +65,104 @@
 
 
 /* until component support available */
 
 
 /* until component support available */

-int    device_pager_workaround;
-
-typedef int device_port_t;
+const struct memory_object_pager_ops device_pager_ops = {
+       .memory_object_reference = device_pager_reference,
+       .memory_object_deallocate = device_pager_deallocate,
+       .memory_object_init = device_pager_init,
+       .memory_object_terminate = device_pager_terminate,
+       .memory_object_data_request = device_pager_data_request,
+       .memory_object_data_return = device_pager_data_return,
+       .memory_object_data_initialize = device_pager_data_initialize,
+       .memory_object_data_unlock = device_pager_data_unlock,
+       .memory_object_synchronize = device_pager_synchronize,
+       .memory_object_map = device_pager_map,
+       .memory_object_last_unmap = device_pager_last_unmap,
+       .memory_object_data_reclaim = NULL,
+       .memory_object_backing_object = NULL,
+       .memory_object_pager_name = "device pager"
+};
+
+typedef uintptr_t device_port_t;

 
 

+/*
+ * The start of "struct device_pager" MUST match a "struct memory_object".
+ */

 typedef struct device_pager {
 typedef struct device_pager {

-       int             *pager;         /* pager workaround pointer     */
-       unsigned int    pager_ikot;     /* fake ip_kotype()             */
-       unsigned int    ref_count;      /* reference count              */
-       memory_object_control_t control_handle; /* mem object's cntrl handle */
+       /* mandatory generic header */
+       struct memory_object dev_pgr_hdr;
+
+       /* pager-specific data */
+       lck_mtx_t       lock;
+       struct os_refcnt ref_count;     /* reference count */

        device_port_t   device_handle;  /* device_handle */
        device_port_t   device_handle;  /* device_handle */

-       vm_size_t       size;
-       int             flags;
+       vm_size_t       size;
+       int             flags;
+       boolean_t       is_mapped;

 } *device_pager_t;
 
 } *device_pager_t;
 

+LCK_GRP_DECLARE(device_pager_lck_grp, "device_pager");

 
 

+ZONE_DECLARE(device_pager_zone, "device node pager structures",
+    sizeof(struct device_pager), ZC_NONE);

 
 

-void 
-device_pager_bootstrap(
-       void);
-
-
-memory_object_t
-device_pager_setup(
-       memory_object_t,
-       int,
-       vm_size_t,
-       int);
+#define device_pager_lock_init(pager) \
+       lck_mtx_init(&(pager)->lock, &device_pager_lck_grp, LCK_ATTR_NULL)
+#define device_pager_lock_destroy(pager) \
+       lck_mtx_destroy(&(pager)->lock, &device_pager_lck_grp)
+#define device_pager_lock(pager) lck_mtx_lock(&(pager)->lock)
+#define device_pager_unlock(pager) lck_mtx_unlock(&(pager)->lock)

 
 device_pager_t
 
 device_pager_t

-device_pager_lookup(
-       memory_object_t);
-
-kern_return_t
-device_pager_init(
-       memory_object_t, 
-       memory_object_control_t, 
-       vm_size_t);
-
-
-kern_return_t
-device_pager_data_request( 
-       memory_object_t, 
-       memory_object_offset_t,
-       vm_size_t, 
-       vm_prot_t);
-
-kern_return_t
-device_pager_data_return(
-       memory_object_t,
-       memory_object_offset_t,
-       vm_size_t,
-       boolean_t,
-       boolean_t);
-
-void
-device_pager_reference(
-       memory_object_t);
-
-void
-device_pager_deallocate(
-       memory_object_t);
-
-kern_return_t
-device_pager_data_initialize(
-       memory_object_t,
-       memory_object_offset_t,
-       vm_size_t);
-
-kern_return_t
-device_pager_data_unlock(
-       memory_object_t,
-       memory_object_offset_t,
-       vm_size_t,
-       vm_prot_t);
-
-kern_return_t
-device_pager_terminate(
-       memory_object_t);
-
-kern_return_t
-device_pager_synchronize(
-       memory_object_t,
-       memory_object_offset_t,
-       vm_offset_t,
-       vm_sync_t);
-
-kern_return_t
-device_pager_unmap(
+device_pager_lookup(            /* forward */

        memory_object_t);
 
 device_pager_t
        memory_object_t);
 
 device_pager_t

-device_object_create(void);
-
-zone_t device_pager_zone;
-
-
-#define        DEVICE_PAGER_NULL       ((device_pager_t) 0)
-
-
-#define        MAX_DNODE               10000
+device_object_create(void);     /* forward */

 
 

+#define DEVICE_PAGER_NULL       ((device_pager_t) 0)

 
 

+#define MAX_DNODE               10000

 
 
 
 

-
-/*
- *
- */
-void
-device_pager_bootstrap(void)
-{
-       register vm_size_t      size;
-
-       size = (vm_size_t) sizeof(struct device_pager);
-       device_pager_zone = zinit(size, (vm_size_t) MAX_DNODE*size,
-                               PAGE_SIZE, "device node pager structures");
-
-       return;
-}
-

 /*
  *
  */
 memory_object_t
 device_pager_setup(
 /*
  *
  */
 memory_object_t
 device_pager_setup(

-       memory_object_t device,
-       int             device_handle,
-       vm_size_t       size,
-       int             flags)
+       __unused memory_object_t device,
+       uintptr_t               device_handle,
+       vm_size_t       size,
+       int             flags)

 {
 {

-       device_pager_t  device_object;
+       device_pager_t  device_object;
+       memory_object_control_t control;
+       vm_object_t     object;

 
        device_object = device_object_create();
 
        device_object = device_object_create();

-       if (device_object == DEVICE_PAGER_NULL)
+       if (device_object == DEVICE_PAGER_NULL) {

                panic("device_pager_setup: device_object_create() failed");
                panic("device_pager_setup: device_object_create() failed");

+       }

 
        device_object->device_handle = device_handle;
        device_object->size = size;
 
        device_object->device_handle = device_handle;
        device_object->size = size;

-       device_object->flags = 0;
-       if(flags & DEVICE_PAGER_CONTIGUOUS) {
-               device_object->flags |= DEVICE_PAGER_CONTIGUOUS;
-       }
-       if(flags & DEVICE_PAGER_NOPHYSCACHE) {
-               device_object->flags |= DEVICE_PAGER_NOPHYSCACHE;
+       device_object->flags = flags;
+
+       memory_object_create_named((memory_object_t) device_object,
+           size,
+           &control);
+       object = memory_object_control_to_vm_object(control);
+
+       memory_object_mark_trusted(control);
+
+       assert(object != VM_OBJECT_NULL);
+       vm_object_lock(object);
+       object->true_share = TRUE;
+       if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) {
+               object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;

        }
        }

+       vm_object_unlock(object);

 
 

-       return((memory_object_t)device_object);
+       return (memory_object_t)device_object;

 }
 
 /*
 }
 
 /*


@@ -206,41 +170,47 @@ device_pager_setup(
  */
 kern_return_t
 device_pager_populate_object(
  */
 kern_return_t
 device_pager_populate_object(

-       memory_object_t         device,
-       memory_object_offset_t  offset,
-       vm_offset_t             phys_addr,
-       vm_size_t               size)
+       memory_object_t         device,
+       memory_object_offset_t  offset,
+       ppnum_t                 page_num,
+       vm_size_t               size)

 {
 {

-       device_pager_t  device_object;
-       vm_object_t     vm_object;
-       kern_return_t   kr;
-       upl_t           upl;
-       ipc_port_t      previous;
+       device_pager_t  device_object;
+       vm_object_t     vm_object;
+       kern_return_t   kr;
+       upl_t           upl;

 
        device_object = device_pager_lookup(device);
 
        device_object = device_pager_lookup(device);

-       if(device_object == DEVICE_PAGER_NULL)
+       if (device_object == DEVICE_PAGER_NULL) {

                return KERN_FAILURE;
                return KERN_FAILURE;

+       }

 
        vm_object = (vm_object_t)memory_object_control_to_vm_object(
 
        vm_object = (vm_object_t)memory_object_control_to_vm_object(

-                                       device_object->control_handle);
-       if(vm_object == NULL) 
+               device_object->dev_pgr_hdr.mo_control);
+       if (vm_object == NULL) {

                return KERN_FAILURE;
                return KERN_FAILURE;

+       }

 
        kr =  vm_object_populate_with_private(
 
        kr =  vm_object_populate_with_private(

-                               vm_object, offset, phys_addr, size);
-       if(kr != KERN_SUCCESS)
+               vm_object, offset, page_num, size);
+       if (kr != KERN_SUCCESS) {

                return kr;
                return kr;

+       }

 
 

-       if(!vm_object->phys_contiguous) {
-               int null_size = 0;
-               kr = vm_object_upl_request(vm_object,
-                       (vm_object_offset_t)offset, size, &upl,  NULL,
-                       &null_size, (UPL_NO_SYNC | UPL_CLEAN_IN_PLACE)); 
-
-               if(kr != KERN_SUCCESS)
+       if (!vm_object->phys_contiguous) {
+               unsigned int null_size = 0;
+               assert((upl_size_t) size == size);
+               kr = vm_object_upl_request(vm_object,
+                   (vm_object_offset_t)offset,
+                   (upl_size_t) size, &upl, NULL,
+                   &null_size,
+                   (UPL_NO_SYNC | UPL_CLEAN_IN_PLACE),
+                   VM_KERN_MEMORY_NONE);
+               if (kr != KERN_SUCCESS) {

                        panic("device_pager_populate_object: list_req failed");
                        panic("device_pager_populate_object: list_req failed");

+               }

 
 

-               upl_commit(upl, NULL);
+               upl_commit(upl, NULL, 0);

                upl_deallocate(upl);
        }
 
                upl_deallocate(upl);
        }
 


@@ -253,37 +223,40 @@ device_pager_populate_object(
  */
 device_pager_t
 device_pager_lookup(
  */
 device_pager_t
 device_pager_lookup(

-       memory_object_t name)
+       memory_object_t mem_obj)

 {
 {

-       device_pager_t  device_object;
+       device_pager_t  device_object;

 
 

-       device_object = (device_pager_t)name;
-       assert(device_object->pager == &device_pager_workaround);
-       return (device_object);
+       assert(mem_obj->mo_pager_ops == &device_pager_ops);
+       device_object = (device_pager_t)mem_obj;
+       assert(os_ref_get_count(&device_object->ref_count) > 0);
+       return device_object;

 }
 
 /*
  *
  */
 kern_return_t
 }
 
 /*
  *
  */
 kern_return_t

-device_pager_init(memory_object_t mem_obj, 
-               memory_object_control_t control, 
-               vm_size_t pg_size)
+device_pager_init(
+       memory_object_t mem_obj,
+       memory_object_control_t control,
+       __unused memory_object_cluster_size_t pg_size)

 {
        device_pager_t   device_object;
        kern_return_t   kr;
        memory_object_attr_info_data_t  attributes;
 
 {
        device_pager_t   device_object;
        kern_return_t   kr;
        memory_object_attr_info_data_t  attributes;
 

-       vm_object_t     vm_object;
+       vm_object_t     vm_object;

 
 
 
 

-       if (control == MEMORY_OBJECT_CONTROL_NULL)
+       if (control == MEMORY_OBJECT_CONTROL_NULL) {

                return KERN_INVALID_ARGUMENT;
                return KERN_INVALID_ARGUMENT;

+       }

 
        device_object = device_pager_lookup(mem_obj);
 
        memory_object_control_reference(control);
 
        device_object = device_pager_lookup(mem_obj);
 
        memory_object_control_reference(control);

-       device_object->control_handle = control;
+       device_object->dev_pgr_hdr.mo_control = control;

 
 
 /* The following settings should be done through an expanded change */
 
 
 /* The following settings should be done through an expanded change */


@@ -292,10 +265,14 @@ device_pager_init(memory_object_t mem_obj,
        vm_object = (vm_object_t)memory_object_control_to_vm_object(control);
        vm_object_lock(vm_object);
        vm_object->private = TRUE;
        vm_object = (vm_object_t)memory_object_control_to_vm_object(control);
        vm_object_lock(vm_object);
        vm_object->private = TRUE;

-       if(device_object->flags & DEVICE_PAGER_CONTIGUOUS)
+       if (device_object->flags & DEVICE_PAGER_CONTIGUOUS) {

                vm_object->phys_contiguous = TRUE;
                vm_object->phys_contiguous = TRUE;

-       if(device_object->flags & DEVICE_PAGER_NOPHYSCACHE)
+       }
+       if (device_object->flags & DEVICE_PAGER_NOPHYSCACHE) {

                vm_object->nophyscache = TRUE;
                vm_object->nophyscache = TRUE;

+       }
+
+       vm_object->wimg_bits = device_object->flags & VM_WIMG_MASK;

        vm_object_unlock(vm_object);
 
 
        vm_object_unlock(vm_object);
 
 


@@ -306,56 +283,67 @@ device_pager_init(memory_object_t mem_obj,
        attributes.temporary = TRUE;
 
        kr = memory_object_change_attributes(
        attributes.temporary = TRUE;
 
        kr = memory_object_change_attributes(

-                                       control,
-                                       MEMORY_OBJECT_ATTRIBUTE_INFO,
-                                       (memory_object_info_t) &attributes,
-                                       MEMORY_OBJECT_ATTR_INFO_COUNT);
-       if (kr != KERN_SUCCESS)
+               control,
+               MEMORY_OBJECT_ATTRIBUTE_INFO,
+               (memory_object_info_t) &attributes,
+               MEMORY_OBJECT_ATTR_INFO_COUNT);
+       if (kr != KERN_SUCCESS) {

                panic("device_pager_init: memory_object_change_attributes() failed");
                panic("device_pager_init: memory_object_change_attributes() failed");

+       }

 
 

-       return(KERN_SUCCESS);
+       return KERN_SUCCESS;

 }
 
 /*
  *
  */
 }
 
 /*
  *
  */

+/*ARGSUSED6*/

 kern_return_t
 device_pager_data_return(
 kern_return_t
 device_pager_data_return(

-        memory_object_t                mem_obj,
-        memory_object_offset_t offset,
-        vm_size_t              data_cnt,
-        boolean_t              dirty,
-        boolean_t              kernel_copy)  
+       memory_object_t                 mem_obj,
+       memory_object_offset_t          offset,
+       memory_object_cluster_size_t                    data_cnt,
+       __unused memory_object_offset_t *resid_offset,
+       __unused int                    *io_error,
+       __unused boolean_t              dirty,
+       __unused boolean_t              kernel_copy,
+       __unused int                    upl_flags)

 {
 {

-       device_pager_t  device_object;
+       device_pager_t  device_object;

 
        device_object = device_pager_lookup(mem_obj);
 
        device_object = device_pager_lookup(mem_obj);

-       if (device_object == DEVICE_PAGER_NULL)
+       if (device_object == DEVICE_PAGER_NULL) {

                panic("device_pager_data_return: lookup failed");
                panic("device_pager_data_return: lookup failed");

+       }

 
 

-       return device_data_action(device_object->device_handle, device_object,
-                       VM_PROT_READ | VM_PROT_WRITE, offset, data_cnt);
+       __IGNORE_WCASTALIGN(return device_data_action(device_object->device_handle,
+           (ipc_port_t) device_object,
+           VM_PROT_READ | VM_PROT_WRITE,
+           offset, data_cnt));

 }
 
 /*
  *
  */
 }
 
 /*
  *
  */

-kern_return_t  
+kern_return_t

 device_pager_data_request(
 device_pager_data_request(

-       memory_object_t         mem_obj,
-       memory_object_offset_t  offset,
-       vm_size_t               length,
-       vm_prot_t               protection_required)
+       memory_object_t         mem_obj,
+       memory_object_offset_t  offset,
+       memory_object_cluster_size_t            length,
+       __unused vm_prot_t      protection_required,
+       __unused memory_object_fault_info_t     fault_info)

 {
 {

-       device_pager_t  device_object;
+       device_pager_t  device_object;

 
        device_object = device_pager_lookup(mem_obj);
 
 
        device_object = device_pager_lookup(mem_obj);
 

-       if (device_object == DEVICE_PAGER_NULL)
+       if (device_object == DEVICE_PAGER_NULL) {

                panic("device_pager_data_request: lookup failed");
                panic("device_pager_data_request: lookup failed");

+       }

 
 

-       device_data_action(device_object->device_handle, device_object,
-                                       VM_PROT_READ, offset, length);
+       __IGNORE_WCASTALIGN(device_data_action(device_object->device_handle,
+           (ipc_port_t) device_object,
+           VM_PROT_READ, offset, length));

        return KERN_SUCCESS;
 }
 
        return KERN_SUCCESS;
 }
 


@@ -364,14 +352,15 @@ device_pager_data_request(
  */
 void
 device_pager_reference(
  */
 void
 device_pager_reference(

-       memory_object_t         mem_obj)
-{      
-       device_pager_t          device_object;
-       unsigned int            prev_ref_count;
+       memory_object_t         mem_obj)
+{
+       device_pager_t          device_object;

 
        device_object = device_pager_lookup(mem_obj);
 
        device_object = device_pager_lookup(mem_obj);

-       prev_ref_count = OSIncrementAtomic((UInt32 *)&device_object->ref_count);
-       assert(prev_ref_count > 0);
+       os_ref_retain(&device_object->ref_count);
+       DTRACE_VM2(device_pager_reference,
+           device_pager_t, device_object,
+           unsigned int, os_ref_get_count(&device_object->ref_count));

 }
 
 /*
 }
 
 /*


@@ -379,42 +368,74 @@ device_pager_reference(
  */
 void
 device_pager_deallocate(
  */
 void
 device_pager_deallocate(

-       memory_object_t         mem_obj)
+       memory_object_t         mem_obj)

 {
 {

-       device_pager_t  device_object;
+       device_pager_t          device_object;
+       memory_object_control_t device_control;

 
        device_object = device_pager_lookup(mem_obj);
 
 
        device_object = device_pager_lookup(mem_obj);
 

-       if (OSDecrementAtomic((UInt32 *)&device_object->ref_count) == 1) {
+       DTRACE_VM2(device_pager_deallocate,
+           device_pager_t, device_object,
+           unsigned int, os_ref_get_count(&device_object->ref_count));
+
+       os_ref_count_t ref_count = os_ref_release(&device_object->ref_count);
+
+       if (ref_count == 1) {
+               /*
+                * The last reference is our "named" reference.
+                * Close the device and "destroy" the VM object.
+                */
+
+               DTRACE_VM2(device_pager_destroy,
+                   device_pager_t, device_object,
+                   unsigned int, os_ref_get_count(&device_object->ref_count));
+
+               assert(device_object->is_mapped == FALSE);

                if (device_object->device_handle != (device_port_t) NULL) {
                        device_close(device_object->device_handle);
                if (device_object->device_handle != (device_port_t) NULL) {
                        device_close(device_object->device_handle);

+                       device_object->device_handle = (device_port_t) NULL;

                }
                }

-               zfree(device_pager_zone, (vm_offset_t) device_object);
+               device_control = device_object->dev_pgr_hdr.mo_control;
+               memory_object_destroy(device_control, 0);
+       } else if (ref_count == 0) {
+               /*
+                * No more references: free the pager.
+                */
+               DTRACE_VM2(device_pager_free,
+                   device_pager_t, device_object,
+                   unsigned int, os_ref_get_count(&device_object->ref_count));
+
+               device_pager_lock_destroy(device_object);
+
+               zfree(device_pager_zone, device_object);

        }
        return;
 }
 
 kern_return_t
 device_pager_data_initialize(
        }
        return;
 }
 
 kern_return_t
 device_pager_data_initialize(

-        memory_object_t                mem_obj,
-        memory_object_offset_t offset,
-        vm_size_t              data_cnt)
+       __unused memory_object_t                mem_obj,
+       __unused memory_object_offset_t offset,
+       __unused memory_object_cluster_size_t           data_cnt)

 {
 {

+       panic("device_pager_data_initialize");

        return KERN_FAILURE;
 }
 
 kern_return_t
 device_pager_data_unlock(
        return KERN_FAILURE;
 }
 
 kern_return_t
 device_pager_data_unlock(

-       memory_object_t         mem_obj,
-       memory_object_offset_t  offset,
-       vm_size_t               size,
-       vm_prot_t               desired_access)
+       __unused memory_object_t                mem_obj,
+       __unused memory_object_offset_t offset,
+       __unused memory_object_size_t           size,
+       __unused vm_prot_t              desired_access)

 {
        return KERN_FAILURE;
 }
 
 {
        return KERN_FAILURE;
 }
 

+kern_return_t

 device_pager_terminate(
 device_pager_terminate(

-       memory_object_t mem_obj)
+       __unused memory_object_t        mem_obj)

 {
        return KERN_SUCCESS;
 }
 {
        return KERN_SUCCESS;
 }


@@ -426,28 +447,66 @@ device_pager_terminate(
  */
 kern_return_t
 device_pager_synchronize(
  */
 kern_return_t
 device_pager_synchronize(

-       memory_object_t         mem_obj,
-       memory_object_offset_t  offset,
-       vm_offset_t             length,
-       vm_sync_t               sync_flags)
+       __unused memory_object_t        mem_obj,
+       __unused memory_object_offset_t offset,
+       __unused memory_object_size_t   length,
+       __unused vm_sync_t              sync_flags)
+{
+       panic("device_pager_synchronize: memory_object_synchronize no longer supported\n");
+       return KERN_FAILURE;
+}
+
+/*
+ *
+ */
+kern_return_t
+device_pager_map(
+       memory_object_t mem_obj,
+       __unused vm_prot_t              prot)

 {
 {

-       device_pager_t  device_object;
+       device_pager_t          device_object;

 
        device_object = device_pager_lookup(mem_obj);
 
 
        device_object = device_pager_lookup(mem_obj);
 

-       memory_object_synchronize_completed(
-                       device_object->control_handle, offset, length);
+       device_pager_lock(device_object);
+       assert(os_ref_get_count(&device_object->ref_count) > 0);
+       if (device_object->is_mapped == FALSE) {
+               /*
+                * First mapping of this pager: take an extra reference
+                * that will remain until all the mappings of this pager
+                * are removed.
+                */
+               device_object->is_mapped = TRUE;
+               device_pager_reference(mem_obj);
+       }
+       device_pager_unlock(device_object);

 
        return KERN_SUCCESS;
 }
 
 
        return KERN_SUCCESS;
 }
 

-/*
- *
- */

 kern_return_t
 kern_return_t

-device_pager_unmap(
-       memory_object_t mem_obj)
+device_pager_last_unmap(
+       memory_object_t mem_obj)

 {
 {

+       device_pager_t  device_object;
+       boolean_t       drop_ref;
+
+       device_object = device_pager_lookup(mem_obj);
+
+       device_pager_lock(device_object);
+       assert(os_ref_get_count(&device_object->ref_count) > 0);
+       if (device_object->is_mapped) {
+               device_object->is_mapped = FALSE;
+               drop_ref = TRUE;
+       } else {
+               drop_ref = FALSE;
+       }
+       device_pager_unlock(device_object);
+
+       if (drop_ref) {
+               device_pager_deallocate(mem_obj);
+       }
+

        return KERN_SUCCESS;
 }
 
        return KERN_SUCCESS;
 }
 


@@ -457,19 +516,37 @@ device_pager_unmap(
  *
  */
 device_pager_t
  *
  */
 device_pager_t

-device_object_create()
+device_object_create(void)

 {
 {

-       register device_pager_t  device_object;
+       device_pager_t  device_object;

 
        device_object = (struct device_pager *) zalloc(device_pager_zone);
 
        device_object = (struct device_pager *) zalloc(device_pager_zone);

-       if (device_object == DEVICE_PAGER_NULL)
-               return(DEVICE_PAGER_NULL);
-       device_object->pager = &device_pager_workaround;
-       device_object->pager_ikot = IKOT_MEMORY_OBJECT;
-       device_object->ref_count = 1;
-       device_object->control_handle = MEMORY_OBJECT_CONTROL_NULL;
+       if (device_object == DEVICE_PAGER_NULL) {
+               return DEVICE_PAGER_NULL;
+       }
+
+       bzero(device_object, sizeof(*device_object));
+
+       device_object->dev_pgr_hdr.mo_ikot = IKOT_MEMORY_OBJECT;
+       device_object->dev_pgr_hdr.mo_pager_ops = &device_pager_ops;
+       device_object->dev_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL;

 
 

+       device_pager_lock_init(device_object);
+       os_ref_init(&device_object->ref_count, NULL);
+       device_object->is_mapped = FALSE;

 
 

-       return(device_object);
+       DTRACE_VM2(device_pager_create,
+           device_pager_t, device_object,
+           unsigned int, os_ref_get_count(&device_object->ref_count));
+
+       return device_object;

 }
 
 }
 

+boolean_t
+is_device_pager_ops(const struct memory_object_pager_ops *pager_ops)
+{
+       if (pager_ops == &device_pager_ops) {
+               return TRUE;
+       }
+       return FALSE;
+}