osfmk/vm/device_vm.c

   1 /*
   2  * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <sys/errno.h>
  30
  31 #include <mach/mach_types.h>
  32 #include <mach/kern_return.h>
  33 #include <mach/memory_object_control.h>
  34 #include <mach/memory_object_types.h>
  35 #include <mach/port.h>
  36 #include <mach/policy.h>
  37 #include <mach/upl.h>
  38 #include <kern/kern_types.h>
  39 #include <kern/ipc_kobject.h>
  40 #include <kern/host.h>
  41 #include <kern/thread.h>
  42 #include <ipc/ipc_port.h>
  43 #include <ipc/ipc_space.h>
  44 #include <device/device_port.h>
  45 #include <vm/memory_object.h>
  46 #include <vm/vm_pageout.h>
  47 #include <vm/vm_map.h>
  48 #include <vm/vm_kern.h>
  49 #include <vm/vm_pageout.h>
  50 #include <vm/vm_protos.h>
  51
  52
  53 /* Device VM COMPONENT INTERFACES */
  54
  55
  56 /*
  57  * Device PAGER
  58  */
  59
  60
  61 /* until component support available */
  62
  63
  64
  65 /* until component support available */
  66 const struct memory_object_pager_ops device_pager_ops = {
  67         device_pager_reference,
  68         device_pager_deallocate,
  69         device_pager_init,
  70         device_pager_terminate,
  71         device_pager_data_request,
  72         device_pager_data_return,
  73         device_pager_data_initialize,
  74         device_pager_data_unlock,
  75         device_pager_synchronize,
  76         device_pager_map,
  77         device_pager_last_unmap,
  78         NULL, /* data_reclaim */
  79         "device pager"
  80 };
  81
  82 typedef uintptr_t device_port_t;
  83
  84 /*
  85  * The start of "struct device_pager" MUST match a "struct memory_object".
  86  */
  87 typedef struct device_pager {
  88         struct ipc_object_header        pager_header;   /* fake ip_kotype()     */
  89         memory_object_pager_ops_t pager_ops; /* == &device_pager_ops    */
  90         unsigned int    ref_count;      /* reference count              */
  91         memory_object_control_t control_handle; /* mem object's cntrl handle */
  92         device_port_t   device_handle;  /* device_handle */
  93         vm_size_t       size;
  94         int             flags;
  95 } *device_pager_t;
  96
  97 #define pager_ikot pager_header.io_bits
  98
  99
 100 device_pager_t
 101 device_pager_lookup(            /* forward */
 102         memory_object_t);
 103
 104 device_pager_t
 105 device_object_create(void);     /* forward */
 106
 107 zone_t  device_pager_zone;
 108
 109
 110 #define DEVICE_PAGER_NULL       ((device_pager_t) 0)
 111
 112
 113 #define MAX_DNODE               10000
 114
 115
 116
 117
 118
 119 /*
 120  *
 121  */
 122 void
 123 device_pager_bootstrap(void)
 124 {
 125         vm_size_t      size;
 126
 127         size = (vm_size_t) sizeof(struct device_pager);
 128         device_pager_zone = zinit(size, (vm_size_t) MAX_DNODE*size,
 129                                 PAGE_SIZE, "device node pager structures");
 130         zone_change(device_pager_zone, Z_CALLERACCT, FALSE);
 131         return;
 132 }
 133
 134 /*
 135  *
 136  */
 137 memory_object_t
 138 device_pager_setup(
 139         __unused memory_object_t device,
 140         uintptr_t               device_handle,
 141         vm_size_t       size,
 142         int             flags)
 143 {
 144         device_pager_t  device_object;
 145
 146         device_object = device_object_create();
 147         if (device_object == DEVICE_PAGER_NULL)
 148                 panic("device_pager_setup: device_object_create() failed");
 149
 150         device_object->device_handle = device_handle;
 151         device_object->size = size;
 152         device_object->flags = flags;
 153
 154         return((memory_object_t)device_object);
 155 }
 156
 157 /*
 158  *
 159  */
 160 kern_return_t
 161 device_pager_populate_object(
 162         memory_object_t         device,
 163         memory_object_offset_t  offset,
 164         ppnum_t                 page_num,
 165         vm_size_t               size)
 166 {
 167         device_pager_t  device_object;
 168         vm_object_t     vm_object;
 169         kern_return_t   kr;
 170         upl_t           upl;
 171
 172         device_object = device_pager_lookup(device);
 173         if(device_object == DEVICE_PAGER_NULL)
 174                 return KERN_FAILURE;
 175
 176         vm_object = (vm_object_t)memory_object_control_to_vm_object(
 177                                         device_object->control_handle);
 178         if(vm_object == NULL)
 179                 return KERN_FAILURE;
 180
 181         kr =  vm_object_populate_with_private(
 182                                 vm_object, offset, page_num, size);
 183         if(kr != KERN_SUCCESS)
 184                 return kr;
 185
 186         if(!vm_object->phys_contiguous) {
 187                 unsigned int null_size = 0;
 188                 assert((upl_size_t) size == size);
 189                 kr = vm_object_upl_request(vm_object,
 190                                            (vm_object_offset_t)offset,
 191                                            (upl_size_t) size, &upl,  NULL,
 192                                            &null_size,
 193                                            (UPL_NO_SYNC | UPL_CLEAN_IN_PLACE));
 194                 if(kr != KERN_SUCCESS)
 195                         panic("device_pager_populate_object: list_req failed");
 196
 197                 upl_commit(upl, NULL, 0);
 198                 upl_deallocate(upl);
 199         }
 200
 201
 202         return kr;
 203 }
 204
 205 /*
 206  *
 207  */
 208 device_pager_t
 209 device_pager_lookup(
 210         memory_object_t name)
 211 {
 212         device_pager_t  device_object;
 213
 214         device_object = (device_pager_t)name;
 215         assert(device_object->pager_ops == &device_pager_ops);
 216         return (device_object);
 217 }
 218
 219 /*
 220  *
 221  */
 222 kern_return_t
 223 device_pager_init(
 224         memory_object_t mem_obj,
 225         memory_object_control_t control,
 226         __unused memory_object_cluster_size_t pg_size)
 227 {
 228         device_pager_t   device_object;
 229         kern_return_t   kr;
 230         memory_object_attr_info_data_t  attributes;
 231
 232         vm_object_t     vm_object;
 233
 234
 235         if (control == MEMORY_OBJECT_CONTROL_NULL)
 236                 return KERN_INVALID_ARGUMENT;
 237
 238         device_object = device_pager_lookup(mem_obj);
 239
 240         memory_object_control_reference(control);
 241         device_object->control_handle = control;
 242
 243
 244 /* The following settings should be done through an expanded change */
 245 /* attributes call */
 246
 247         vm_object = (vm_object_t)memory_object_control_to_vm_object(control);
 248         vm_object_lock(vm_object);
 249         vm_object->private = TRUE;
 250         if(device_object->flags & DEVICE_PAGER_CONTIGUOUS)
 251                 vm_object->phys_contiguous = TRUE;
 252         if(device_object->flags & DEVICE_PAGER_NOPHYSCACHE)
 253                 vm_object->nophyscache = TRUE;
 254
 255         vm_object->wimg_bits = device_object->flags & VM_WIMG_MASK;
 256         vm_object_unlock(vm_object);
 257
 258
 259         attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY;
 260         /* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/
 261         attributes.cluster_size = (1 << (PAGE_SHIFT));
 262         attributes.may_cache_object = FALSE;
 263         attributes.temporary = TRUE;
 264
 265         kr = memory_object_change_attributes(
 266                                         control,
 267                                         MEMORY_OBJECT_ATTRIBUTE_INFO,
 268                                         (memory_object_info_t) &attributes,
 269                                         MEMORY_OBJECT_ATTR_INFO_COUNT);
 270         if (kr != KERN_SUCCESS)
 271                 panic("device_pager_init: memory_object_change_attributes() failed");
 272
 273         return(KERN_SUCCESS);
 274 }
 275
 276 /*
 277  *
 278  */
 279 /*ARGSUSED6*/
 280 kern_return_t
 281 device_pager_data_return(
 282         memory_object_t                 mem_obj,
 283         memory_object_offset_t          offset,
 284         memory_object_cluster_size_t                    data_cnt,
 285         __unused memory_object_offset_t *resid_offset,
 286         __unused int                    *io_error,
 287         __unused boolean_t              dirty,
 288         __unused boolean_t              kernel_copy,
 289         __unused int                    upl_flags)
 290 {
 291         device_pager_t  device_object;
 292
 293         device_object = device_pager_lookup(mem_obj);
 294         if (device_object == DEVICE_PAGER_NULL)
 295                 panic("device_pager_data_return: lookup failed");
 296
 297         __IGNORE_WCASTALIGN(return device_data_action(device_object->device_handle,
 298                           (ipc_port_t) device_object,
 299                           VM_PROT_READ | VM_PROT_WRITE,
 300                           offset, data_cnt));
 301 }
 302
 303 /*
 304  *
 305  */
 306 kern_return_t
 307 device_pager_data_request(
 308         memory_object_t         mem_obj,
 309         memory_object_offset_t  offset,
 310         memory_object_cluster_size_t            length,
 311         __unused vm_prot_t      protection_required,
 312         __unused memory_object_fault_info_t     fault_info)
 313 {
 314         device_pager_t  device_object;
 315
 316         device_object = device_pager_lookup(mem_obj);
 317
 318         if (device_object == DEVICE_PAGER_NULL)
 319                 panic("device_pager_data_request: lookup failed");
 320
 321         __IGNORE_WCASTALIGN(device_data_action(device_object->device_handle,
 322                            (ipc_port_t) device_object,
 323                            VM_PROT_READ, offset, length));
 324         return KERN_SUCCESS;
 325 }
 326
 327 /*
 328  *
 329  */
 330 void
 331 device_pager_reference(
 332         memory_object_t         mem_obj)
 333 {
 334         device_pager_t          device_object;
 335         unsigned int            new_ref_count;
 336
 337         device_object = device_pager_lookup(mem_obj);
 338         new_ref_count = hw_atomic_add(&device_object->ref_count, 1);
 339         assert(new_ref_count > 1);
 340 }
 341
 342 /*
 343  *
 344  */
 345 void
 346 device_pager_deallocate(
 347         memory_object_t         mem_obj)
 348 {
 349         device_pager_t          device_object;
 350         memory_object_control_t device_control;
 351
 352         device_object = device_pager_lookup(mem_obj);
 353
 354         if (hw_atomic_sub(&device_object->ref_count, 1) == 0) {
 355                 if (device_object->device_handle != (device_port_t) NULL) {
 356                         device_close(device_object->device_handle);
 357                         device_object->device_handle = (device_port_t) NULL;
 358                 }
 359                 device_control = device_object->control_handle;
 360                 if (device_control != MEMORY_OBJECT_CONTROL_NULL) {
 361                         /*
 362                          * The VM object should already have been disconnected
 363                          * from the pager at this point.
 364                          * We still have to release the "memory object control"
 365                          * handle.
 366                          */
 367                         assert(device_control->moc_object == VM_OBJECT_NULL);
 368                         memory_object_control_deallocate(device_control);
 369                         device_object->control_handle =
 370                                 MEMORY_OBJECT_CONTROL_NULL;
 371                 }
 372
 373                 zfree(device_pager_zone, device_object);
 374         }
 375         return;
 376 }
 377
 378 kern_return_t
 379 device_pager_data_initialize(
 380         __unused memory_object_t                mem_obj,
 381         __unused memory_object_offset_t offset,
 382         __unused memory_object_cluster_size_t           data_cnt)
 383 {
 384         panic("device_pager_data_initialize");
 385         return KERN_FAILURE;
 386 }
 387
 388 kern_return_t
 389 device_pager_data_unlock(
 390         __unused memory_object_t                mem_obj,
 391         __unused memory_object_offset_t offset,
 392         __unused memory_object_size_t           size,
 393         __unused vm_prot_t              desired_access)
 394 {
 395         return KERN_FAILURE;
 396 }
 397
 398 kern_return_t
 399 device_pager_terminate(
 400         __unused memory_object_t        mem_obj)
 401 {
 402         return KERN_SUCCESS;
 403 }
 404
 405
 406
 407 /*
 408  *
 409  */
 410 kern_return_t
 411 device_pager_synchronize(
 412         memory_object_t         mem_obj,
 413         memory_object_offset_t  offset,
 414         memory_object_size_t            length,
 415         __unused vm_sync_t              sync_flags)
 416 {
 417         device_pager_t  device_object;
 418
 419         device_object = device_pager_lookup(mem_obj);
 420
 421         memory_object_synchronize_completed(
 422                         device_object->control_handle, offset, length);
 423
 424         return KERN_SUCCESS;
 425 }
 426
 427 /*
 428  *
 429  */
 430 kern_return_t
 431 device_pager_map(
 432         __unused memory_object_t        mem_obj,
 433         __unused vm_prot_t              prot)
 434 {
 435         return KERN_SUCCESS;
 436 }
 437
 438 kern_return_t
 439 device_pager_last_unmap(
 440         __unused memory_object_t        mem_obj)
 441 {
 442         return KERN_SUCCESS;
 443 }
 444
 445
 446
 447 /*
 448  *
 449  */
 450 device_pager_t
 451 device_object_create(void)
 452 {
 453         device_pager_t  device_object;
 454
 455         device_object = (struct device_pager *) zalloc(device_pager_zone);
 456         if (device_object == DEVICE_PAGER_NULL)
 457                 return(DEVICE_PAGER_NULL);
 458         device_object->pager_ops = &device_pager_ops;
 459         device_object->pager_ikot = IKOT_MEMORY_OBJECT;
 460         device_object->ref_count = 1;
 461         device_object->control_handle = MEMORY_OBJECT_CONTROL_NULL;
 462
 463
 464         return(device_object);
 465 }
 466
 467 boolean_t
 468 is_device_pager_ops(const struct memory_object_pager_ops *pager_ops)
 469 {
 470         if (pager_ops == &device_pager_ops) {
 471                 return TRUE;
 472         }
 473         return FALSE;
 474 }