osfmk/vm/bsd_vm.c

   1 /*
   2  * Copyright (c) 2000-2001 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * Copyright (c) 1999-2003 Apple Computer, Inc.  All Rights Reserved.
   7  *
   8  * This file contains Original Code and/or Modifications of Original Code
   9  * as defined in and that are subject to the Apple Public Source License
  10  * Version 2.0 (the 'License'). You may not use this file except in
  11  * compliance with the License. Please obtain a copy of the License at
  12  * http://www.opensource.apple.com/apsl/ and read it before using this
  13  * file.
  14  *
  15  * The Original Code and all software distributed under the License are
  16  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  17  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  18  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  20  * Please see the License for the specific language governing rights and
  21  * limitations under the License.
  22  *
  23  * @APPLE_LICENSE_HEADER_END@
  24  */
  25
  26 #include <sys/errno.h>
  27 #include <kern/host.h>
  28 #include <mach/mach_types.h>
  29 #include <vm/vm_map.h>
  30 #include <vm/vm_kern.h>
  31 #include <vm/vm_pageout.h>
  32 #include <mach/kern_return.h>
  33 #include <mach/memory_object_types.h>
  34 #include <mach/port.h>
  35 #include <mach/policy.h>
  36 #include <ipc/ipc_port.h>
  37 #include <ipc/ipc_space.h>
  38 #include <kern/thread.h>
  39 #include <vm/memory_object.h>
  40 #include <vm/vm_pageout.h>
  41
  42 #include <default_pager/default_pager_types.h>
  43
  44 /* BSD VM COMPONENT INTERFACES */
  45 int
  46 get_map_nentries(
  47         vm_map_t);
  48
  49 vm_offset_t
  50 get_map_start(
  51         vm_map_t);
  52
  53 vm_offset_t
  54 get_map_end(
  55         vm_map_t);
  56
  57 /*
  58  *
  59  */
  60 int
  61 get_map_nentries(
  62         vm_map_t map)
  63 {
  64         return(map->hdr.nentries);
  65 }
  66
  67 /*
  68  *
  69  */
  70 vm_offset_t
  71 get_map_start(
  72         vm_map_t map)
  73 {
  74         return(vm_map_first_entry(map)->vme_start);
  75 }
  76
  77 /*
  78  *
  79  */
  80 vm_offset_t
  81 get_map_end(
  82         vm_map_t map)
  83 {
  84         return(vm_map_last_entry(map)->vme_end);
  85 }
  86
  87 /*
  88  * BSD VNODE PAGER
  89  */
  90
  91 /* until component support available */
  92 int     vnode_pager_workaround;
  93
  94 typedef int vnode_port_t;
  95
  96 typedef struct vnode_pager {
  97         int                     *pager;         /* pager workaround pointer  */
  98         unsigned int            pager_ikot;     /* JMM: fake ip_kotype()     */
  99         unsigned int            ref_count;      /* reference count           */
 100         memory_object_control_t control_handle; /* mem object control handle */
 101         vnode_port_t            vnode_handle;   /* vnode handle              */
 102 } *vnode_pager_t;
 103
 104
 105 ipc_port_t
 106 trigger_name_to_port(
 107         mach_port_t);
 108
 109 void
 110 vnode_pager_bootstrap(
 111         void);
 112
 113 void
 114 vnode_pager_alloc_map(
 115         void);
 116
 117 memory_object_t
 118 vnode_pager_setup(
 119         vnode_port_t,
 120         memory_object_t);
 121
 122
 123 kern_return_t
 124 vnode_pager_init(
 125         memory_object_t,
 126         memory_object_control_t,
 127         vm_size_t);
 128
 129 kern_return_t
 130 vnode_pager_get_object_size(
 131         memory_object_t,
 132         memory_object_offset_t *);
 133
 134 kern_return_t
 135 vnode_pager_data_request(
 136         memory_object_t,
 137         memory_object_offset_t,
 138         vm_size_t,
 139         vm_prot_t);
 140
 141 kern_return_t
 142 vnode_pager_data_return(
 143         memory_object_t,
 144         memory_object_offset_t,
 145         vm_size_t,
 146         boolean_t,
 147         boolean_t);
 148
 149 kern_return_t
 150 vnode_pager_data_initialize(
 151         memory_object_t,
 152         memory_object_offset_t,
 153         vm_size_t);
 154
 155 void
 156 vnode_pager_deallocate(
 157         memory_object_t);
 158
 159 kern_return_t
 160 vnode_pager_terminate(
 161         memory_object_t);
 162
 163 kern_return_t
 164 vnode_pager_cluster_read(
 165         vnode_pager_t,
 166         vm_object_offset_t,
 167         vm_size_t);
 168
 169 void
 170 vnode_pager_cluster_write(
 171         vnode_pager_t,
 172         vm_object_offset_t,
 173         vm_size_t);
 174
 175
 176 int
 177 vnode_pagein(
 178         vnode_port_t,
 179         upl_t,
 180         vm_offset_t,
 181         vm_object_offset_t,
 182         int,
 183         int,
 184         int *);
 185 int
 186 vnode_pageout(
 187         vnode_port_t,
 188         upl_t,
 189         vm_offset_t,
 190         vm_object_offset_t,
 191         int,
 192         int,
 193         int *);
 194
 195 vm_object_offset_t
 196 vnode_pager_get_filesize(
 197         vnode_port_t);
 198
 199 vnode_pager_t
 200 vnode_object_create(
 201         vnode_port_t    vp);
 202
 203 vnode_pager_t
 204 vnode_pager_lookup(
 205         memory_object_t);
 206
 207 void
 208 vnode_pager_release_from_cache(
 209         int     *cnt);
 210
 211 zone_t  vnode_pager_zone;
 212
 213
 214 #define VNODE_PAGER_NULL        ((vnode_pager_t) 0)
 215
 216 /* TODO: Should be set dynamically by vnode_pager_init() */
 217 #define CLUSTER_SHIFT   1
 218
 219 /* TODO: Should be set dynamically by vnode_pager_bootstrap() */
 220 #define MAX_VNODE               10000
 221
 222
 223 #if DEBUG
 224 int pagerdebug=0;
 225
 226 #define PAGER_ALL               0xffffffff
 227 #define PAGER_INIT              0x00000001
 228 #define PAGER_PAGEIN    0x00000002
 229
 230 #define PAGER_DEBUG(LEVEL, A) {if ((pagerdebug & LEVEL)==LEVEL){printf A;}}
 231 #else
 232 #define PAGER_DEBUG(LEVEL, A)
 233 #endif
 234
 235 /*
 236  *      Routine:        macx_triggers
 237  *      Function:
 238  *              Syscall interface to set the call backs for low and
 239  *              high water marks.
 240  */
 241 int
 242 macx_triggers(
 243         int     hi_water,
 244         int     low_water,
 245         int     flags,
 246         mach_port_t     trigger_name)
 247 {
 248         kern_return_t kr;
 249         memory_object_default_t default_pager;
 250         ipc_port_t              trigger_port;
 251
 252         default_pager = MEMORY_OBJECT_DEFAULT_NULL;
 253         kr = host_default_memory_manager(host_priv_self(),
 254                                         &default_pager, 0);
 255         if(kr != KERN_SUCCESS) {
 256                 return EINVAL;
 257         }
 258         if (flags & HI_WAT_ALERT) {
 259                 trigger_port = trigger_name_to_port(trigger_name);
 260                 if(trigger_port == NULL) {
 261                         return EINVAL;
 262                 }
 263                 /* trigger_port is locked and active */
 264                 ipc_port_make_send_locked(trigger_port);
 265                 /* now unlocked */
 266                 default_pager_triggers(default_pager,
 267                                        hi_water, low_water,
 268                                        HI_WAT_ALERT, trigger_port);
 269         }
 270
 271         if (flags & LO_WAT_ALERT) {
 272                 trigger_port = trigger_name_to_port(trigger_name);
 273                 if(trigger_port == NULL) {
 274                         return EINVAL;
 275                 }
 276                 /* trigger_port is locked and active */
 277                 ipc_port_make_send_locked(trigger_port);
 278                 /* and now its unlocked */
 279                 default_pager_triggers(default_pager,
 280                                        hi_water, low_water,
 281                                        LO_WAT_ALERT, trigger_port);
 282         }
 283
 284         /*
 285          * Set thread scheduling priority and policy for the current thread
 286          * it is assumed for the time being that the thread setting the alert
 287          * is the same one which will be servicing it.
 288          *
 289          * XXX This does not belong in the kernel XXX
 290          */
 291         {
 292                 thread_precedence_policy_data_t         pre;
 293                 thread_extended_policy_data_t           ext;
 294
 295                 ext.timeshare = FALSE;
 296                 pre.importance = INT32_MAX;
 297
 298                 thread_policy_set(current_act(),
 299                                                         THREAD_EXTENDED_POLICY, (thread_policy_t)&ext,
 300                                                                         THREAD_EXTENDED_POLICY_COUNT);
 301
 302                 thread_policy_set(current_act(),
 303                                                         THREAD_PRECEDENCE_POLICY, (thread_policy_t)&pre,
 304                                                                         THREAD_PRECEDENCE_POLICY_COUNT);
 305         }
 306
 307         current_thread()->vm_privilege = TRUE;
 308 }
 309
 310 /*
 311  *
 312  */
 313 ipc_port_t
 314 trigger_name_to_port(
 315         mach_port_t     trigger_name)
 316 {
 317         ipc_port_t      trigger_port;
 318         ipc_space_t     space;
 319
 320         if (trigger_name == 0)
 321                 return (NULL);
 322
 323         space  = current_space();
 324         if(ipc_port_translate_receive(space, (mach_port_name_t)trigger_name,
 325                                                 &trigger_port) != KERN_SUCCESS)
 326                 return (NULL);
 327         return trigger_port;
 328 }
 329
 330 /*
 331  *
 332  */
 333 void
 334 vnode_pager_bootstrap(void)
 335 {
 336         register vm_size_t      size;
 337
 338         size = (vm_size_t) sizeof(struct vnode_pager);
 339         vnode_pager_zone = zinit(size, (vm_size_t) MAX_VNODE*size,
 340                                 PAGE_SIZE, "vnode pager structures");
 341         return;
 342 }
 343
 344 /*
 345  *
 346  */
 347 memory_object_t
 348 vnode_pager_setup(
 349         vnode_port_t    vp,
 350         memory_object_t pager)
 351 {
 352         vnode_pager_t   vnode_object;
 353
 354         vnode_object = vnode_object_create(vp);
 355         if (vnode_object == VNODE_PAGER_NULL)
 356                 panic("vnode_pager_setup: vnode_object_create() failed");
 357         return((memory_object_t)vnode_object);
 358 }
 359
 360 /*
 361  *
 362  */
 363 kern_return_t
 364 vnode_pager_init(memory_object_t mem_obj,
 365                 memory_object_control_t control,
 366                 vm_size_t pg_size)
 367 {
 368         vnode_pager_t   vnode_object;
 369         kern_return_t   kr;
 370         memory_object_attr_info_data_t  attributes;
 371
 372
 373         PAGER_DEBUG(PAGER_ALL, ("vnode_pager_init: %x, %x, %x\n", pager, pager_request, pg_size));
 374
 375         if (control == MEMORY_OBJECT_CONTROL_NULL)
 376                 return KERN_INVALID_ARGUMENT;
 377
 378         vnode_object = vnode_pager_lookup(mem_obj);
 379
 380         memory_object_control_reference(control);
 381         vnode_object->control_handle = control;
 382
 383         attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY;
 384         /* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/
 385         attributes.cluster_size = (1 << (PAGE_SHIFT));
 386         attributes.may_cache_object = TRUE;
 387         attributes.temporary = TRUE;
 388
 389         kr = memory_object_change_attributes(
 390                                         control,
 391                                         MEMORY_OBJECT_ATTRIBUTE_INFO,
 392                                         (memory_object_info_t) &attributes,
 393                                         MEMORY_OBJECT_ATTR_INFO_COUNT);
 394         if (kr != KERN_SUCCESS)
 395                 panic("vnode_pager_init: memory_object_change_attributes() failed");
 396
 397         return(KERN_SUCCESS);
 398 }
 399
 400 /*
 401  *
 402  */
 403 kern_return_t
 404 vnode_pager_data_return(
 405         memory_object_t         mem_obj,
 406         memory_object_offset_t  offset,
 407         vm_size_t               data_cnt,
 408         boolean_t               dirty,
 409         boolean_t               kernel_copy)
 410 {
 411         register vnode_pager_t  vnode_object;
 412
 413         vnode_object = vnode_pager_lookup(mem_obj);
 414
 415         vnode_pager_cluster_write(vnode_object, offset, data_cnt);
 416
 417         return KERN_SUCCESS;
 418 }
 419
 420 kern_return_t
 421 vnode_pager_data_initialize(
 422         memory_object_t         mem_obj,
 423         memory_object_offset_t  offset,
 424         vm_size_t               data_cnt)
 425 {
 426         return KERN_FAILURE;
 427 }
 428
 429 kern_return_t
 430 vnode_pager_data_unlock(
 431         memory_object_t         mem_obj,
 432         memory_object_offset_t  offset,
 433         vm_size_t               size,
 434         vm_prot_t               desired_access)
 435 {
 436         return KERN_FAILURE;
 437 }
 438
 439 kern_return_t
 440 vnode_pager_get_object_size(
 441         memory_object_t         mem_obj,
 442         memory_object_offset_t  *length)
 443 {
 444         vnode_pager_t   vnode_object;
 445
 446         vnode_object = vnode_pager_lookup(mem_obj);
 447
 448         *length = vnode_pager_get_filesize(vnode_object->vnode_handle);
 449         return KERN_SUCCESS;
 450 }
 451
 452 /*
 453  *
 454  */
 455 kern_return_t
 456 vnode_pager_data_request(
 457         memory_object_t         mem_obj,
 458         memory_object_offset_t  offset,
 459         vm_size_t               length,
 460         vm_prot_t               protection_required)
 461 {
 462         register vnode_pager_t  vnode_object;
 463
 464         PAGER_DEBUG(PAGER_ALL, ("vnode_pager_data_request: %x, %x, %x, %x\n", mem_obj, offset, length, protection_required));
 465
 466         vnode_object = vnode_pager_lookup(mem_obj);
 467
 468         PAGER_DEBUG(PAGER_PAGEIN, ("vnode_pager_data_request: %x, %x, %x, %x, vnode_object %x\n", mem_obj, offset, length, protection_required, vnode_object));
 469
 470         vnode_pager_cluster_read(vnode_object, offset, length);
 471
 472         return KERN_SUCCESS;
 473 }
 474
 475 /*
 476  *
 477  */
 478 void
 479 vnode_pager_reference(
 480         memory_object_t         mem_obj)
 481 {
 482         register vnode_pager_t  vnode_object;
 483         unsigned int            new_ref_count;
 484
 485         vnode_object = vnode_pager_lookup(mem_obj);
 486         new_ref_count = hw_atomic_add(&vnode_object->ref_count, 1);
 487         assert(new_ref_count > 1);
 488 }
 489
 490 /*
 491  *
 492  */
 493 void
 494 vnode_pager_deallocate(
 495         memory_object_t         mem_obj)
 496 {
 497         register vnode_pager_t  vnode_object;
 498
 499         PAGER_DEBUG(PAGER_ALL, ("vnode_pager_deallocate: %x\n", mem_obj));
 500
 501         vnode_object = vnode_pager_lookup(mem_obj);
 502
 503         if (hw_atomic_sub(&vnode_object->ref_count, 1) == 0) {
 504                 if (vnode_object->vnode_handle != (vnode_port_t) NULL) {
 505                         vnode_pager_vrele(vnode_object->vnode_handle);
 506                 }
 507                 zfree(vnode_pager_zone, (vm_offset_t) vnode_object);
 508         }
 509         return;
 510 }
 511
 512 /*
 513  *
 514  */
 515 kern_return_t
 516 vnode_pager_terminate(
 517         memory_object_t mem_obj)
 518 {
 519         PAGER_DEBUG(PAGER_ALL, ("vnode_pager_terminate: %x\n", mem_obj));
 520
 521         return(KERN_SUCCESS);
 522 }
 523
 524 /*
 525  *
 526  */
 527 kern_return_t
 528 vnode_pager_synchronize(
 529         memory_object_t         mem_obj,
 530         memory_object_offset_t  offset,
 531         vm_size_t               length,
 532         vm_sync_t               sync_flags)
 533 {
 534         register vnode_pager_t  vnode_object;
 535
 536         PAGER_DEBUG(PAGER_ALL, ("vnode_pager_synchronize: %x\n", mem_obj));
 537
 538         vnode_object = vnode_pager_lookup(mem_obj);
 539
 540         memory_object_synchronize_completed(vnode_object->control_handle, offset, length);
 541
 542         return (KERN_SUCCESS);
 543 }
 544
 545 /*
 546  *
 547  */
 548 kern_return_t
 549 vnode_pager_unmap(
 550         memory_object_t         mem_obj)
 551 {
 552         register vnode_pager_t  vnode_object;
 553
 554         PAGER_DEBUG(PAGER_ALL, ("vnode_pager_unmap: %x\n", mem_obj));
 555
 556         vnode_object = vnode_pager_lookup(mem_obj);
 557
 558         ubc_unmap(vnode_object->vnode_handle);
 559         return KERN_SUCCESS;
 560 }
 561
 562
 563 /*
 564  *
 565  */
 566 void
 567 vnode_pager_cluster_write(
 568         vnode_pager_t           vnode_object,
 569         vm_object_offset_t      offset,
 570         vm_size_t               cnt)
 571 {
 572         int             error = 0;
 573         int             local_error = 0;
 574         int             kret;
 575         int             size;
 576
 577         if (cnt & PAGE_MASK) {
 578                 panic("vs_cluster_write: cnt not a multiple of PAGE_SIZE");
 579         }
 580         size = (cnt < (PAGE_SIZE*32)) ? cnt : (PAGE_SIZE*32); /* effective min */
 581
 582         while (cnt) {
 583
 584                 kret = vnode_pageout(vnode_object->vnode_handle,
 585                         (upl_t )NULL, (vm_offset_t)NULL,
 586                         offset, size, 0, &local_error);
 587 /*
 588                 if(kret == PAGER_ABSENT) {
 589                         Need to work out the defs here, 1 corresponds to
 590                         PAGER_ABSENT defined in bsd/vm/vm_pager.h  However,
 591                         we should not be including that file here it is a
 592                         layering violation.
 593 */
 594                 if(kret == 1) {
 595                         int     uplflags;
 596                         upl_t   upl = NULL;
 597                         int     count = 0;
 598                         kern_return_t   kr;
 599
 600                         uplflags = (UPL_NO_SYNC | UPL_CLEAN_IN_PLACE |
 601                                         UPL_SET_INTERNAL | UPL_COPYOUT_FROM);
 602                         count = 0;
 603                         kr = memory_object_upl_request(
 604                                         vnode_object->control_handle,
 605                                         offset, size, &upl, NULL, &count, uplflags);
 606                         if(kr != KERN_SUCCESS) {
 607                                 panic("vnode_pager_cluster_write: upl request failed\n");
 608                         }
 609                         upl_abort(upl, 0);
 610                         upl_deallocate(upl);
 611
 612                         error = 0;
 613                         local_error = 0;
 614                 }
 615
 616                 if (local_error != 0) {
 617                         error = local_error;
 618                         local_error = 0;
 619                 }
 620                 cnt -= size;
 621                 offset += size;
 622                 size = (cnt < (PAGE_SIZE*32)) ? cnt : (PAGE_SIZE*32); /* effective min */
 623         }
 624 #if 0
 625         if (error != 0)
 626                 return(KERN_FAILURE);
 627
 628         return(KERN_SUCCESS);
 629 #endif /* 0 */
 630 }
 631
 632
 633 /*
 634  *
 635  */
 636 kern_return_t
 637 vnode_pager_cluster_read(
 638         vnode_pager_t           vnode_object,
 639         vm_object_offset_t      offset,
 640         vm_size_t               cnt)
 641 {
 642         int             error = 0;
 643         int             local_error = 0;
 644         int             kret;
 645
 646         if(cnt & PAGE_MASK) {
 647                 panic("vs_cluster_read: cnt not a multiple of PAGE_SIZE");
 648         }
 649
 650         kret = vnode_pagein(vnode_object->vnode_handle, (upl_t)NULL, (vm_offset_t)NULL, offset, cnt, 0, &local_error);
 651 /*
 652         if(kret == PAGER_ABSENT) {
 653         Need to work out the defs here, 1 corresponds to PAGER_ABSENT
 654         defined in bsd/vm/vm_pager.h  However, we should not be including
 655         that file here it is a layering violation.
 656 */
 657         if(kret == 1) {
 658                         int     uplflags;
 659                         upl_t   upl = NULL;
 660                         int     count = 0;
 661                         kern_return_t   kr;
 662
 663                         uplflags = (UPL_NO_SYNC |
 664                                 UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL);
 665                         count = 0;
 666                         kr = memory_object_upl_request(
 667                                 vnode_object->control_handle, offset, cnt,
 668                                 &upl, NULL, &count, uplflags);
 669                         if(kr != KERN_SUCCESS) {
 670                                 panic("vnode_pager_cluster_read: upl request failed\n");
 671                         }
 672                         upl_abort(upl, 0);
 673                         upl_deallocate(upl);
 674
 675                 error = 1;
 676         }
 677
 678         if (error != 0)
 679                 return(KERN_FAILURE);
 680
 681         return(KERN_SUCCESS);
 682
 683 }
 684
 685
 686 /*
 687  *
 688  */
 689 void
 690 vnode_pager_release_from_cache(
 691                 int     *cnt)
 692 {
 693         memory_object_free_from_cache(
 694                         &realhost, &vnode_pager_workaround, cnt);
 695 }
 696
 697 /*
 698  *
 699  */
 700 vnode_pager_t
 701 vnode_object_create(
 702         vnode_port_t   vp)
 703 {
 704         register vnode_pager_t  vnode_object;
 705
 706         vnode_object = (struct vnode_pager *) zalloc(vnode_pager_zone);
 707         if (vnode_object == VNODE_PAGER_NULL)
 708                 return(VNODE_PAGER_NULL);
 709
 710         /*
 711          * The vm_map call takes both named entry ports and raw memory
 712          * objects in the same parameter.  We need to make sure that
 713          * vm_map does not see this object as a named entry port.  So,
 714          * we reserve the second word in the object for a fake ip_kotype
 715          * setting - that will tell vm_map to use it as a memory object.
 716          */
 717         vnode_object->pager = &vnode_pager_workaround;
 718         vnode_object->pager_ikot = IKOT_MEMORY_OBJECT;
 719         vnode_object->ref_count = 1;
 720         vnode_object->control_handle = MEMORY_OBJECT_CONTROL_NULL;
 721         vnode_object->vnode_handle = vp;
 722
 723         return(vnode_object);
 724 }
 725
 726 /*
 727  *
 728  */
 729 vnode_pager_t
 730 vnode_pager_lookup(
 731         memory_object_t  name)
 732 {
 733         vnode_pager_t   vnode_object;
 734
 735         vnode_object = (vnode_pager_t)name;
 736         assert(vnode_object->pager == &vnode_pager_workaround);
 737         return (vnode_object);
 738 }
 739