bsd/vm/vnode_pager.c

   1 /*
   2  * Copyright (c) 2006 Apple Computer, Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_OSREFERENCE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License.  The rights granted to you under the
  10  * License may not be used to create, or enable the creation or
  11  * redistribution of, unlawful or unlicensed copies of an Apple operating
  12  * system, or to circumvent, violate, or enable the circumvention or
  13  * violation of, any terms of an Apple operating system software license
  14  * agreement.
  15  *
  16  * Please obtain a copy of the License at
  17  * http://www.opensource.apple.com/apsl/ and read it before using this
  18  * file.
  19  *
  20  * The Original Code and all software distributed under the License are
  21  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  22  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  23  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  24  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  25  * Please see the License for the specific language governing rights and
  26  * limitations under the License.
  27  *
  28  * @APPLE_LICENSE_OSREFERENCE_HEADER_END@
  29  */
  30 /*
  31  * Mach Operating System
  32  * Copyright (c) 1987 Carnegie-Mellon University
  33  * All rights reserved.  The CMU software License Agreement specifies
  34  * the terms and conditions for use and redistribution.
  35  */
  36 /*
  37  *      File:   vnode_pager.c
  38  *
  39  *      "Swap" pager that pages to/from vnodes.  Also
  40  *      handles demand paging from files.
  41  *
  42  */
  43
  44 #include <mach/boolean.h>
  45 #include <sys/param.h>
  46 #include <sys/systm.h>
  47 #include <sys/user.h>
  48 #include <sys/proc.h>
  49 #include <sys/kauth.h>
  50 #include <sys/buf.h>
  51 #include <sys/uio.h>
  52 #include <sys/vnode_internal.h>
  53 #include <sys/namei.h>
  54 #include <sys/mount_internal.h> /* needs internal due to fhandle_t */
  55 #include <sys/ubc_internal.h>
  56 #include <sys/lock.h>
  57
  58 #include <mach/mach_types.h>
  59 #include <mach/memory_object_types.h>
  60
  61 #include <vm/vm_map.h>
  62 #include <vm/vm_kern.h>
  63 #include <kern/zalloc.h>
  64 #include <kern/kalloc.h>
  65 #include <libkern/libkern.h>
  66
  67 #include <vm/vnode_pager.h>
  68 #include <vm/vm_pageout.h>
  69
  70 #include <kern/assert.h>
  71 #include <sys/kdebug.h>
  72 #include <machine/spl.h>
  73
  74 #include <nfs/rpcv2.h>
  75 #include <nfs/nfsproto.h>
  76 #include <nfs/nfs.h>
  77
  78 #include <vm/vm_protos.h>
  79
  80 unsigned int vp_pagein=0;
  81 unsigned int vp_pgodirty=0;
  82 unsigned int vp_pgoclean=0;
  83 unsigned int dp_pgouts=0;       /* Default pager pageouts */
  84 unsigned int dp_pgins=0;        /* Default pager pageins */
  85
  86 vm_object_offset_t
  87 vnode_pager_get_filesize(struct vnode *vp)
  88 {
  89
  90         return (vm_object_offset_t) ubc_getsize(vp);
  91 }
  92
  93 pager_return_t
  94 vnode_pageout(struct vnode *vp,
  95         upl_t                   upl,
  96         vm_offset_t             upl_offset,
  97         vm_object_offset_t      f_offset,
  98         vm_size_t               size,
  99         int                     flags,
 100         int                     *errorp)
 101 {
 102         struct proc     *p = current_proc();
 103         int             result = PAGER_SUCCESS;
 104         int             error = 0;
 105         int             error_ret = 0;
 106         daddr64_t blkno;
 107         int isize;
 108         int pg_index;
 109         int base_index;
 110         int offset;
 111         upl_page_info_t *pl;
 112         struct vfs_context context;
 113
 114         context.vc_proc = p;
 115         context.vc_ucred = kauth_cred_get();
 116
 117         isize = (int)size;
 118
 119         if (isize <= 0) {
 120                 result    = PAGER_ERROR;
 121                 error_ret = EINVAL;
 122                 goto out;
 123         }
 124         UBCINFOCHECK("vnode_pageout", vp);
 125
 126         if (UBCINVALID(vp)) {
 127                 result    = PAGER_ERROR;
 128                 error_ret = EINVAL;
 129
 130                 if (upl && !(flags & UPL_NOCOMMIT))
 131                         ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
 132                 goto out;
 133         }
 134         if ( !(flags & UPL_VNODE_PAGER)) {
 135                 /*
 136                  * This is a pageout from the default pager,
 137                  * just go ahead and call vnop_pageout since
 138                  * it has already sorted out the dirty ranges
 139                  */
 140                 dp_pgouts++;
 141
 142                 KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START,
 143                                       size, 1, 0, 0, 0);
 144
 145                 if ( (error_ret = VNOP_PAGEOUT(vp, upl, upl_offset, (off_t)f_offset,
 146                                                (size_t)size, flags, &context)) )
 147                         result = PAGER_ERROR;
 148
 149                 KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END,
 150                                       size, 1, 0, 0, 0);
 151
 152                 goto out;
 153         }
 154         /*
 155          * we come here for pageouts to 'real' files and
 156          * for msyncs...  the upl may not contain any
 157          * dirty pages.. it's our responsibility to sort
 158          * through it and find the 'runs' of dirty pages
 159          * to call VNOP_PAGEOUT on...
 160          */
 161         pl = ubc_upl_pageinfo(upl);
 162
 163         if (ubc_getsize(vp) == 0) {
 164                 /*
 165                  * if the file has been effectively deleted, then
 166                  * we need to go through the UPL and invalidate any
 167                  * buffer headers we might have that reference any
 168                  * of it's pages
 169                  */
 170                 for (offset = upl_offset; isize; isize -= PAGE_SIZE, offset += PAGE_SIZE) {
 171 #if NFSCLIENT
 172                         if (vp->v_tag == VT_NFS)
 173                                 /* check with nfs if page is OK to drop */
 174                                 error = nfs_buf_page_inval(vp, (off_t)f_offset);
 175                         else
 176 #endif
 177                         {
 178                                 blkno = ubc_offtoblk(vp, (off_t)f_offset);
 179                                 error = buf_invalblkno(vp, blkno, 0);
 180                         }
 181                         if (error) {
 182                                 if ( !(flags & UPL_NOCOMMIT))
 183                                         ubc_upl_abort_range(upl, offset, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
 184                                 if (error_ret == 0)
 185                                         error_ret = error;
 186                                 result = PAGER_ERROR;
 187
 188                         } else if ( !(flags & UPL_NOCOMMIT)) {
 189                                 ubc_upl_commit_range(upl, offset, PAGE_SIZE, UPL_COMMIT_FREE_ON_EMPTY);
 190                         }
 191                         f_offset += PAGE_SIZE;
 192                 }
 193                 goto out;
 194         }
 195         /*
 196          * Ignore any non-present pages at the end of the
 197          * UPL so that we aren't looking at a upl that
 198          * may already have been freed by the preceeding
 199          * aborts/completions.
 200          */
 201         base_index = upl_offset / PAGE_SIZE;
 202
 203         for (pg_index = (upl_offset + isize) / PAGE_SIZE; pg_index > base_index;) {
 204                 if (upl_page_present(pl, --pg_index))
 205                         break;
 206                 if (pg_index == base_index) {
 207                         /*
 208                          * no pages were returned, so release
 209                          * our hold on the upl and leave
 210                          */
 211                         if ( !(flags & UPL_NOCOMMIT))
 212                                 ubc_upl_abort_range(upl, upl_offset, isize, UPL_ABORT_FREE_ON_EMPTY);
 213
 214                         goto out;
 215                 }
 216         }
 217         isize = (pg_index + 1) * PAGE_SIZE;
 218
 219         offset = upl_offset;
 220         pg_index = base_index;
 221
 222         while (isize) {
 223                 int  xsize;
 224                 int  num_of_pages;
 225
 226                 if ( !upl_page_present(pl, pg_index)) {
 227                         /*
 228                          * we asked for RET_ONLY_DIRTY, so it's possible
 229                          * to get back empty slots in the UPL
 230                          * just skip over them
 231                          */
 232                         offset += PAGE_SIZE;
 233                         isize  -= PAGE_SIZE;
 234                         pg_index++;
 235
 236                         continue;
 237                 }
 238                 if ( !upl_dirty_page(pl, pg_index)) {
 239                         /*
 240                          * if the page is not dirty and reached here it is
 241                          * marked precious or it is due to invalidation in
 242                          * memory_object_lock request as part of truncation
 243                          * We also get here from vm_object_terminate()
 244                          * So all you need to do in these
 245                          * cases is to invalidate incore buffer if it is there
 246                          * Note we must not sleep here if the buffer is busy - that is
 247                          * a lock inversion which causes deadlock.
 248                          */
 249                         vp_pgoclean++;
 250
 251 #if NFSCLIENT
 252                         if (vp->v_tag == VT_NFS)
 253                                 /* check with nfs if page is OK to drop */
 254                                 error = nfs_buf_page_inval(vp, (off_t)(f_offset + offset));
 255                         else
 256 #endif
 257                         {
 258                                 blkno = ubc_offtoblk(vp, (off_t)(f_offset + offset));
 259                                 error = buf_invalblkno(vp, blkno, 0);
 260                         }
 261                         if (error) {
 262                                 if ( !(flags & UPL_NOCOMMIT))
 263                                         ubc_upl_abort_range(upl, offset, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
 264                                 if (error_ret == 0)
 265                                         error_ret = error;
 266                                 result = PAGER_ERROR;
 267
 268                         } else if ( !(flags & UPL_NOCOMMIT)) {
 269                                 ubc_upl_commit_range(upl, offset, PAGE_SIZE, UPL_COMMIT_FREE_ON_EMPTY);
 270                         }
 271                         offset += PAGE_SIZE;
 272                         isize  -= PAGE_SIZE;
 273                         pg_index++;
 274
 275                         continue;
 276                 }
 277                 vp_pgodirty++;
 278
 279                 num_of_pages = 1;
 280                 xsize = isize - PAGE_SIZE;
 281
 282                 while (xsize) {
 283                         if ( !upl_dirty_page(pl, pg_index + num_of_pages))
 284                                 break;
 285                         num_of_pages++;
 286                         xsize -= PAGE_SIZE;
 287                 }
 288                 xsize = num_of_pages * PAGE_SIZE;
 289
 290                 KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START,
 291                                       xsize, (int)(f_offset + offset), 0, 0, 0);
 292
 293                 if ( (error = VNOP_PAGEOUT(vp, upl, (vm_offset_t)offset,
 294                                         (off_t)(f_offset + offset), xsize,
 295                                            flags, &context)) ) {
 296                         if (error_ret == 0)
 297                                 error_ret = error;
 298                         result = PAGER_ERROR;
 299                 }
 300                 KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END,
 301                                       xsize, 0, 0, 0, 0);
 302
 303                 offset += xsize;
 304                 isize  -= xsize;
 305                 pg_index += num_of_pages;
 306         }
 307 out:
 308         if (errorp)
 309                 *errorp = error_ret;
 310
 311         return (result);
 312 }
 313
 314
 315 void IOSleep(int);
 316
 317 pager_return_t
 318 vnode_pagein(
 319         struct vnode            *vp,
 320         upl_t                   upl,
 321         vm_offset_t             upl_offset,
 322         vm_object_offset_t      f_offset,
 323         vm_size_t               size,
 324         int                     flags,
 325         int                     *errorp)
 326 {
 327         struct proc     *p = current_proc();
 328         struct uthread  *ut;
 329         upl_page_info_t *pl;
 330         int             result = PAGER_SUCCESS;
 331         int             error = 0;
 332         int             pages_in_upl;
 333         int             start_pg;
 334         int             last_pg;
 335         int             first_pg;
 336         int             xsize;
 337         int             abort_needed = 1;
 338
 339
 340         UBCINFOCHECK("vnode_pagein", vp);
 341
 342         if (UBCINVALID(vp)) {
 343                 result = PAGER_ERROR;
 344                 error  = PAGER_ERROR;
 345                 if (upl && !(flags & UPL_NOCOMMIT)) {
 346                         ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);
 347                 }
 348                 goto out;
 349         }
 350         if (upl == (upl_t)NULL) {
 351                 if (size > (MAX_UPL_TRANSFER * PAGE_SIZE)) {
 352                         result = PAGER_ERROR;
 353                         error  = PAGER_ERROR;
 354                         goto out;
 355                 }
 356                 ubc_create_upl(vp, f_offset, size, &upl, &pl, UPL_RET_ONLY_ABSENT | UPL_SET_LITE);
 357
 358                 if (upl == (upl_t)NULL) {
 359                         result =  PAGER_ABSENT;
 360                         error = PAGER_ABSENT;
 361                         goto out;
 362                 }
 363                 upl_offset = 0;
 364                 /*
 365                  * if we get here, we've created the upl and
 366                  * are responsible for commiting/aborting it
 367                  * regardless of what the caller has passed in
 368                  */
 369                 flags &= ~UPL_NOCOMMIT;
 370
 371                 vp_pagein++;
 372         } else {
 373                 pl = ubc_upl_pageinfo(upl);
 374
 375                 dp_pgins++;
 376         }
 377         pages_in_upl = size / PAGE_SIZE;
 378         first_pg     = upl_offset / PAGE_SIZE;
 379
 380         /*
 381          * before we start marching forward, we must make sure we end on
 382          * a present page, otherwise we will be working with a freed
 383          * upl
 384          */
 385         for (last_pg = pages_in_upl - 1; last_pg >= first_pg; last_pg--) {
 386                 if (upl_page_present(pl, last_pg))
 387                         break;
 388         }
 389         pages_in_upl = last_pg + 1;
 390
 391         for (last_pg = first_pg; last_pg < pages_in_upl;) {
 392                 /*
 393                  * scan the upl looking for the next
 394                  * page that is present.... if all of the
 395                  * pages are absent, we're done
 396                  */
 397                 for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
 398                         if (upl_page_present(pl, last_pg))
 399                                 break;
 400                 }
 401                 if (last_pg == pages_in_upl)
 402                         break;
 403
 404                 /*
 405                  * if we get here, we've sitting on a page
 406                  * that is present... we want to skip over
 407                  * any range of 'valid' pages... if this takes
 408                  * us to the end of the request, than we're done
 409                  */
 410                 for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
 411                         if (!upl_valid_page(pl, last_pg) || !upl_page_present(pl, last_pg))
 412                                 break;
 413                 }
 414                 if (last_pg > start_pg) {
 415                         /*
 416                          * we've found a range of valid pages
 417                          * if we've got COMMIT responsibility
 418                          * commit this range of pages back to the
 419                          * cache unchanged
 420                          */
 421                         xsize = (last_pg - start_pg) * PAGE_SIZE;
 422
 423                         if (!(flags & UPL_NOCOMMIT))
 424                                 ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, xsize, UPL_ABORT_FREE_ON_EMPTY);
 425
 426                         abort_needed = 0;
 427                 }
 428                 if (last_pg == pages_in_upl)
 429                         break;
 430
 431                 if (!upl_page_present(pl, last_pg))
 432                         /*
 433                          * if we found a range of valid pages
 434                          * terminated by a non-present page
 435                          * than start over
 436                          */
 437                         continue;
 438
 439                 /*
 440                  * scan from the found invalid page looking for a valid
 441                  * or non-present page before the end of the upl is reached, if we
 442                  * find one, then it will be the last page of the request to
 443                  * 'cluster_io'
 444                  */
 445                 for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
 446                         if (upl_valid_page(pl, last_pg) || !upl_page_present(pl, last_pg))
 447                                 break;
 448                 }
 449                 if (last_pg > start_pg) {
 450                         int xoff;
 451                         struct vfs_context context;
 452
 453                         context.vc_proc = p;
 454                         context.vc_ucred = kauth_cred_get();
 455                         xsize = (last_pg - start_pg) * PAGE_SIZE;
 456                         xoff  = start_pg * PAGE_SIZE;
 457
 458                         if ( (error = VNOP_PAGEIN(vp, upl, (vm_offset_t) xoff,
 459                                                (off_t)f_offset + xoff,
 460                                                xsize, flags, &context)) ) {
 461                                 result = PAGER_ERROR;
 462                                 error  = PAGER_ERROR;
 463
 464                         }
 465                         abort_needed = 0;
 466                 }
 467         }
 468         if (!(flags & UPL_NOCOMMIT) && abort_needed)
 469                 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
 470 out:
 471         if (errorp)
 472                 *errorp = result;
 473
 474         ut = get_bsdthread_info(current_thread());
 475
 476         if (ut->uu_lowpri_delay) {
 477                 /*
 478                  * task is marked as a low priority I/O type
 479                  * and the I/O we issued while in this system call
 480                  * collided with normal I/O operations... we'll
 481                  * delay in order to mitigate the impact of this
 482                  * task on the normal operation of the system
 483                  */
 484                 IOSleep(ut->uu_lowpri_delay);
 485                 ut->uu_lowpri_delay = 0;
 486         }
 487         return (error);
 488 }
 489
 490 void
 491 vnode_pager_shutdown(void)
 492 {
 493         int i;
 494         vnode_t vp;
 495
 496         for(i = 0; i < MAX_BACKING_STORE; i++) {
 497                 vp = (vnode_t)(bs_port_table[i]).vp;
 498                 if (vp) {
 499                         (bs_port_table[i]).vp = 0;
 500
 501                         /* get rid of macx_swapon() reference */
 502                         vnode_rele(vp);
 503                 }
 504         }
 505 }
 506
 507
 508 void *
 509 upl_get_internal_page_list(upl_t upl)
 510 {
 511   return(UPL_GET_INTERNAL_PAGE_LIST(upl));
 512
 513 }