bsd/vm/vnode_pager.c

   1 /*
   2  * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * The contents of this file constitute Original Code as defined in and
   7  * are subject to the Apple Public Source License Version 1.1 (the
   8  * "License").  You may not use this file except in compliance with the
   9  * License.  Please obtain a copy of the License at
  10  * http://www.apple.com/publicsource and read it before using this file.
  11  *
  12  * This Original Code and all software distributed under the License are
  13  * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  14  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  15  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
  17  * License for the specific language governing rights and limitations
  18  * under the License.
  19  *
  20  * @APPLE_LICENSE_HEADER_END@
  21  */
  22 /*
  23  * Mach Operating System
  24  * Copyright (c) 1987 Carnegie-Mellon University
  25  * All rights reserved.  The CMU software License Agreement specifies
  26  * the terms and conditions for use and redistribution.
  27  */
  28 /*
  29  *      File:   vnode_pager.c
  30  *
  31  *      "Swap" pager that pages to/from vnodes.  Also
  32  *      handles demand paging from files.
  33  *
  34  */
  35
  36 #include <mach/boolean.h>
  37 #include <sys/param.h>
  38 #include <sys/systm.h>
  39 #include <sys/proc.h>
  40 #include <sys/buf.h>
  41 #include <sys/uio.h>
  42 #include <sys/vnode.h>
  43 #include <sys/namei.h>
  44 #include <sys/mount.h>
  45 #include <sys/ubc.h>
  46 #include <sys/lock.h>
  47
  48 #include <mach/mach_types.h>
  49 #include <mach/memory_object_types.h>
  50
  51 #include <vm/vm_map.h>
  52 #include <vm/vm_kern.h>
  53 #include <kern/zalloc.h>
  54 #include <kern/kalloc.h>
  55 #include <libkern/libkern.h>
  56
  57 #include <vm/vnode_pager.h>
  58 #include <vm/vm_pageout.h>
  59
  60 #include <kern/assert.h>
  61 #include <sys/kdebug.h>
  62
  63 unsigned int vp_pagein=0;
  64 unsigned int vp_pgodirty=0;
  65 unsigned int vp_pgoclean=0;
  66 unsigned int dp_pgouts=0;       /* Default pager pageouts */
  67 unsigned int dp_pgins=0;        /* Default pager pageins */
  68
  69 vm_object_offset_t
  70 vnode_pager_get_filesize(struct vnode *vp)
  71 {
  72         if (UBCINVALID(vp)) {
  73                 return (vm_object_offset_t) 0;
  74         }
  75
  76         return (vm_object_offset_t) ubc_getsize(vp);
  77
  78 }
  79
  80 pager_return_t
  81 vnode_pageout(struct vnode *vp,
  82         upl_t                   upl,
  83         vm_offset_t             upl_offset,
  84         vm_object_offset_t      f_offset,
  85         vm_size_t               size,
  86         int                     flags,
  87         int                     *errorp)
  88 {
  89         int             result = PAGER_SUCCESS;
  90         struct proc     *p = current_proc();
  91         int             error = 0;
  92         int blkno=0, s;
  93         int cnt, isize;
  94         int pg_index;
  95         int offset;
  96         struct buf *bp;
  97         boolean_t       funnel_state;
  98         upl_page_info_t *pl;
  99         upl_t vpupl = NULL;
 100
 101         funnel_state = thread_funnel_set(kernel_flock, TRUE);
 102
 103         isize = (int)size;
 104
 105         if (isize <= 0) {
 106                 result = error = PAGER_ERROR;
 107                 goto out;
 108         }
 109         UBCINFOCHECK("vnode_pageout", vp);
 110
 111         if (UBCINVALID(vp)) {
 112                 result = error = PAGER_ERROR;
 113
 114                 if (upl && !(flags & UPL_NOCOMMIT))
 115                         ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
 116                 goto out;
 117         }
 118         if (upl) {
 119                 /*
 120                  * This is a pageout from the Default pager,
 121                  * just go ahead and call VOP_PAGEOUT
 122                  */
 123                 dp_pgouts++;
 124
 125                 KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START,
 126                                       size, 1, 0, 0, 0);
 127
 128                 if (error = VOP_PAGEOUT(vp, upl, upl_offset, (off_t)f_offset,
 129                                         (size_t)size, p->p_ucred, flags))
 130                         result = error = PAGER_ERROR;
 131
 132                 KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END,
 133                                       size, 1, 0, 0, 0);
 134
 135                 goto out;
 136         }
 137         ubc_create_upl(vp, f_offset, isize, &vpupl, &pl, UPL_FOR_PAGEOUT | UPL_COPYOUT_FROM | UPL_SET_LITE);
 138
 139         if (vpupl == (upl_t) 0) {
 140                 result = error = PAGER_ABSENT;
 141                 goto out;
 142         }
 143         /*
 144          * if we get here, we've created the upl and
 145          * are responsible for commiting/aborting it
 146          * regardless of what the caller has passed in
 147          */
 148         flags &= ~UPL_NOCOMMIT;
 149
 150         if (ubc_getsize(vp) == 0) {
 151                 for (offset = 0; isize; isize -= PAGE_SIZE,
 152                                         offset += PAGE_SIZE) {
 153                         blkno = ubc_offtoblk(vp, (off_t)f_offset);
 154                         f_offset += PAGE_SIZE;
 155                         if ((bp = incore(vp, blkno)) &&
 156                             ISSET(bp->b_flags, B_BUSY)) {
 157                                 ubc_upl_abort_range(vpupl, offset, PAGE_SIZE,
 158                                                     UPL_ABORT_FREE_ON_EMPTY);
 159                                 result = error = PAGER_ERROR;
 160                                 continue;
 161                         } else if (bp) {
 162                                 bremfree(bp);
 163                                 SET(bp->b_flags, B_BUSY | B_INVAL);
 164                                 brelse(bp);
 165                         }
 166                         ubc_upl_commit_range(vpupl, offset, PAGE_SIZE,
 167                                              UPL_COMMIT_FREE_ON_EMPTY);
 168                 }
 169                 goto out;
 170         }
 171         pg_index = 0;
 172         offset   = 0;
 173
 174         while (isize) {
 175                 int  xsize;
 176                 int  num_of_pages;
 177
 178                 if ( !upl_valid_page(pl, pg_index)) {
 179                         ubc_upl_abort_range(vpupl, offset, PAGE_SIZE,
 180                                             UPL_ABORT_FREE_ON_EMPTY);
 181                         offset += PAGE_SIZE;
 182                         isize  -= PAGE_SIZE;
 183                         pg_index++;
 184
 185                         continue;
 186                 }
 187                 if ( !upl_dirty_page(pl, pg_index)) {
 188                         /*
 189                          * if the page is not dirty and reached here it is
 190                          * marked precious or it is due to invalidation in
 191                          * memory_object_lock request as part of truncation
 192                          * We also get here from vm_object_terminate()
 193                          * So all you need to do in these
 194                          * cases is to invalidate incore buffer if it is there
 195                          * Note we must not sleep here if B_BUSY - that is
 196                          * a lock inversion which causes deadlock.
 197                          */
 198                         blkno = ubc_offtoblk(vp, (off_t)(f_offset + offset));
 199                         s = splbio();
 200                         vp_pgoclean++;
 201                         if (vp->v_tag == VT_NFS) {
 202                                 /* check with nfs if page is OK to drop */
 203                                 error = nfs_buf_page_inval(vp, (off_t)(f_offset + offset));
 204                                 splx(s);
 205                                 if (error) {
 206                                         ubc_upl_abort_range(vpupl, offset, PAGE_SIZE,
 207                                                             UPL_ABORT_FREE_ON_EMPTY);
 208                                         result = error = PAGER_ERROR;
 209                                         offset += PAGE_SIZE;
 210                                         isize -= PAGE_SIZE;
 211                                         pg_index++;
 212                                         continue;
 213                                 }
 214                         } else if ((bp = incore(vp, blkno)) &&
 215                             ISSET(bp->b_flags, B_BUSY | B_NEEDCOMMIT)) {
 216                                 splx(s);
 217                                 ubc_upl_abort_range(vpupl, offset, PAGE_SIZE,
 218                                                     UPL_ABORT_FREE_ON_EMPTY);
 219                                 result = error = PAGER_ERROR;
 220                                 offset += PAGE_SIZE;
 221                                 isize -= PAGE_SIZE;
 222                                 pg_index++;
 223                                 continue;
 224                         } else if (bp) {
 225                                 bremfree(bp);
 226                                 SET(bp->b_flags, B_BUSY | B_INVAL );
 227                                 splx(s);
 228                                 brelse(bp);
 229                         } else
 230                                 splx(s);
 231
 232                         ubc_upl_commit_range(vpupl, offset, PAGE_SIZE,
 233                                              UPL_COMMIT_FREE_ON_EMPTY);
 234                         offset += PAGE_SIZE;
 235                         isize  -= PAGE_SIZE;
 236                         pg_index++;
 237
 238                         continue;
 239                 }
 240                 vp_pgodirty++;
 241
 242                 num_of_pages = 1;
 243                 xsize = isize - PAGE_SIZE;
 244
 245                 while (xsize) {
 246                         if ( !upl_valid_page(pl, pg_index + num_of_pages))
 247                                 break;
 248                         if ( !upl_dirty_page(pl, pg_index + num_of_pages))
 249                                 break;
 250                         num_of_pages++;
 251                         xsize -= PAGE_SIZE;
 252                 }
 253                 xsize = num_of_pages * PAGE_SIZE;
 254
 255                 KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START,
 256                                       xsize, 0, 0, 0, 0);
 257
 258                 if (error = VOP_PAGEOUT(vp, vpupl, (vm_offset_t)offset,
 259                                         (off_t)(f_offset + offset), xsize,
 260                                         p->p_ucred, flags))
 261                         result = error = PAGER_ERROR;
 262
 263                 KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END,
 264                                       xsize, 0, 0, 0, 0);
 265
 266                 offset += xsize;
 267                 isize  -= xsize;
 268                 pg_index += num_of_pages;
 269         }
 270 out:
 271         if (errorp)
 272                 *errorp = result;
 273
 274         thread_funnel_set(kernel_flock, funnel_state);
 275
 276         return (error);
 277 }
 278
 279
 280 pager_return_t
 281 vnode_pagein(
 282         struct vnode            *vp,
 283         upl_t                   upl,
 284         vm_offset_t             upl_offset,
 285         vm_object_offset_t      f_offset,
 286         vm_size_t               size,
 287         int                     flags,
 288         int                     *errorp)
 289 {
 290         struct proc     *p = current_proc();
 291         upl_page_info_t *pl;
 292         int             result = PAGER_SUCCESS;
 293         int             error = 0;
 294         int             xfer_size;
 295         int             pages_in_upl;
 296         int             start_pg;
 297         int             last_pg;
 298         int             first_pg;
 299         int             xsize;
 300         int             abort_needed = 1;
 301         boolean_t       funnel_state;
 302
 303
 304         funnel_state = thread_funnel_set(kernel_flock, TRUE);
 305
 306         UBCINFOCHECK("vnode_pagein", vp);
 307
 308         if (UBCINVALID(vp)) {
 309                 result = PAGER_ERROR;
 310                 error  = PAGER_ERROR;
 311                 if (upl && !(flags & UPL_NOCOMMIT)) {
 312                         ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);
 313                 }
 314                 goto out;
 315         }
 316         if (upl == (upl_t)NULL) {
 317                 if (size > (MAX_UPL_TRANSFER * PAGE_SIZE)) {
 318                         result = PAGER_ERROR;
 319                         error  = PAGER_ERROR;
 320                         goto out;
 321                 }
 322                 ubc_create_upl(vp, f_offset, size, &upl, &pl, UPL_RET_ONLY_ABSENT | UPL_SET_LITE);
 323
 324                 if (upl == (upl_t)NULL) {
 325                         result =  PAGER_ABSENT;
 326                         error = PAGER_ABSENT;
 327                         goto out;
 328                 }
 329                 upl_offset = 0;
 330                 /*
 331                  * if we get here, we've created the upl and
 332                  * are responsible for commiting/aborting it
 333                  * regardless of what the caller has passed in
 334                  */
 335                 flags &= ~UPL_NOCOMMIT;
 336
 337                 vp_pagein++;
 338         } else {
 339                 pl = ubc_upl_pageinfo(upl);
 340
 341                 dp_pgins++;
 342         }
 343         pages_in_upl = size / PAGE_SIZE;
 344         first_pg     = upl_offset / PAGE_SIZE;
 345
 346         /*
 347          * before we start marching forward, we must make sure we end on
 348          * a present page, otherwise we will be working with a freed
 349          * upl
 350          */
 351         for (last_pg = pages_in_upl - 1; last_pg >= first_pg; last_pg--) {
 352                 if (upl_page_present(pl, last_pg))
 353                         break;
 354         }
 355         pages_in_upl = last_pg + 1;
 356
 357         for (last_pg = first_pg; last_pg < pages_in_upl;) {
 358                 /*
 359                  * scan the upl looking for the next
 360                  * page that is present.... if all of the
 361                  * pages are absent, we're done
 362                  */
 363                 for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
 364                         if (upl_page_present(pl, last_pg))
 365                                 break;
 366                 }
 367                 if (last_pg == pages_in_upl)
 368                         break;
 369
 370                 /*
 371                  * if we get here, we've sitting on a page
 372                  * that is present... we want to skip over
 373                  * any range of 'valid' pages... if this takes
 374                  * us to the end of the request, than we're done
 375                  */
 376                 for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
 377                         if (!upl_valid_page(pl, last_pg) || !upl_page_present(pl, last_pg))
 378                                 break;
 379                 }
 380                 if (last_pg > start_pg) {
 381                         /*
 382                          * we've found a range of valid pages
 383                          * if we've got COMMIT responsibility
 384                          * commit this range of pages back to the
 385                          * cache unchanged
 386                          */
 387                         xsize = (last_pg - start_pg) * PAGE_SIZE;
 388
 389                         if (!(flags & UPL_NOCOMMIT))
 390                                 ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, xsize, UPL_ABORT_FREE_ON_EMPTY);
 391
 392                         abort_needed = 0;
 393                 }
 394                 if (last_pg == pages_in_upl)
 395                         break;
 396
 397                 if (!upl_page_present(pl, last_pg))
 398                         /*
 399                          * if we found a range of valid pages
 400                          * terminated by a non-present page
 401                          * than start over
 402                          */
 403                         continue;
 404
 405                 /*
 406                  * scan from the found invalid page looking for a valid
 407                  * or non-present page before the end of the upl is reached, if we
 408                  * find one, then it will be the last page of the request to
 409                  * 'cluster_io'
 410                  */
 411                 for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
 412                         if (upl_valid_page(pl, last_pg) || !upl_page_present(pl, last_pg))
 413                                 break;
 414                 }
 415                 if (last_pg > start_pg) {
 416                         int xoff;
 417
 418                         xsize = (last_pg - start_pg) * PAGE_SIZE;
 419                         xoff  = start_pg * PAGE_SIZE;
 420
 421                         if (error = VOP_PAGEIN(vp, upl, (vm_offset_t) xoff,
 422                                                (off_t)f_offset + xoff,
 423                                                xsize, p->p_ucred,
 424                                                flags)) {
 425                                 result = PAGER_ERROR;
 426                                 error  = PAGER_ERROR;
 427
 428                         }
 429                         abort_needed = 0;
 430                 }
 431         }
 432         if (!(flags & UPL_NOCOMMIT) && abort_needed)
 433                 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
 434 out:
 435         if (errorp)
 436                 *errorp = result;
 437         thread_funnel_set(kernel_flock, funnel_state);
 438
 439         return (error);
 440 }
 441
 442 void
 443 vnode_pager_shutdown()
 444 {
 445         int i;
 446         extern struct bs_map  bs_port_table[];
 447         struct vnode *vp;
 448
 449         for(i = 0; i < MAX_BACKING_STORE; i++) {
 450                 vp = (struct vnode *)(bs_port_table[i]).vp;
 451                 if (vp) {
 452                         (bs_port_table[i]).vp = 0;
 453                         ubc_rele(vp);
 454                         /* get rid of macx_swapon() namei() reference */
 455                         vrele(vp);
 456
 457                         /* get rid of macx_swapon() "extra" reference */
 458                         vrele(vp);
 459                 }
 460         }
 461 }
 462
 463
 464 void *
 465 upl_get_internal_page_list(upl_t upl)
 466 {
 467   return(UPL_GET_INTERNAL_PAGE_LIST(upl));
 468
 469 }