osfmk/vm/vm_page.h

   1 /*
   2  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_OSREFERENCE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License.  The rights granted to you under the
  10  * License may not be used to create, or enable the creation or
  11  * redistribution of, unlawful or unlicensed copies of an Apple operating
  12  * system, or to circumvent, violate, or enable the circumvention or
  13  * violation of, any terms of an Apple operating system software license
  14  * agreement.
  15  *
  16  * Please obtain a copy of the License at
  17  * http://www.opensource.apple.com/apsl/ and read it before using this
  18  * file.
  19  *
  20  * The Original Code and all software distributed under the License are
  21  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  22  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  23  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  24  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  25  * Please see the License for the specific language governing rights and
  26  * limitations under the License.
  27  *
  28  * @APPLE_LICENSE_OSREFERENCE_HEADER_END@
  29  */
  30 /*
  31  * @OSF_COPYRIGHT@
  32  */
  33 /*
  34  * Mach Operating System
  35  * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
  36  * All Rights Reserved.
  37  *
  38  * Permission to use, copy, modify and distribute this software and its
  39  * documentation is hereby granted, provided that both the copyright
  40  * notice and this permission notice appear in all copies of the
  41  * software, derivative works or modified versions, and any portions
  42  * thereof, and that both notices appear in supporting documentation.
  43  *
  44  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  45  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  46  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  47  *
  48  * Carnegie Mellon requests users of this software to return to
  49  *
  50  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  51  *  School of Computer Science
  52  *  Carnegie Mellon University
  53  *  Pittsburgh PA 15213-3890
  54  *
  55  * any improvements or extensions that they make and grant Carnegie Mellon
  56  * the rights to redistribute these changes.
  57  */
  58 /*
  59  */
  60 /*
  61  *      File:   vm/vm_page.h
  62  *      Author: Avadis Tevanian, Jr., Michael Wayne Young
  63  *      Date:   1985
  64  *
  65  *      Resident memory system definitions.
  66  */
  67
  68 #ifndef _VM_VM_PAGE_H_
  69 #define _VM_VM_PAGE_H_
  70
  71 #include <debug.h>
  72
  73 #include <mach/boolean.h>
  74 #include <mach/vm_prot.h>
  75 #include <mach/vm_param.h>
  76 #include <vm/vm_object.h>
  77 #include <kern/queue.h>
  78 #include <kern/lock.h>
  79
  80 #include <kern/macro_help.h>
  81
  82 /*
  83  * Each page entered on the inactive queue obtains a ticket from a
  84  * particular ticket roll.  Pages granted tickets from a particular
  85  * roll  generally flow through the queue as a group.  In this way when a
  86  * page with a ticket from a particular roll is pulled from the top of the
  87  * queue it is extremely likely that the pages near the top will have tickets
  88  * from the same or adjacent rolls.  In this way the proximity to the top
  89  * of the queue can be loosely ascertained by determining the identity of
  90  * the roll the pages ticket came from.
  91  */
  92
  93
  94 extern unsigned int     vm_page_ticket_roll;
  95 extern unsigned int     vm_page_ticket;
  96
  97
  98 #define VM_PAGE_TICKETS_IN_ROLL  512
  99 #define VM_PAGE_TICKET_ROLL_IDS  16
 100
 101 /*
 102  *      Management of resident (logical) pages.
 103  *
 104  *      A small structure is kept for each resident
 105  *      page, indexed by page number.  Each structure
 106  *      is an element of several lists:
 107  *
 108  *              A hash table bucket used to quickly
 109  *              perform object/offset lookups
 110  *
 111  *              A list of all pages for a given object,
 112  *              so they can be quickly deactivated at
 113  *              time of deallocation.
 114  *
 115  *              An ordered list of pages due for pageout.
 116  *
 117  *      In addition, the structure contains the object
 118  *      and offset to which this page belongs (for pageout),
 119  *      and sundry status bits.
 120  *
 121  *      Fields in this structure are locked either by the lock on the
 122  *      object that the page belongs to (O) or by the lock on the page
 123  *      queues (P).  [Some fields require that both locks be held to
 124  *      change that field; holding either lock is sufficient to read.]
 125  */
 126
 127 struct vm_page {
 128         queue_chain_t   pageq;          /* queue info for FIFO
 129                                          * queue or free list (P) */
 130         queue_chain_t   listq;          /* all pages in same object (O) */
 131         struct vm_page  *next;          /* VP bucket link (O) */
 132
 133         vm_object_t     object;         /* which object am I in (O&P) */
 134         vm_object_offset_t offset;      /* offset into that object (O,P) */
 135
 136         /*
 137          * The following word of flags is protected
 138          * by the "page queues" lock.
 139          */
 140         unsigned int    wire_count:16,  /* how many wired down maps use me? (O&P) */
 141                         page_ticket:4,  /* age of the page on the       */
 142                                         /* inactive queue.                  */
 143         /* boolean_t */ inactive:1,     /* page is in inactive list (P) */
 144                         active:1,       /* page is in active list (P) */
 145                         pageout_queue:1,/* page is on queue for pageout (P) */
 146                         laundry:1,      /* page is being cleaned now (P)*/
 147                         free:1,         /* page is on free list (P) */
 148                         reference:1,    /* page has been used (P) */
 149                         pageout:1,      /* page wired & busy for pageout (P) */
 150                         gobbled:1,      /* page used internally (P) */
 151                         private:1,      /* Page should not be returned to
 152                                          *  the free list (P) */
 153                         zero_fill:1,
 154                         :0;
 155
 156         /*
 157          * The following word of flags is protected
 158          * by the "VM object" lock.
 159          */
 160         unsigned int
 161                         page_error:8,   /* error from I/O operations */
 162         /* boolean_t */ busy:1,         /* page is in transit (O) */
 163                         wanted:1,       /* someone is waiting for page (O) */
 164                         tabled:1,       /* page is in VP table (O) */
 165                         fictitious:1,   /* Physical page doesn't exist (O) */
 166                         no_isync:1,     /* page has not been instruction synced */
 167                         absent:1,       /* Data has been requested, but is
 168                                          *  not yet available (O) */
 169                         error:1,        /* Data manager was unable to provide
 170                                          *  data due to error (O) */
 171                         dirty:1,        /* Page must be cleaned (O) */
 172                         cleaning:1,     /* Page clean has begun (O) */
 173                         precious:1,     /* Page is precious; data must be
 174                                          *  returned even if clean (O) */
 175                         clustered:1,    /* page is not the faulted page (O) */
 176                         overwriting:1,  /* Request to unlock has been made
 177                                          * without having data. (O)
 178                                          * [See vm_fault_page_overwrite] */
 179                         restart:1,      /* Page was pushed higher in shadow
 180                                            chain by copy_call-related pagers;
 181                                            start again at top of chain */
 182                         lock_supplied:1,/* protection supplied by pager (O) */
 183         /* vm_prot_t */ page_lock:3,    /* Uses prohibited by pager (O) */
 184         /* vm_prot_t */ unlock_request:3,/* Outstanding unlock request (O) */
 185                         unusual:1,      /* Page is absent, error, restart or
 186                                            page locked */
 187                         encrypted:1,    /* encrypted for secure swap (O) */
 188                         list_req_pending:1, /* pagein/pageout alt mechanism */
 189                                             /* allows creation of list      */
 190                                             /* requests on pages that are   */
 191                                             /* actively being paged.        */
 192                         dump_cleaning:1;   /* set by the pageout daemon when */
 193                                            /* a page being cleaned is       */
 194                                            /* encountered and targeted as   */
 195                                            /* a pageout candidate           */
 196         /* we've used up all 32 bits */
 197
 198         ppnum_t         phys_page;      /* Physical address of page, passed
 199                                          *  to pmap_enter (read-only) */
 200 };
 201
 202 #define DEBUG_ENCRYPTED_SWAP    1
 203 #if DEBUG_ENCRYPTED_SWAP
 204 #define ASSERT_PAGE_DECRYPTED(page)                                     \
 205         MACRO_BEGIN                                                     \
 206         if ((page)->encrypted) {                                        \
 207                 panic("VM page %p should not be encrypted here\n",      \
 208                       (page));                                          \
 209         }                                                               \
 210         MACRO_END
 211 #else   /* DEBUG_ENCRYPTED_SWAP */
 212 #define ASSERT_PAGE_DECRYPTED(page) assert(!(page)->encrypted)
 213 #endif  /* DEBUG_ENCRYPTED_SWAP */
 214
 215 typedef struct vm_page  *vm_page_t;
 216
 217 #define VM_PAGE_NULL            ((vm_page_t) 0)
 218 #define NEXT_PAGE(m)            ((vm_page_t) (m)->pageq.next)
 219 #define NEXT_PAGE_PTR(m)        ((vm_page_t *) &(m)->pageq.next)
 220
 221 /*
 222  * XXX  The unusual bit should not be necessary.  Most of the bit
 223  * XXX  fields above really want to be masks.
 224  */
 225
 226 /*
 227  *      For debugging, this macro can be defined to perform
 228  *      some useful check on a page structure.
 229  */
 230
 231 #define VM_PAGE_CHECK(mem)
 232
 233 /*
 234  *      Each pageable resident page falls into one of three lists:
 235  *
 236  *      free
 237  *              Available for allocation now.
 238  *      inactive
 239  *              Not referenced in any map, but still has an
 240  *              object/offset-page mapping, and may be dirty.
 241  *              This is the list of pages that should be
 242  *              paged out next.
 243  *      active
 244  *              A list of pages which have been placed in
 245  *              at least one physical map.  This list is
 246  *              ordered, in LRU-like fashion.
 247  */
 248
 249 extern
 250 vm_page_t       vm_page_queue_free;     /* memory free queue */
 251 extern
 252 vm_page_t       vm_page_queue_fictitious;       /* fictitious free queue */
 253 extern
 254 queue_head_t    vm_page_queue_active;   /* active memory queue */
 255 extern
 256 queue_head_t    vm_page_queue_inactive; /* inactive memory queue */
 257 queue_head_t    vm_page_queue_zf;       /* inactive memory queue for zero fill */
 258
 259 extern
 260 vm_offset_t     first_phys_addr;        /* physical address for first_page */
 261 extern
 262 vm_offset_t     last_phys_addr;         /* physical address for last_page */
 263
 264 extern
 265 unsigned int    vm_page_free_count;     /* How many pages are free? */
 266 extern
 267 unsigned int    vm_page_fictitious_count;/* How many fictitious pages are free? */
 268 extern
 269 unsigned int    vm_page_active_count;   /* How many pages are active? */
 270 extern
 271 unsigned int    vm_page_inactive_count; /* How many pages are inactive? */
 272 extern
 273 unsigned int    vm_page_wire_count;     /* How many pages are wired? */
 274 extern
 275 unsigned int    vm_page_free_target;    /* How many do we want free? */
 276 extern
 277 unsigned int    vm_page_free_min;       /* When to wakeup pageout */
 278 extern
 279 unsigned int    vm_page_inactive_target;/* How many do we want inactive? */
 280 extern
 281 unsigned int    vm_page_free_reserved;  /* How many pages reserved to do pageout */
 282 extern
 283 unsigned int    vm_page_throttled_count;/* Count of zero-fill allocations throttled */
 284 extern
 285 unsigned int    vm_page_gobble_count;
 286
 287 extern
 288 unsigned int    vm_page_purgeable_count;/* How many pages are purgeable now ? */
 289 extern
 290 uint64_t        vm_page_purged_count;   /* How many pages got purged so far ? */
 291
 292 decl_mutex_data(,vm_page_queue_lock)
 293                                 /* lock on active and inactive page queues */
 294 decl_mutex_data(,vm_page_queue_free_lock)
 295                                 /* lock on free page queue */
 296
 297 extern unsigned int     vm_page_free_wanted;
 298                                 /* how many threads are waiting for memory */
 299
 300 extern vm_offset_t      vm_page_fictitious_addr;
 301                                 /* (fake) phys_addr of fictitious pages */
 302
 303 extern boolean_t        vm_page_deactivate_hint;
 304
 305 // 0 = all pages avail, 1 = disable high mem, 2 = prefer himem
 306 extern int              vm_himemory_mode;
 307
 308 extern ppnum_t          vm_lopage_poolend;
 309 extern int              vm_lopage_poolsize;
 310 extern uint64_t         max_valid_dma_address;
 311
 312
 313 /*
 314  * Prototypes for functions exported by this module.
 315  */
 316 extern void             vm_page_bootstrap(
 317                                         vm_offset_t     *startp,
 318                                         vm_offset_t     *endp);
 319
 320 extern void             vm_page_module_init(void);
 321
 322 extern void             vm_page_create(
 323                                         ppnum_t         start,
 324                                         ppnum_t         end);
 325
 326 extern vm_page_t        vm_page_lookup(
 327                                         vm_object_t             object,
 328                                         vm_object_offset_t      offset);
 329
 330 extern vm_page_t        vm_page_grab_fictitious(void);
 331
 332 extern void             vm_page_release_fictitious(
 333                                         vm_page_t page);
 334
 335 extern boolean_t        vm_page_convert(
 336                                         vm_page_t       page);
 337
 338 extern void             vm_page_more_fictitious(void);
 339
 340 extern int              vm_pool_low(void);
 341
 342 extern vm_page_t        vm_page_grab(void);
 343
 344 extern vm_page_t        vm_page_grablo(void);
 345
 346 extern void             vm_page_release(
 347                                         vm_page_t       page);
 348
 349 extern boolean_t        vm_page_wait(
 350                                         int             interruptible );
 351
 352 extern vm_page_t        vm_page_alloc(
 353                                         vm_object_t             object,
 354                                         vm_object_offset_t      offset);
 355
 356 extern vm_page_t        vm_page_alloclo(
 357                                         vm_object_t             object,
 358                                         vm_object_offset_t      offset);
 359
 360 extern void             vm_page_init(
 361                                         vm_page_t       page,
 362                                         ppnum_t         phys_page);
 363
 364 extern void             vm_page_free(
 365                                         vm_page_t       page);
 366
 367 extern void             vm_page_activate(
 368                                         vm_page_t       page);
 369
 370 extern void             vm_page_deactivate(
 371                                         vm_page_t       page);
 372
 373 extern void             vm_page_rename(
 374                                         vm_page_t               page,
 375                                         vm_object_t             new_object,
 376                                         vm_object_offset_t      new_offset);
 377
 378 extern void             vm_page_insert(
 379                                         vm_page_t               page,
 380                                         vm_object_t             object,
 381                                         vm_object_offset_t      offset);
 382
 383 extern void             vm_page_replace(
 384                                         vm_page_t               mem,
 385                                         vm_object_t             object,
 386                                         vm_object_offset_t      offset);
 387
 388 extern void             vm_page_remove(
 389                                         vm_page_t       page);
 390
 391 extern void             vm_page_zero_fill(
 392                                         vm_page_t       page);
 393
 394 extern void             vm_page_part_zero_fill(
 395                                         vm_page_t       m,
 396                                         vm_offset_t     m_pa,
 397                                         vm_size_t       len);
 398
 399 extern void             vm_page_copy(
 400                                         vm_page_t       src_page,
 401                                         vm_page_t       dest_page);
 402
 403 extern void             vm_page_part_copy(
 404                                         vm_page_t       src_m,
 405                                         vm_offset_t     src_pa,
 406                                         vm_page_t       dst_m,
 407                                         vm_offset_t     dst_pa,
 408                                         vm_size_t       len);
 409
 410 extern void             vm_page_wire(
 411                                         vm_page_t       page);
 412
 413 extern void             vm_page_unwire(
 414                                         vm_page_t       page);
 415
 416 extern void             vm_set_page_size(void);
 417
 418 extern void             vm_page_gobble(
 419                                         vm_page_t      page);
 420
 421 /*
 422  *      Functions implemented as macros. m->wanted and m->busy are
 423  *      protected by the object lock.
 424  */
 425
 426 #define PAGE_ASSERT_WAIT(m, interruptible)                      \
 427                 (((m)->wanted = TRUE),                          \
 428                  assert_wait((event_t) (m), (interruptible)))
 429
 430 #define PAGE_SLEEP(o, m, interruptible)                         \
 431                 (((m)->wanted = TRUE),                          \
 432                  thread_sleep_vm_object((o), (m), (interruptible)))
 433
 434 #define PAGE_WAKEUP_DONE(m)                                     \
 435                 MACRO_BEGIN                                     \
 436                 (m)->busy = FALSE;                              \
 437                 if ((m)->wanted) {                              \
 438                         (m)->wanted = FALSE;                    \
 439                         thread_wakeup((event_t) (m));           \
 440                 }                                               \
 441                 MACRO_END
 442
 443 #define PAGE_WAKEUP(m)                                          \
 444                 MACRO_BEGIN                                     \
 445                 if ((m)->wanted) {                              \
 446                         (m)->wanted = FALSE;                    \
 447                         thread_wakeup((event_t) (m));           \
 448                 }                                               \
 449                 MACRO_END
 450
 451 #define VM_PAGE_FREE(p)                         \
 452                 MACRO_BEGIN                     \
 453                 vm_page_lock_queues();          \
 454                 vm_page_free(p);                \
 455                 vm_page_unlock_queues();        \
 456                 MACRO_END
 457
 458 #define VM_PAGE_GRAB_FICTITIOUS(M)                                      \
 459                 MACRO_BEGIN                                             \
 460                 while ((M = vm_page_grab_fictitious()) == VM_PAGE_NULL) \
 461                         vm_page_more_fictitious();                      \
 462                 MACRO_END
 463
 464 #define VM_PAGE_THROTTLED()                                             \
 465                 (vm_page_free_count < vm_page_free_min &&               \
 466                  !(current_thread()->options & TH_OPT_VMPRIV) &&                        \
 467                  ++vm_page_throttled_count)
 468
 469 #define VM_PAGE_WAIT()          ((void)vm_page_wait(THREAD_UNINT))
 470
 471 #define vm_page_lock_queues()   mutex_lock(&vm_page_queue_lock)
 472 #define vm_page_unlock_queues() mutex_unlock(&vm_page_queue_lock)
 473
 474 #define VM_PAGE_QUEUES_REMOVE(mem)                              \
 475         MACRO_BEGIN                                             \
 476         assert(!mem->laundry);                                  \
 477         if (mem->active) {                                      \
 478                 assert(mem->object != kernel_object);           \
 479                 assert(!mem->inactive);                         \
 480                 queue_remove(&vm_page_queue_active,             \
 481                         mem, vm_page_t, pageq);                 \
 482                 mem->pageq.next = NULL;                         \
 483                 mem->pageq.prev = NULL;                         \
 484                 mem->active = FALSE;                            \
 485                 if (!mem->fictitious)                           \
 486                         vm_page_active_count--;                 \
 487         }                                                       \
 488                                                                 \
 489         if (mem->inactive) {                                    \
 490                 assert(mem->object != kernel_object);           \
 491                 assert(!mem->active);                           \
 492                 if (mem->zero_fill) {                           \
 493                         queue_remove(&vm_page_queue_zf,         \
 494                         mem, vm_page_t, pageq);                 \
 495                 } else {                                        \
 496                         queue_remove(&vm_page_queue_inactive,   \
 497                         mem, vm_page_t, pageq);                 \
 498                 }                                               \
 499                 mem->pageq.next = NULL;                         \
 500                 mem->pageq.prev = NULL;                         \
 501                 mem->inactive = FALSE;                          \
 502                 if (!mem->fictitious)                           \
 503                         vm_page_inactive_count--;               \
 504         }                                                       \
 505         MACRO_END
 506
 507 #endif  /* _VM_VM_PAGE_H_ */