osfmk/vm/vm_page.h

   1 /*
   2  * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  */
  31 /*
  32  * Mach Operating System
  33  * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
  34  * All Rights Reserved.
  35  *
  36  * Permission to use, copy, modify and distribute this software and its
  37  * documentation is hereby granted, provided that both the copyright
  38  * notice and this permission notice appear in all copies of the
  39  * software, derivative works or modified versions, and any portions
  40  * thereof, and that both notices appear in supporting documentation.
  41  *
  42  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  43  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  44  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  45  *
  46  * Carnegie Mellon requests users of this software to return to
  47  *
  48  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  49  *  School of Computer Science
  50  *  Carnegie Mellon University
  51  *  Pittsburgh PA 15213-3890
  52  *
  53  * any improvements or extensions that they make and grant Carnegie Mellon
  54  * the rights to redistribute these changes.
  55  */
  56 /*
  57  */
  58 /*
  59  *      File:   vm/vm_page.h
  60  *      Author: Avadis Tevanian, Jr., Michael Wayne Young
  61  *      Date:   1985
  62  *
  63  *      Resident memory system definitions.
  64  */
  65
  66 #ifndef _VM_VM_PAGE_H_
  67 #define _VM_VM_PAGE_H_
  68
  69 #include <debug.h>
  70
  71 #include <mach/boolean.h>
  72 #include <mach/vm_prot.h>
  73 #include <mach/vm_param.h>
  74 #include <vm/vm_object.h>
  75 #include <kern/queue.h>
  76 #include <kern/lock.h>
  77
  78 #include <kern/macro_help.h>
  79 #include <libkern/OSAtomic.h>
  80
  81
  82 /*
  83  * VM_PAGE_MIN_SPECULATIVE_AGE_Q through VM_PAGE_MAX_SPECULATIVE_AGE_Q
  84  * represents a set of aging bins that are 'protected'...
  85  *
  86  * VM_PAGE_SPECULATIVE_AGED_Q is a list of the speculative pages that have
  87  * not yet been 'claimed' but have been aged out of the protective bins
  88  * this occurs in vm_page_speculate when it advances to the next bin
  89  * and discovers that it is still occupied... at that point, all of the
  90  * pages in that bin are moved to the VM_PAGE_SPECULATIVE_AGED_Q.  the pages
  91  * in that bin are all guaranteed to have reached at least the maximum age
  92  * we allow for a protected page... they can be older if there is no
  93  * memory pressure to pull them from the bin, or there are no new speculative pages
  94  * being generated to push them out.
  95  * this list is the one that vm_pageout_scan will prefer when looking
  96  * for pages to move to the underweight free list
  97  *
  98  * VM_PAGE_MAX_SPECULATIVE_AGE_Q * VM_PAGE_SPECULATIVE_Q_AGE_MS
  99  * defines the amount of time a speculative page is normally
 100  * allowed to live in the 'protected' state (i.e. not available
 101  * to be stolen if vm_pageout_scan is running and looking for
 102  * pages)...  however, if the total number of speculative pages
 103  * in the protected state exceeds our limit (defined in vm_pageout.c)
 104  * and there are none available in VM_PAGE_SPECULATIVE_AGED_Q, then
 105  * vm_pageout_scan is allowed to steal pages from the protected
 106  * bucket even if they are underage.
 107  *
 108  * vm_pageout_scan is also allowed to pull pages from a protected
 109  * bin if the bin has reached the "age of consent" we've set
 110  */
 111 #define VM_PAGE_MAX_SPECULATIVE_AGE_Q   10
 112 #define VM_PAGE_MIN_SPECULATIVE_AGE_Q   1
 113 #define VM_PAGE_SPECULATIVE_AGED_Q      0
 114
 115 #define VM_PAGE_SPECULATIVE_Q_AGE_MS    500
 116
 117 struct vm_speculative_age_q {
 118         /*
 119          * memory queue for speculative pages via clustered pageins
 120          */
 121         queue_head_t    age_q;
 122         mach_timespec_t age_ts;
 123 };
 124
 125
 126
 127 extern
 128 struct vm_speculative_age_q     vm_page_queue_speculative[];
 129
 130 extern int                      speculative_steal_index;
 131 extern int                      speculative_age_index;
 132 extern unsigned int             vm_page_speculative_q_age_ms;
 133
 134
 135 /*
 136  *      Management of resident (logical) pages.
 137  *
 138  *      A small structure is kept for each resident
 139  *      page, indexed by page number.  Each structure
 140  *      is an element of several lists:
 141  *
 142  *              A hash table bucket used to quickly
 143  *              perform object/offset lookups
 144  *
 145  *              A list of all pages for a given object,
 146  *              so they can be quickly deactivated at
 147  *              time of deallocation.
 148  *
 149  *              An ordered list of pages due for pageout.
 150  *
 151  *      In addition, the structure contains the object
 152  *      and offset to which this page belongs (for pageout),
 153  *      and sundry status bits.
 154  *
 155  *      Fields in this structure are locked either by the lock on the
 156  *      object that the page belongs to (O) or by the lock on the page
 157  *      queues (P).  [Some fields require that both locks be held to
 158  *      change that field; holding either lock is sufficient to read.]
 159  */
 160
 161 struct vm_page {
 162         queue_chain_t   pageq;          /* queue info for FIFO */
 163                                         /* queue or free list (P) */
 164
 165         queue_chain_t   listq;          /* all pages in same object (O) */
 166         struct vm_page  *next;          /* VP bucket link (O) */
 167
 168         vm_object_t     object;         /* which object am I in (O&P) */
 169         vm_object_offset_t offset;      /* offset into that object (O,P) */
 170
 171         /*
 172          * The following word of flags is protected
 173          * by the "page queues" lock.
 174          *
 175          * we use the 'wire_count' field to store the local
 176          * queue id if local queues are enabled...
 177          * see the comments at 'VM_PAGE_QUEUES_REMOVE' as to
 178          * why this is safe to do
 179          */
 180 #define local_id wire_count
 181         unsigned int    wire_count:16,  /* how many wired down maps use me? (O&P) */
 182         /* boolean_t */ active:1,       /* page is in active list (P) */
 183                         inactive:1,     /* page is in inactive list (P) */
 184                         clean_queue:1,  /* page is in pre-cleaned list (P) */
 185                         local:1,        /* page is in one of the local queues (P) */
 186                         speculative:1,  /* page is in speculative list (P) */
 187                         throttled:1,    /* pager is not responding (P) */
 188                         free:1,         /* page is on free list (P) */
 189                         pageout_queue:1,/* page is on queue for pageout (P) */
 190                         laundry:1,      /* page is being cleaned now (P)*/
 191                         reference:1,    /* page has been used (P) */
 192                         gobbled:1,      /* page used internally (P) */
 193                         private:1,      /* Page should not be returned to
 194                                          *  the free list (P) */
 195                         no_cache:1,     /* page is not to be cached and should
 196                                          * be reused ahead of other pages (P) */
 197                         __unused_pageq_bits:3;  /* 3 bits available here */
 198
 199         ppnum_t         phys_page;      /* Physical address of page, passed
 200                                          *  to pmap_enter (read-only) */
 201
 202         /*
 203          * The following word of flags is protected
 204          * by the "VM object" lock.
 205          */
 206         unsigned int
 207         /* boolean_t */ busy:1,         /* page is in transit (O) */
 208                         wanted:1,       /* someone is waiting for page (O) */
 209                         tabled:1,       /* page is in VP table (O) */
 210                         fictitious:1,   /* Physical page doesn't exist (O) */
 211         /*
 212          * IMPORTANT: the "pmapped" bit can be turned on while holding the
 213          * VM object "shared" lock.  See vm_fault_enter().
 214          * This is OK as long as it's the only bit in this bit field that
 215          * can be updated without holding the VM object "exclusive" lock.
 216          */
 217                         pmapped:1,      /* page has been entered at some
 218                                          * point into a pmap (O **shared**) */
 219                         wpmapped:1,     /* page has been entered at some
 220                                          * point into a pmap for write (O) */
 221                         pageout:1,      /* page wired & busy for pageout (O) */
 222                         absent:1,       /* Data has been requested, but is
 223                                          *  not yet available (O) */
 224                         error:1,        /* Data manager was unable to provide
 225                                          *  data due to error (O) */
 226                         dirty:1,        /* Page must be cleaned (O) */
 227                         cleaning:1,     /* Page clean has begun (O) */
 228                         precious:1,     /* Page is precious; data must be
 229                                          *  returned even if clean (O) */
 230                         clustered:1,    /* page is not the faulted page (O) */
 231                         overwriting:1,  /* Request to unlock has been made
 232                                          * without having data. (O)
 233                                          * [See vm_fault_page_overwrite] */
 234                         restart:1,      /* Page was pushed higher in shadow
 235                                            chain by copy_call-related pagers;
 236                                            start again at top of chain */
 237                         unusual:1,      /* Page is absent, error, restart or
 238                                            page locked */
 239                         encrypted:1,    /* encrypted for secure swap (O) */
 240                         encrypted_cleaning:1,   /* encrypting page */
 241                         cs_validated:1,    /* code-signing: page was checked */
 242                         cs_tainted:1,      /* code-signing: page is tainted */
 243                         reusable:1,
 244                         lopage:1,
 245                         slid:1,
 246                         was_dirty:1,    /* was this page previously dirty? */
 247                         __unused_object_bits:8;  /* 8 bits available here */
 248
 249 #if __LP64__
 250         unsigned int __unused_padding;  /* Pad structure explicitly
 251                                         * to 8-byte multiple for LP64 */
 252 #endif
 253 };
 254
 255 #define DEBUG_ENCRYPTED_SWAP    1
 256 #if DEBUG_ENCRYPTED_SWAP
 257 #define ASSERT_PAGE_DECRYPTED(page)                                     \
 258         MACRO_BEGIN                                                     \
 259         if ((page)->encrypted) {                                        \
 260                 panic("VM page %p should not be encrypted here\n",      \
 261                       (page));                                          \
 262         }                                                               \
 263         MACRO_END
 264 #else   /* DEBUG_ENCRYPTED_SWAP */
 265 #define ASSERT_PAGE_DECRYPTED(page) assert(!(page)->encrypted)
 266 #endif  /* DEBUG_ENCRYPTED_SWAP */
 267
 268 typedef struct vm_page  *vm_page_t;
 269
 270
 271 typedef struct vm_locks_array {
 272         char    pad  __attribute__ ((aligned (64)));
 273         lck_mtx_t       vm_page_queue_lock2 __attribute__ ((aligned (64)));
 274         lck_mtx_t       vm_page_queue_free_lock2 __attribute__ ((aligned (64)));
 275         char    pad2  __attribute__ ((aligned (64)));
 276 } vm_locks_array_t;
 277
 278
 279 #define VM_PAGE_WIRED(m)        ((!(m)->local && (m)->wire_count))
 280 #define VM_PAGE_NULL            ((vm_page_t) 0)
 281 #define NEXT_PAGE(m)            ((vm_page_t) (m)->pageq.next)
 282 #define NEXT_PAGE_PTR(m)        ((vm_page_t *) &(m)->pageq.next)
 283
 284 /*
 285  * XXX  The unusual bit should not be necessary.  Most of the bit
 286  * XXX  fields above really want to be masks.
 287  */
 288
 289 /*
 290  *      For debugging, this macro can be defined to perform
 291  *      some useful check on a page structure.
 292  */
 293
 294 #define VM_PAGE_CHECK(mem)                      \
 295         MACRO_BEGIN                             \
 296         VM_PAGE_QUEUES_ASSERT(mem, 1);          \
 297         MACRO_END
 298
 299 /*     Page coloring:
 300  *
 301  *     The free page list is actually n lists, one per color,
 302  *     where the number of colors is a function of the machine's
 303  *     cache geometry set at system initialization.  To disable
 304  *     coloring, set vm_colors to 1 and vm_color_mask to 0.
 305  *     The boot-arg "colors" may be used to override vm_colors.
 306  *     Note that there is little harm in having more colors than needed.
 307  */
 308
 309 #define MAX_COLORS      128
 310 #define DEFAULT_COLORS  32
 311
 312 extern
 313 unsigned int    vm_colors;              /* must be in range 1..MAX_COLORS */
 314 extern
 315 unsigned int    vm_color_mask;          /* must be (vm_colors-1) */
 316 extern
 317 unsigned int    vm_cache_geometry_colors; /* optimal #colors based on cache geometry */
 318
 319 /*
 320  * Wired memory is a very limited resource and we can't let users exhaust it
 321  * and deadlock the entire system.  We enforce the following limits:
 322  *
 323  * vm_user_wire_limit (default: all memory minus vm_global_no_user_wire_amount)
 324  *      how much memory can be user-wired in one user task
 325  *
 326  * vm_global_user_wire_limit (default: same as vm_user_wire_limit)
 327  *      how much memory can be user-wired in all user tasks
 328  *
 329  * vm_global_no_user_wire_amount (default: VM_NOT_USER_WIREABLE)
 330  *      how much memory must remain user-unwired at any time
 331  */
 332 #define VM_NOT_USER_WIREABLE (64*1024*1024)     /* 64MB */
 333 extern
 334 vm_map_size_t   vm_user_wire_limit;
 335 extern
 336 vm_map_size_t   vm_global_user_wire_limit;
 337 extern
 338 vm_map_size_t   vm_global_no_user_wire_amount;
 339
 340 /*
 341  *      Each pageable resident page falls into one of three lists:
 342  *
 343  *      free
 344  *              Available for allocation now.  The free list is
 345  *              actually an array of lists, one per color.
 346  *      inactive
 347  *              Not referenced in any map, but still has an
 348  *              object/offset-page mapping, and may be dirty.
 349  *              This is the list of pages that should be
 350  *              paged out next.  There are actually two
 351  *              inactive lists, one for pages brought in from
 352  *              disk or other backing store, and another
 353  *              for "zero-filled" pages.  See vm_pageout_scan()
 354  *              for the distinction and usage.
 355  *      active
 356  *              A list of pages which have been placed in
 357  *              at least one physical map.  This list is
 358  *              ordered, in LRU-like fashion.
 359  */
 360
 361
 362 #define VPL_LOCK_SPIN 1
 363
 364 struct vpl {
 365         unsigned int    vpl_count;
 366         queue_head_t    vpl_queue;
 367 #ifdef  VPL_LOCK_SPIN
 368         lck_spin_t      vpl_lock;
 369 #else
 370         lck_mtx_t       vpl_lock;
 371         lck_mtx_ext_t   vpl_lock_ext;
 372 #endif
 373 };
 374
 375 struct  vplq {
 376         union {
 377                 char   cache_line_pad[128];
 378                 struct vpl vpl;
 379         } vpl_un;
 380 };
 381 extern
 382 unsigned int    vm_page_local_q_count;
 383 extern
 384 struct vplq     *vm_page_local_q;
 385 extern
 386 unsigned int    vm_page_local_q_soft_limit;
 387 extern
 388 unsigned int    vm_page_local_q_hard_limit;
 389 extern
 390 vm_locks_array_t vm_page_locks;
 391
 392 extern
 393 queue_head_t    vm_page_queue_free[MAX_COLORS]; /* memory free queue */
 394 extern
 395 queue_head_t    vm_lopage_queue_free;           /* low memory free queue */
 396 extern
 397 queue_head_t    vm_page_queue_active;   /* active memory queue */
 398 extern
 399 queue_head_t    vm_page_queue_inactive; /* inactive memory queue for normal pages */
 400 extern
 401 queue_head_t    vm_page_queue_cleaned; /* clean-queue inactive memory */
 402 extern
 403 queue_head_t    vm_page_queue_anonymous;        /* inactive memory queue for anonymous pages */
 404 extern
 405 queue_head_t    vm_page_queue_throttled;        /* memory queue for throttled pageout pages */
 406
 407 extern
 408 vm_offset_t     first_phys_addr;        /* physical address for first_page */
 409 extern
 410 vm_offset_t     last_phys_addr;         /* physical address for last_page */
 411
 412 extern
 413 unsigned int    vm_page_free_count;     /* How many pages are free? (sum of all colors) */
 414 extern
 415 unsigned int    vm_page_fictitious_count;/* How many fictitious pages are free? */
 416 extern
 417 unsigned int    vm_page_active_count;   /* How many pages are active? */
 418 extern
 419 unsigned int    vm_page_inactive_count; /* How many pages are inactive? */
 420 extern
 421 unsigned int    vm_page_cleaned_count; /* How many pages are in the clean queue? */
 422 extern
 423 unsigned int    vm_page_throttled_count;/* How many inactives are throttled */
 424 extern
 425 unsigned int    vm_page_speculative_count;      /* How many speculative pages are unclaimed? */
 426 extern
 427 unsigned int    vm_page_wire_count;             /* How many pages are wired? */
 428 extern
 429 unsigned int    vm_page_wire_count_initial;     /* How many pages wired at startup */
 430 extern
 431 unsigned int    vm_page_free_target;    /* How many do we want free? */
 432 extern
 433 unsigned int    vm_page_free_min;       /* When to wakeup pageout */
 434 extern
 435 unsigned int    vm_page_throttle_limit; /* When to throttle new page creation */
 436 extern
 437 uint32_t        vm_page_creation_throttle;      /* When to throttle new page creation */
 438 extern
 439 unsigned int    vm_page_inactive_target;/* How many do we want inactive? */
 440 extern
 441 unsigned int    vm_page_anonymous_min;  /* When it's ok to pre-clean */
 442 extern
 443 unsigned int    vm_page_inactive_min;   /* When do wakeup pageout */
 444 extern
 445 unsigned int    vm_page_free_reserved;  /* How many pages reserved to do pageout */
 446 extern
 447 unsigned int    vm_page_throttle_count; /* Count of page allocations throttled */
 448 extern
 449 unsigned int    vm_page_gobble_count;
 450
 451 #if DEVELOPMENT || DEBUG
 452 extern
 453 unsigned int    vm_page_speculative_used;
 454 #endif
 455
 456 extern
 457 unsigned int    vm_page_purgeable_count;/* How many pages are purgeable now ? */
 458 extern
 459 unsigned int    vm_page_purgeable_wired_count;/* How many purgeable pages are wired now ? */
 460 extern
 461 uint64_t        vm_page_purged_count;   /* How many pages got purged so far ? */
 462
 463 extern unsigned int     vm_page_free_wanted;
 464                                 /* how many threads are waiting for memory */
 465
 466 extern unsigned int     vm_page_free_wanted_privileged;
 467                                 /* how many VM privileged threads are waiting for memory */
 468
 469 extern ppnum_t  vm_page_fictitious_addr;
 470                                 /* (fake) phys_addr of fictitious pages */
 471
 472 extern ppnum_t  vm_page_guard_addr;
 473                                 /* (fake) phys_addr of guard pages */
 474
 475
 476 extern boolean_t        vm_page_deactivate_hint;
 477
 478 /*
 479    0 = all pages avail ( default. )
 480    1 = disable high mem ( cap max pages to 4G)
 481    2 = prefer himem
 482 */
 483 extern int              vm_himemory_mode;
 484
 485 extern boolean_t        vm_lopage_needed;
 486 extern uint32_t         vm_lopage_free_count;
 487 extern uint32_t         vm_lopage_free_limit;
 488 extern uint32_t         vm_lopage_lowater;
 489 extern boolean_t        vm_lopage_refill;
 490 extern uint64_t         max_valid_dma_address;
 491 extern ppnum_t          max_valid_low_ppnum;
 492
 493 /*
 494  * Prototypes for functions exported by this module.
 495  */
 496 extern void             vm_page_bootstrap(
 497                                         vm_offset_t     *startp,
 498                                         vm_offset_t     *endp) __attribute__((section("__TEXT, initcode")));
 499
 500 extern void             vm_page_module_init(void) __attribute__((section("__TEXT, initcode")));
 501
 502 extern void             vm_page_init_local_q(void);
 503
 504 extern void             vm_page_create(
 505                                         ppnum_t         start,
 506                                         ppnum_t         end);
 507
 508 extern vm_page_t        vm_page_lookup(
 509                                         vm_object_t             object,
 510                                         vm_object_offset_t      offset);
 511
 512 extern vm_page_t        vm_page_grab_fictitious(void);
 513
 514 extern vm_page_t        vm_page_grab_guard(void);
 515
 516 extern void             vm_page_release_fictitious(
 517                                         vm_page_t page);
 518
 519 extern void             vm_page_more_fictitious(void);
 520
 521 extern int              vm_pool_low(void);
 522
 523 extern vm_page_t        vm_page_grab(void);
 524
 525 extern vm_page_t        vm_page_grablo(void);
 526
 527 extern void             vm_page_release(
 528                                         vm_page_t       page);
 529
 530 extern boolean_t        vm_page_wait(
 531                                         int             interruptible );
 532
 533 extern vm_page_t        vm_page_alloc(
 534                                         vm_object_t             object,
 535                                         vm_object_offset_t      offset);
 536
 537 extern vm_page_t        vm_page_alloclo(
 538                                         vm_object_t             object,
 539                                         vm_object_offset_t      offset);
 540
 541 extern vm_page_t        vm_page_alloc_guard(
 542         vm_object_t             object,
 543         vm_object_offset_t      offset);
 544
 545 extern void             vm_page_init(
 546                                         vm_page_t       page,
 547                                         ppnum_t         phys_page,
 548                                         boolean_t       lopage);
 549
 550 extern void             vm_page_free(
 551                                         vm_page_t       page);
 552
 553 extern void             vm_page_free_unlocked(
 554                                         vm_page_t       page,
 555                                         boolean_t       remove_from_hash);
 556
 557 extern void             vm_page_activate(
 558                                         vm_page_t       page);
 559
 560 extern void             vm_page_deactivate(
 561                                         vm_page_t       page);
 562
 563 extern void             vm_page_deactivate_internal(
 564                                         vm_page_t       page,
 565                                         boolean_t       clear_hw_reference);
 566
 567 extern void             vm_page_enqueue_cleaned(vm_page_t page);
 568
 569 extern void             vm_page_lru(
 570                                         vm_page_t       page);
 571
 572 extern void             vm_page_speculate(
 573                                         vm_page_t       page,
 574                                         boolean_t       new);
 575
 576 extern void             vm_page_speculate_ageit(
 577                                         struct vm_speculative_age_q *aq);
 578
 579 extern void             vm_page_reactivate_all_throttled(void);
 580
 581 extern void             vm_page_reactivate_local(uint32_t lid, boolean_t force, boolean_t nolocks);
 582
 583 extern void             vm_page_rename(
 584                                         vm_page_t               page,
 585                                         vm_object_t             new_object,
 586                                         vm_object_offset_t      new_offset,
 587                                         boolean_t               encrypted_ok);
 588
 589 extern void             vm_page_insert(
 590                                         vm_page_t               page,
 591                                         vm_object_t             object,
 592                                         vm_object_offset_t      offset);
 593
 594 extern void             vm_page_insert_internal(
 595                                         vm_page_t               page,
 596                                         vm_object_t             object,
 597                                         vm_object_offset_t      offset,
 598                                         boolean_t               queues_lock_held,
 599                                         boolean_t               insert_in_hash,
 600                                         boolean_t               batch_pmap_op);
 601
 602 extern void             vm_page_replace(
 603                                         vm_page_t               mem,
 604                                         vm_object_t             object,
 605                                         vm_object_offset_t      offset);
 606
 607 extern void             vm_page_remove(
 608                                         vm_page_t       page,
 609                                         boolean_t       remove_from_hash);
 610
 611 extern void             vm_page_zero_fill(
 612                                         vm_page_t       page);
 613
 614 extern void             vm_page_part_zero_fill(
 615                                         vm_page_t       m,
 616                                         vm_offset_t     m_pa,
 617                                         vm_size_t       len);
 618
 619 extern void             vm_page_copy(
 620                                         vm_page_t       src_page,
 621                                         vm_page_t       dest_page);
 622
 623 extern void             vm_page_part_copy(
 624                                         vm_page_t       src_m,
 625                                         vm_offset_t     src_pa,
 626                                         vm_page_t       dst_m,
 627                                         vm_offset_t     dst_pa,
 628                                         vm_size_t       len);
 629
 630 extern void             vm_page_wire(
 631                                         vm_page_t       page);
 632
 633 extern void             vm_page_unwire(
 634                                         vm_page_t       page,
 635                                         boolean_t       queueit);
 636
 637 extern void             vm_set_page_size(void);
 638
 639 extern void             vm_page_gobble(
 640                                         vm_page_t      page);
 641
 642 extern void             vm_page_validate_cs(vm_page_t   page);
 643 extern void             vm_page_validate_cs_mapped(
 644         vm_page_t       page,
 645         const void      *kaddr);
 646
 647 extern void             vm_page_free_prepare_queues(
 648                                         vm_page_t       page);
 649
 650 extern void             vm_page_free_prepare_object(
 651                                         vm_page_t       page,
 652                                         boolean_t       remove_from_hash);
 653
 654 #if CONFIG_JETSAM
 655 extern void memorystatus_update(unsigned int pages_avail);
 656
 657 #define VM_CHECK_MEMORYSTATUS do { \
 658         memorystatus_update(            \
 659                 vm_page_active_count +          \
 660                 vm_page_inactive_count +        \
 661                 vm_page_speculative_count +     \
 662                 vm_page_free_count +            \
 663                 (VM_DYNAMIC_PAGING_ENABLED(memory_manager_default) ? 0 : vm_page_purgeable_count) \
 664                 ); \
 665         } while(0)
 666 #else
 667 #define VM_CHECK_MEMORYSTATUS do {} while(0)
 668 #endif
 669
 670 /*
 671  *      Functions implemented as macros. m->wanted and m->busy are
 672  *      protected by the object lock.
 673  */
 674
 675 #if CONFIG_EMBEDDED
 676 #define SET_PAGE_DIRTY(m, set_pmap_modified)                            \
 677                 MACRO_BEGIN                                             \
 678                 vm_page_t __page__ = (m);                               \
 679                 if (__page__->dirty == FALSE && (set_pmap_modified)) {  \
 680                         pmap_set_modify(__page__->phys_page);           \
 681                 }                                                       \
 682                 __page__->dirty = TRUE;                                 \
 683                 MACRO_END
 684 #else /* CONFIG_EMBEDDED */
 685 #define SET_PAGE_DIRTY(m, set_pmap_modified)                            \
 686                 MACRO_BEGIN                                             \
 687                 vm_page_t __page__ = (m);                               \
 688                 __page__->dirty = TRUE;                                 \
 689                 MACRO_END
 690 #endif /* CONFIG_EMBEDDED */
 691
 692 #define PAGE_ASSERT_WAIT(m, interruptible)                      \
 693                 (((m)->wanted = TRUE),                          \
 694                  assert_wait((event_t) (m), (interruptible)))
 695
 696 #define PAGE_SLEEP(o, m, interruptible)                         \
 697                 (((m)->wanted = TRUE),                          \
 698                  thread_sleep_vm_object((o), (m), (interruptible)))
 699
 700 #define PAGE_WAKEUP_DONE(m)                                     \
 701                 MACRO_BEGIN                                     \
 702                 (m)->busy = FALSE;                              \
 703                 if ((m)->wanted) {                              \
 704                         (m)->wanted = FALSE;                    \
 705                         thread_wakeup((event_t) (m));           \
 706                 }                                               \
 707                 MACRO_END
 708
 709 #define PAGE_WAKEUP(m)                                          \
 710                 MACRO_BEGIN                                     \
 711                 if ((m)->wanted) {                              \
 712                         (m)->wanted = FALSE;                    \
 713                         thread_wakeup((event_t) (m));           \
 714                 }                                               \
 715                 MACRO_END
 716
 717 #define VM_PAGE_FREE(p)                         \
 718                 MACRO_BEGIN                     \
 719                 vm_page_free_unlocked(p, TRUE); \
 720                 MACRO_END
 721
 722 #define VM_PAGE_GRAB_FICTITIOUS(M)                                      \
 723                 MACRO_BEGIN                                             \
 724                 while ((M = vm_page_grab_fictitious()) == VM_PAGE_NULL) \
 725                         vm_page_more_fictitious();                      \
 726                 MACRO_END
 727
 728 #define VM_PAGE_WAIT()          ((void)vm_page_wait(THREAD_UNINT))
 729
 730 #define vm_page_queue_lock (vm_page_locks.vm_page_queue_lock2)
 731 #define vm_page_queue_free_lock (vm_page_locks.vm_page_queue_free_lock2)
 732
 733 #define vm_page_lock_queues()   lck_mtx_lock(&vm_page_queue_lock)
 734 #define vm_page_unlock_queues() lck_mtx_unlock(&vm_page_queue_lock)
 735
 736 #define vm_page_lockspin_queues()       lck_mtx_lock_spin(&vm_page_queue_lock)
 737 #define vm_page_trylockspin_queues()    lck_mtx_try_lock_spin(&vm_page_queue_lock)
 738 #define vm_page_lockconvert_queues()    lck_mtx_convert_spin(&vm_page_queue_lock)
 739
 740 #ifdef  VPL_LOCK_SPIN
 741 #define VPL_LOCK_INIT(vlq, vpl_grp, vpl_attr) lck_spin_init(&vlq->vpl_lock, vpl_grp, vpl_attr)
 742 #define VPL_LOCK(vpl) lck_spin_lock(vpl)
 743 #define VPL_UNLOCK(vpl) lck_spin_unlock(vpl)
 744 #else
 745 #define VPL_LOCK_INIT(vlq, vpl_grp, vpl_attr) lck_mtx_init_ext(&vlq->vpl_lock, &vlq->vpl_lock_ext, vpl_grp, vpl_attr)
 746 #define VPL_LOCK(vpl) lck_mtx_lock_spin(vpl)
 747 #define VPL_UNLOCK(vpl) lck_mtx_unlock(vpl)
 748 #endif
 749
 750 #if MACH_ASSERT
 751 extern void vm_page_queues_assert(vm_page_t mem, int val);
 752 #define VM_PAGE_QUEUES_ASSERT(mem, val) vm_page_queues_assert((mem), (val))
 753 #else
 754 #define VM_PAGE_QUEUES_ASSERT(mem, val)
 755 #endif
 756
 757
 758 /*
 759  * 'vm_fault_enter' will place newly created pages (zero-fill and COW) onto the
 760  * local queues if they exist... its the only spot in the system where we add pages
 761  * to those queues...  once on those queues, those pages can only move to one of the
 762  * global page queues or the free queues... they NEVER move from local q to local q.
 763  * the 'local' state is stable when VM_PAGE_QUEUES_REMOVE is called since we're behind
 764  * the global vm_page_queue_lock at this point...  we still need to take the local lock
 765  * in case this operation is being run on a different CPU then the local queue's identity,
 766  * but we don't have to worry about the page moving to a global queue or becoming wired
 767  * while we're grabbing the local lock since those operations would require the global
 768  * vm_page_queue_lock to be held, and we already own it.
 769  *
 770  * this is why its safe to utilze the wire_count field in the vm_page_t as the local_id...
 771  * 'wired' and local are ALWAYS mutually exclusive conditions.
 772  */
 773
 774 #define VM_PAGE_QUEUES_REMOVE(mem)                              \
 775         MACRO_BEGIN                                             \
 776         VM_PAGE_QUEUES_ASSERT(mem, 1);                          \
 777         assert(!mem->laundry);                                  \
 778 /*                                                              \
 779  *      if (mem->pageout_queue)                                 \
 780  *              NOTE: VM_PAGE_QUEUES_REMOVE does not deal with removing pages from the pageout queue... \
 781  *              the caller is responsible for determing if the page is on that queue, and if so, must   \
 782  *              either first remove it (it needs both the page queues lock and the object lock to do    \
 783  *              this via vm_pageout_steal_laundry), or avoid the call to VM_PAGE_QUEUES_REMOVE          \
 784  */                                                             \
 785         if (mem->local) {                                       \
 786                 struct vpl      *lq;                            \
 787                 assert(mem->object != kernel_object);           \
 788                 assert(!mem->inactive && !mem->speculative);    \
 789                 assert(!mem->active && !mem->throttled);        \
 790                 assert(!mem->clean_queue);                      \
 791                 assert(!mem->fictitious);                       \
 792                 lq = &vm_page_local_q[mem->local_id].vpl_un.vpl;        \
 793                 VPL_LOCK(&lq->vpl_lock);                        \
 794                 queue_remove(&lq->vpl_queue,                    \
 795                              mem, vm_page_t, pageq);            \
 796                 mem->local = FALSE;                             \
 797                 mem->local_id = 0;                              \
 798                 lq->vpl_count--;                                \
 799                 VPL_UNLOCK(&lq->vpl_lock);                      \
 800         }                                                       \
 801                                                                 \
 802         else if (mem->active) {                                 \
 803                 assert(mem->object != kernel_object);           \
 804                 assert(!mem->inactive && !mem->speculative);    \
 805                 assert(!mem->clean_queue);                      \
 806                 assert(!mem->throttled);                        \
 807                 assert(!mem->fictitious);                       \
 808                 queue_remove(&vm_page_queue_active,             \
 809                         mem, vm_page_t, pageq);                 \
 810                 mem->active = FALSE;                            \
 811                 vm_page_active_count--;                         \
 812         }                                                       \
 813                                                                 \
 814         else if (mem->inactive) {                               \
 815                 assert(mem->object != kernel_object);           \
 816                 assert(!mem->active && !mem->speculative);      \
 817                 assert(!mem->throttled);                        \
 818                 assert(!mem->fictitious);                       \
 819                 vm_page_inactive_count--;                       \
 820                 if (mem->clean_queue) {                         \
 821                         queue_remove(&vm_page_queue_cleaned,    \
 822                         mem, vm_page_t, pageq);                 \
 823                         mem->clean_queue = FALSE;               \
 824                         vm_page_cleaned_count--;                \
 825                 } else {                                        \
 826                         if (mem->object->internal) {            \
 827                                 queue_remove(&vm_page_queue_anonymous,  \
 828                                 mem, vm_page_t, pageq);         \
 829                                 vm_page_anonymous_count--;      \
 830                         } else {                                \
 831                                 queue_remove(&vm_page_queue_inactive,   \
 832                                 mem, vm_page_t, pageq);         \
 833                         }                                       \
 834                         vm_purgeable_q_advance_all();           \
 835                 }                                               \
 836                 mem->inactive = FALSE;                          \
 837         }                                                       \
 838                                                                 \
 839         else if (mem->throttled) {                              \
 840                 assert(!mem->active && !mem->inactive);         \
 841                 assert(!mem->speculative);                      \
 842                 assert(!mem->fictitious);                       \
 843                 queue_remove(&vm_page_queue_throttled,          \
 844                              mem, vm_page_t, pageq);            \
 845                 mem->throttled = FALSE;                         \
 846                 vm_page_throttled_count--;                      \
 847         }                                                       \
 848                                                                 \
 849         else if (mem->speculative) {                            \
 850                 assert(!mem->active && !mem->inactive);         \
 851                 assert(!mem->throttled);                        \
 852                 assert(!mem->fictitious);                       \
 853                 remque(&mem->pageq);                            \
 854                 mem->speculative = FALSE;                       \
 855                 vm_page_speculative_count--;                    \
 856         }                                                       \
 857                                                                 \
 858         else if (mem->pageq.next || mem->pageq.prev)            \
 859                 panic("VM_PAGE_QUEUES_REMOVE: unmarked page on Q");     \
 860         mem->pageq.next = NULL;                                 \
 861         mem->pageq.prev = NULL;                                 \
 862         VM_PAGE_QUEUES_ASSERT(mem, 0);                          \
 863         MACRO_END
 864
 865
 866 #define VM_PAGE_ENQUEUE_INACTIVE(mem, first)                    \
 867         MACRO_BEGIN                                             \
 868         VM_PAGE_QUEUES_ASSERT(mem, 0);                          \
 869         assert(!mem->fictitious);                               \
 870         assert(!mem->laundry);                                  \
 871         assert(!mem->pageout_queue);                            \
 872         if (mem->object->internal) {                            \
 873                 if (first == TRUE)                              \
 874                         queue_enter_first(&vm_page_queue_anonymous, mem, vm_page_t, pageq);     \
 875                 else                                            \
 876                         queue_enter(&vm_page_queue_anonymous, mem, vm_page_t, pageq);           \
 877                 vm_page_anonymous_count++;                              \
 878         } else {                                                \
 879                 if (first == TRUE)                              \
 880                         queue_enter_first(&vm_page_queue_inactive, mem, vm_page_t, pageq); \
 881                 else                                            \
 882                         queue_enter(&vm_page_queue_inactive, mem, vm_page_t, pageq);    \
 883         }                                                       \
 884         mem->inactive = TRUE;                                   \
 885         vm_page_inactive_count++;                               \
 886         token_new_pagecount++;                                  \
 887         MACRO_END
 888
 889
 890 #if DEVELOPMENT || DEBUG
 891 #define VM_PAGE_SPECULATIVE_USED_ADD()                          \
 892         MACRO_BEGIN                                             \
 893         OSAddAtomic(1, &vm_page_speculative_used);      \
 894         MACRO_END
 895 #else
 896 #define VM_PAGE_SPECULATIVE_USED_ADD()
 897 #endif
 898
 899
 900 #define VM_PAGE_CONSUME_CLUSTERED(mem)                          \
 901         MACRO_BEGIN                                             \
 902         if (mem->clustered) {                                   \
 903                 assert(mem->object);                            \
 904                 mem->object->pages_used++;                      \
 905                 mem->clustered = FALSE;                         \
 906                 VM_PAGE_SPECULATIVE_USED_ADD();                 \
 907         }                                                       \
 908         MACRO_END
 909
 910
 911
 912 #define DW_vm_page_unwire               0x01
 913 #define DW_vm_page_wire                 0x02
 914 #define DW_vm_page_free                 0x04
 915 #define DW_vm_page_activate             0x08
 916 #define DW_vm_page_deactivate_internal  0x10
 917 #define DW_vm_page_speculate            0x20
 918 #define DW_vm_page_lru                  0x40
 919 #define DW_vm_pageout_throttle_up       0x80
 920 #define DW_PAGE_WAKEUP                  0x100
 921 #define DW_clear_busy                   0x200
 922 #define DW_clear_reference              0x400
 923 #define DW_set_reference                0x800
 924 #define DW_move_page                    0x1000
 925 #define DW_VM_PAGE_QUEUES_REMOVE        0x2000
 926 #define DW_enqueue_cleaned              0x4000
 927
 928 struct vm_page_delayed_work {
 929         vm_page_t       dw_m;
 930         int             dw_mask;
 931 };
 932
 933 void vm_page_do_delayed_work(vm_object_t object, struct vm_page_delayed_work *dwp, int dw_count);
 934
 935 extern unsigned int vm_max_delayed_work_limit;
 936
 937 #define DEFAULT_DELAYED_WORK_LIMIT      32
 938
 939 #define DELAYED_WORK_LIMIT(max) ((vm_max_delayed_work_limit >= max ? max : vm_max_delayed_work_limit))
 940
 941 /*
 942  * vm_page_do_delayed_work may need to drop the object lock...
 943  * if it does, we need the pages it's looking at to
 944  * be held stable via the busy bit, so if busy isn't already
 945  * set, we need to set it and ask vm_page_do_delayed_work
 946  * to clear it and wakeup anyone that might have blocked on
 947  * it once we're done processing the page.
 948  */
 949
 950 #define VM_PAGE_ADD_DELAYED_WORK(dwp, mem, dw_cnt)              \
 951         MACRO_BEGIN                                             \
 952         if (mem->busy == FALSE) {                               \
 953                 mem->busy = TRUE;                               \
 954                 if ( !(dwp->dw_mask & DW_vm_page_free))         \
 955                         dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP); \
 956         }                                                       \
 957         dwp->dw_m = mem;                                        \
 958         dwp++;                                                  \
 959         dw_cnt++;                                               \
 960         MACRO_END
 961
 962 extern vm_page_t vm_object_page_grab(vm_object_t);
 963
 964
 965 #endif  /* _VM_VM_PAGE_H_ */