osfmk/vm/vm_page.h

   1 /*
   2  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * Copyright (c) 1999-2003 Apple Computer, Inc.  All Rights Reserved.
   7  *
   8  * This file contains Original Code and/or Modifications of Original Code
   9  * as defined in and that are subject to the Apple Public Source License
  10  * Version 2.0 (the 'License'). You may not use this file except in
  11  * compliance with the License. Please obtain a copy of the License at
  12  * http://www.opensource.apple.com/apsl/ and read it before using this
  13  * file.
  14  *
  15  * The Original Code and all software distributed under the License are
  16  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  17  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  18  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  20  * Please see the License for the specific language governing rights and
  21  * limitations under the License.
  22  *
  23  * @APPLE_LICENSE_HEADER_END@
  24  */
  25 /*
  26  * @OSF_COPYRIGHT@
  27  */
  28 /*
  29  * Mach Operating System
  30  * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
  31  * All Rights Reserved.
  32  *
  33  * Permission to use, copy, modify and distribute this software and its
  34  * documentation is hereby granted, provided that both the copyright
  35  * notice and this permission notice appear in all copies of the
  36  * software, derivative works or modified versions, and any portions
  37  * thereof, and that both notices appear in supporting documentation.
  38  *
  39  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  40  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  41  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  42  *
  43  * Carnegie Mellon requests users of this software to return to
  44  *
  45  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  46  *  School of Computer Science
  47  *  Carnegie Mellon University
  48  *  Pittsburgh PA 15213-3890
  49  *
  50  * any improvements or extensions that they make and grant Carnegie Mellon
  51  * the rights to redistribute these changes.
  52  */
  53 /*
  54  */
  55 /*
  56  *      File:   vm/vm_page.h
  57  *      Author: Avadis Tevanian, Jr., Michael Wayne Young
  58  *      Date:   1985
  59  *
  60  *      Resident memory system definitions.
  61  */
  62
  63 #ifndef _VM_VM_PAGE_H_
  64 #define _VM_VM_PAGE_H_
  65
  66 #include <mach/boolean.h>
  67 #include <mach/vm_prot.h>
  68 #include <mach/vm_param.h>
  69 #include <vm/vm_object.h>
  70 #include <kern/queue.h>
  71 #include <kern/lock.h>
  72
  73 #include <kern/macro_help.h>
  74
  75 /*
  76  * Each page entered on the inactive queue obtains a ticket from a
  77  * particular ticket roll.  Pages granted tickets from a particular
  78  * roll  generally flow through the queue as a group.  In this way when a
  79  * page with a ticket from a particular roll is pulled from the top of the
  80  * queue it is extremely likely that the pages near the top will have tickets
  81  * from the same or adjacent rolls.  In this way the proximity to the top
  82  * of the queue can be loosely ascertained by determining the identity of
  83  * the roll the pages ticket came from.
  84  */
  85
  86
  87 extern int      vm_page_ticket_roll;
  88 extern int      vm_page_ticket;
  89
  90
  91 #define VM_PAGE_TICKETS_IN_ROLL  512
  92 #define VM_PAGE_TICKET_ROLL_IDS  16
  93
  94 /*
  95  *      Management of resident (logical) pages.
  96  *
  97  *      A small structure is kept for each resident
  98  *      page, indexed by page number.  Each structure
  99  *      is an element of several lists:
 100  *
 101  *              A hash table bucket used to quickly
 102  *              perform object/offset lookups
 103  *
 104  *              A list of all pages for a given object,
 105  *              so they can be quickly deactivated at
 106  *              time of deallocation.
 107  *
 108  *              An ordered list of pages due for pageout.
 109  *
 110  *      In addition, the structure contains the object
 111  *      and offset to which this page belongs (for pageout),
 112  *      and sundry status bits.
 113  *
 114  *      Fields in this structure are locked either by the lock on the
 115  *      object that the page belongs to (O) or by the lock on the page
 116  *      queues (P).  [Some fields require that both locks be held to
 117  *      change that field; holding either lock is sufficient to read.]
 118  */
 119
 120 struct vm_page {
 121         queue_chain_t   pageq;          /* queue info for FIFO
 122                                          * queue or free list (P) */
 123         queue_chain_t   listq;          /* all pages in same object (O) */
 124         struct vm_page  *next;          /* VP bucket link (O) */
 125
 126         vm_object_t     object;         /* which object am I in (O&P) */
 127         vm_object_offset_t offset;      /* offset into that object (O,P) */
 128
 129         unsigned int    wire_count:16,  /* how many wired down maps use me? (O&P) */
 130                         page_ticket:4,  /* age of the page on the       */
 131                                         /* inactive queue.                  */
 132         /* boolean_t */ inactive:1,     /* page is in inactive list (P) */
 133                         active:1,       /* page is in active list (P) */
 134                         laundry:1,      /* page is being cleaned now (P)*/
 135                         free:1,         /* page is on free list (P) */
 136                         reference:1,    /* page has been used (P) */
 137                         pageout:1,      /* page wired & busy for pageout (P) */
 138                         gobbled:1,      /* page used internally (P) */
 139                         private:1,      /* Page should not be returned to
 140                                          *  the free list (O) */
 141                         zero_fill:1,
 142                         :0;
 143
 144         unsigned int
 145                         page_error:8,   /* error from I/O operations */
 146         /* boolean_t */ busy:1,         /* page is in transit (O) */
 147                         wanted:1,       /* someone is waiting for page (O) */
 148                         tabled:1,       /* page is in VP table (O) */
 149                         fictitious:1,   /* Physical page doesn't exist (O) */
 150                         no_isync:1,     /* page has not been instruction synced */
 151                         absent:1,       /* Data has been requested, but is
 152                                          *  not yet available (O) */
 153                         error:1,        /* Data manager was unable to provide
 154                                          *  data due to error (O) */
 155                         dirty:1,        /* Page must be cleaned (O) */
 156                         cleaning:1,     /* Page clean has begun (O) */
 157                         precious:1,     /* Page is precious; data must be
 158                                          *  returned even if clean (O) */
 159                         clustered:1,    /* page is not the faulted page (O) */
 160                         overwriting:1,  /* Request to unlock has been made
 161                                          * without having data. (O)
 162                                          * [See vm_fault_page_overwrite] */
 163                         restart:1,      /* Page was pushed higher in shadow
 164                                            chain by copy_call-related pagers;
 165                                            start again at top of chain */
 166                         lock_supplied:1,/* protection supplied by pager (O) */
 167         /* vm_prot_t */ page_lock:3,    /* Uses prohibited by pager (O) */
 168         /* vm_prot_t */ unlock_request:3,/* Outstanding unlock request (O) */
 169                         unusual:1,      /* Page is absent, error, restart or
 170                                            page locked */
 171                         discard_request:1,/* a memory_object_discard_request()
 172                                            * has been sent */
 173                         list_req_pending:1, /* pagein/pageout alt mechanism */
 174                                             /* allows creation of list      */
 175                                             /* requests on pages that are   */
 176                                             /* actively being paged.        */
 177                         dump_cleaning:1;   /* set by the pageout daemon when */
 178                                            /* a page being cleaned is       */
 179                                            /* encountered and targeted as   */
 180                                            /* a pageout candidate           */
 181         /* we've used up all 32 bits */
 182
 183         vm_offset_t     phys_page;      /* Physical address of page, passed
 184                                          *  to pmap_enter (read-only) */
 185 };
 186
 187 typedef struct vm_page  *vm_page_t;
 188
 189 #define VM_PAGE_NULL            ((vm_page_t) 0)
 190 #define NEXT_PAGE(m)    ((vm_page_t) (m)->pageq.next)
 191
 192 /*
 193  * XXX  The unusual bit should not be necessary.  Most of the bit
 194  * XXX  fields above really want to be masks.
 195  */
 196
 197 /*
 198  *      For debugging, this macro can be defined to perform
 199  *      some useful check on a page structure.
 200  */
 201
 202 #define VM_PAGE_CHECK(mem)
 203
 204 /*
 205  *      Each pageable resident page falls into one of three lists:
 206  *
 207  *      free
 208  *              Available for allocation now.
 209  *      inactive
 210  *              Not referenced in any map, but still has an
 211  *              object/offset-page mapping, and may be dirty.
 212  *              This is the list of pages that should be
 213  *              paged out next.
 214  *      active
 215  *              A list of pages which have been placed in
 216  *              at least one physical map.  This list is
 217  *              ordered, in LRU-like fashion.
 218  */
 219
 220 extern
 221 vm_page_t       vm_page_queue_free;     /* memory free queue */
 222 extern
 223 vm_page_t       vm_page_queue_fictitious;       /* fictitious free queue */
 224 extern
 225 queue_head_t    vm_page_queue_active;   /* active memory queue */
 226 extern
 227 queue_head_t    vm_page_queue_inactive; /* inactive memory queue */
 228 queue_head_t    vm_page_queue_zf;       /* inactive memory queue for zero fill */
 229
 230 extern
 231 vm_offset_t     first_phys_addr;        /* physical address for first_page */
 232 extern
 233 vm_offset_t     last_phys_addr;         /* physical address for last_page */
 234
 235 extern
 236 int     vm_page_free_count;     /* How many pages are free? */
 237 extern
 238 int     vm_page_fictitious_count;/* How many fictitious pages are free? */
 239 extern
 240 int     vm_page_active_count;   /* How many pages are active? */
 241 extern
 242 int     vm_page_inactive_count; /* How many pages are inactive? */
 243 extern
 244 int     vm_page_wire_count;     /* How many pages are wired? */
 245 extern
 246 int     vm_page_free_target;    /* How many do we want free? */
 247 extern
 248 int     vm_page_free_min;       /* When to wakeup pageout */
 249 extern
 250 int     vm_page_inactive_target;/* How many do we want inactive? */
 251 extern
 252 int     vm_page_free_reserved;  /* How many pages reserved to do pageout */
 253 extern
 254 int     vm_page_laundry_count;  /* How many pages being laundered? */
 255
 256 decl_mutex_data(,vm_page_queue_lock)
 257                                 /* lock on active and inactive page queues */
 258 decl_mutex_data(,vm_page_queue_free_lock)
 259                                 /* lock on free page queue */
 260 decl_simple_lock_data(extern,vm_page_preppin_lock)      /* lock for prep/pin */
 261 decl_mutex_data(,vm_page_zero_fill_lock)
 262
 263 extern unsigned int     vm_page_free_wanted;
 264                                 /* how many threads are waiting for memory */
 265
 266 extern vm_offset_t      vm_page_fictitious_addr;
 267                                 /* (fake) phys_addr of fictitious pages */
 268
 269 /*
 270  * Prototypes for functions exported by this module.
 271  */
 272 extern void             vm_page_bootstrap(
 273                                         vm_offset_t     *startp,
 274                                         vm_offset_t     *endp);
 275
 276 extern void             vm_page_module_init(void);
 277
 278 extern void             vm_page_create(
 279                                         ppnum_t         start,
 280                                         ppnum_t         end);
 281
 282 extern vm_page_t        vm_page_lookup(
 283                                         vm_object_t             object,
 284                                         vm_object_offset_t      offset);
 285
 286 extern vm_page_t        vm_page_grab_fictitious(void);
 287
 288 extern void             vm_page_release_fictitious(
 289                                         vm_page_t page);
 290
 291 extern boolean_t        vm_page_convert(
 292                                         vm_page_t       page);
 293
 294 extern void             vm_page_more_fictitious(void);
 295
 296 extern int              vm_pool_low(void);
 297
 298 extern vm_page_t        vm_page_grab(void);
 299
 300 extern void             vm_page_release(
 301                                         vm_page_t       page);
 302
 303 extern void             vm_page_release_limbo(
 304                                         vm_page_t       page);
 305
 306 extern void             vm_page_limbo_exchange(
 307                                         vm_page_t       limbo_m,
 308                                         vm_page_t       new_m);
 309
 310 extern boolean_t        vm_page_wait(
 311                                         int             interruptible );
 312
 313 extern vm_page_t        vm_page_alloc(
 314                                         vm_object_t             object,
 315                                         vm_object_offset_t      offset);
 316
 317 extern void             vm_page_init(
 318                                         vm_page_t       page,
 319                                         ppnum_t         phys_page);
 320
 321 extern void             vm_page_free(
 322                                         vm_page_t       page);
 323
 324 extern void             vm_page_activate(
 325                                         vm_page_t       page);
 326
 327 extern void             vm_page_deactivate(
 328                                         vm_page_t       page);
 329
 330 extern void             vm_page_rename(
 331                                         vm_page_t               page,
 332                                         vm_object_t             new_object,
 333                                         vm_object_offset_t      new_offset);
 334
 335 extern void             vm_page_insert(
 336                                         vm_page_t               page,
 337                                         vm_object_t             object,
 338                                         vm_object_offset_t      offset);
 339
 340 extern void             vm_page_replace(
 341                                         vm_page_t               mem,
 342                                         vm_object_t             object,
 343                                         vm_object_offset_t      offset);
 344
 345 extern void             vm_page_remove(
 346                                         vm_page_t       page);
 347
 348 extern void             vm_page_zero_fill(
 349                                         vm_page_t       page);
 350
 351 extern void             vm_page_part_zero_fill(
 352                                         vm_page_t       m,
 353                                         vm_offset_t     m_pa,
 354                                         vm_size_t       len);
 355
 356 extern void             vm_page_copy(
 357                                         vm_page_t       src_page,
 358                                         vm_page_t       dest_page);
 359
 360 extern void             vm_page_part_copy(
 361                                         vm_page_t       src_m,
 362                                         vm_offset_t     src_pa,
 363                                         vm_page_t       dst_m,
 364                                         vm_offset_t     dst_pa,
 365                                         vm_size_t       len);
 366
 367 extern void             vm_page_wire(
 368                                         vm_page_t       page);
 369
 370 extern void             vm_page_unwire(
 371                                         vm_page_t       page);
 372
 373 extern void             vm_set_page_size(void);
 374
 375 extern void             vm_page_gobble(
 376                                         vm_page_t      page);
 377
 378 /*
 379  *      Functions implemented as macros. m->wanted and m->busy are
 380  *      protected by the object lock.
 381  */
 382
 383 #define PAGE_ASSERT_WAIT(m, interruptible)                      \
 384                 (((m)->wanted = TRUE),                          \
 385                  assert_wait((event_t) (m), (interruptible)))
 386
 387 #define PAGE_SLEEP(o, m, interruptible)                         \
 388                 (((m)->wanted = TRUE),                          \
 389                  thread_sleep_vm_object((o), (m), (interruptible)))
 390
 391 #define PAGE_WAKEUP_DONE(m)                                     \
 392                 MACRO_BEGIN                                     \
 393                 (m)->busy = FALSE;                              \
 394                 if ((m)->wanted) {                              \
 395                         (m)->wanted = FALSE;                    \
 396                         thread_wakeup((event_t) (m));           \
 397                 }                                               \
 398                 MACRO_END
 399
 400 #define PAGE_WAKEUP(m)                                          \
 401                 MACRO_BEGIN                                     \
 402                 if ((m)->wanted) {                              \
 403                         (m)->wanted = FALSE;                    \
 404                         thread_wakeup((event_t) (m));           \
 405                 }                                               \
 406                 MACRO_END
 407
 408 #define VM_PAGE_FREE(p)                         \
 409                 MACRO_BEGIN                     \
 410                 vm_page_lock_queues();          \
 411                 vm_page_free(p);                \
 412                 vm_page_unlock_queues();        \
 413                 MACRO_END
 414
 415 #define VM_PAGE_GRAB_FICTITIOUS(M)                                      \
 416                 MACRO_BEGIN                                             \
 417                 while ((M = vm_page_grab_fictitious()) == VM_PAGE_NULL) \
 418                         vm_page_more_fictitious();                      \
 419                 MACRO_END
 420
 421 #define VM_PAGE_THROTTLED()                                             \
 422                 (vm_page_free_count < (vm_page_free_target -            \
 423                  ((vm_page_free_target-vm_page_free_min)>>2)))
 424
 425 #define VM_PAGE_WAIT()          ((void)vm_page_wait(THREAD_UNINT))
 426
 427 #define vm_page_lock_queues()   mutex_lock(&vm_page_queue_lock)
 428 #define vm_page_unlock_queues() mutex_unlock(&vm_page_queue_lock)
 429 #define vm_page_pin_lock()      simple_lock(&vm_page_preppin_lock)
 430 #define vm_page_pin_unlock()    simple_unlock(&vm_page_preppin_lock)
 431
 432 #define VM_PAGE_QUEUES_REMOVE(mem)                              \
 433         MACRO_BEGIN                                             \
 434         if (mem->active) {                                      \
 435                 assert(!mem->inactive);                         \
 436                 queue_remove(&vm_page_queue_active,             \
 437                         mem, vm_page_t, pageq);                 \
 438                 mem->active = FALSE;                            \
 439                 if (!mem->fictitious)                           \
 440                         vm_page_active_count--;                 \
 441         }                                                       \
 442                                                                 \
 443         if (mem->inactive) {                                    \
 444                 assert(!mem->active);                           \
 445                 if (mem->zero_fill) {                           \
 446                         queue_remove(&vm_page_queue_zf,         \
 447                         mem, vm_page_t, pageq);                 \
 448                 } else {                                        \
 449                         queue_remove(&vm_page_queue_inactive,   \
 450                         mem, vm_page_t, pageq);                 \
 451                 }                                               \
 452                 mem->inactive = FALSE;                          \
 453                 if (!mem->fictitious)                           \
 454                         vm_page_inactive_count--;               \
 455         }                                                       \
 456         MACRO_END
 457
 458 #endif  /* _VM_VM_PAGE_H_ */