osfmk/vm/vm_page.h

   1 /*
   2  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * The contents of this file constitute Original Code as defined in and
   7  * are subject to the Apple Public Source License Version 1.1 (the
   8  * "License").  You may not use this file except in compliance with the
   9  * License.  Please obtain a copy of the License at
  10  * http://www.apple.com/publicsource and read it before using this file.
  11  *
  12  * This Original Code and all software distributed under the License are
  13  * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  14  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  15  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
  17  * License for the specific language governing rights and limitations
  18  * under the License.
  19  *
  20  * @APPLE_LICENSE_HEADER_END@
  21  */
  22 /*
  23  * @OSF_COPYRIGHT@
  24  */
  25 /*
  26  * Mach Operating System
  27  * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
  28  * All Rights Reserved.
  29  *
  30  * Permission to use, copy, modify and distribute this software and its
  31  * documentation is hereby granted, provided that both the copyright
  32  * notice and this permission notice appear in all copies of the
  33  * software, derivative works or modified versions, and any portions
  34  * thereof, and that both notices appear in supporting documentation.
  35  *
  36  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  37  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  38  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  39  *
  40  * Carnegie Mellon requests users of this software to return to
  41  *
  42  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  43  *  School of Computer Science
  44  *  Carnegie Mellon University
  45  *  Pittsburgh PA 15213-3890
  46  *
  47  * any improvements or extensions that they make and grant Carnegie Mellon
  48  * the rights to redistribute these changes.
  49  */
  50 /*
  51  */
  52 /*
  53  *      File:   vm/vm_page.h
  54  *      Author: Avadis Tevanian, Jr., Michael Wayne Young
  55  *      Date:   1985
  56  *
  57  *      Resident memory system definitions.
  58  */
  59
  60 #ifndef _VM_VM_PAGE_H_
  61 #define _VM_VM_PAGE_H_
  62
  63 #include <mach/boolean.h>
  64 #include <mach/vm_prot.h>
  65 #include <mach/vm_param.h>
  66 #include <vm/vm_object.h>
  67 #include <kern/queue.h>
  68 #include <kern/lock.h>
  69
  70 #include <kern/macro_help.h>
  71
  72 /*
  73  *      Management of resident (logical) pages.
  74  *
  75  *      A small structure is kept for each resident
  76  *      page, indexed by page number.  Each structure
  77  *      is an element of several lists:
  78  *
  79  *              A hash table bucket used to quickly
  80  *              perform object/offset lookups
  81  *
  82  *              A list of all pages for a given object,
  83  *              so they can be quickly deactivated at
  84  *              time of deallocation.
  85  *
  86  *              An ordered list of pages due for pageout.
  87  *
  88  *      In addition, the structure contains the object
  89  *      and offset to which this page belongs (for pageout),
  90  *      and sundry status bits.
  91  *
  92  *      Fields in this structure are locked either by the lock on the
  93  *      object that the page belongs to (O) or by the lock on the page
  94  *      queues (P).  [Some fields require that both locks be held to
  95  *      change that field; holding either lock is sufficient to read.]
  96  */
  97
  98 struct vm_page {
  99         queue_chain_t   pageq;          /* queue info for FIFO
 100                                          * queue or free list (P) */
 101         queue_chain_t   listq;          /* all pages in same object (O) */
 102         struct vm_page  *next;          /* VP bucket link (O) */
 103
 104         vm_object_t     object;         /* which object am I in (O&P) */
 105         vm_object_offset_t offset;      /* offset into that object (O,P) */
 106
 107         unsigned int    wire_count:16,  /* how many wired down maps use me? (O&P) */
 108                         page_error:8,   /* error from I/O operations */
 109         /* boolean_t */ inactive:1,     /* page is in inactive list (P) */
 110                         active:1,       /* page is in active list (P) */
 111                         laundry:1,      /* page is being cleaned now (P)*/
 112                         free:1,         /* page is on free list (P) */
 113                         reference:1,    /* page has been used (P) */
 114                         limbo:1,        /* page prepped then stolen (P) */
 115                         pageout:1,      /* page wired & busy for pageout (P) */
 116                         gobbled:1;      /* page used internally (P) */
 117         /* we've used up all 32 bits */
 118
 119         unsigned int
 120         /* boolean_t */ busy:1,         /* page is in transit (O) */
 121                         wanted:1,       /* someone is waiting for page (O) */
 122                         tabled:1,       /* page is in VP table (O) */
 123                         fictitious:1,   /* Physical page doesn't exist (O) */
 124                         private:1,      /* Page should not be returned to
 125                                          *  the free list (O) */
 126                         absent:1,       /* Data has been requested, but is
 127                                          *  not yet available (O) */
 128                         error:1,        /* Data manager was unable to provide
 129                                          *  data due to error (O) */
 130                         dirty:1,        /* Page must be cleaned (O) */
 131                         cleaning:1,     /* Page clean has begun (O) */
 132                         precious:1,     /* Page is precious; data must be
 133                                          *  returned even if clean (O) */
 134                         clustered:1,    /* page is not the faulted page (O) */
 135                         overwriting:1,  /* Request to unlock has been made
 136                                          * without having data. (O)
 137                                          * [See vm_fault_page_overwrite] */
 138                         restart:1,      /* Page was pushed higher in shadow
 139                                            chain by copy_call-related pagers;
 140                                            start again at top of chain */
 141                         lock_supplied:1,/* protection supplied by pager (O) */
 142         /* vm_prot_t */ page_lock:3,    /* Uses prohibited by pager (O) */
 143         /* vm_prot_t */ unlock_request:3,/* Outstanding unlock request (O) */
 144                         unusual:1,      /* Page is absent, error, restart or
 145                                            page locked */
 146                         discard_request:1,/* a memory_object_discard_request()
 147                                            * has been sent */
 148                         list_req_pending:1, /* pagein/pageout alt mechanism */
 149                                             /* allows creation of list      */
 150                                             /* requests on pages that are   */
 151                                             /* actively being paged.        */
 152                         :0;
 153
 154         vm_offset_t     phys_addr;      /* Physical address of page, passed
 155                                          *  to pmap_enter (read-only) */
 156         union {
 157           struct {
 158            unsigned int prep:16,        /* page prep count */
 159                         pin:16;         /* page pin pount */
 160           } pp_counts;
 161           unsigned int  pp_both;        /* used to test for both zero */
 162         } prep_pin_u;
 163 };
 164
 165 typedef struct vm_page  *vm_page_t;
 166
 167 #define prep_count      prep_pin_u.pp_counts.prep
 168 #define pin_count       prep_pin_u.pp_counts.pin
 169 #define prep_pin_count  prep_pin_u.pp_both
 170
 171 #define VM_PAGE_NULL            ((vm_page_t) 0)
 172 #define NEXT_PAGE(m)    ((vm_page_t) (m)->pageq.next)
 173
 174 /*
 175  * XXX  The unusual bit should not be necessary.  Most of the bit
 176  * XXX  fields above really want to be masks.
 177  */
 178
 179 /*
 180  *      For debugging, this macro can be defined to perform
 181  *      some useful check on a page structure.
 182  */
 183
 184 #define VM_PAGE_CHECK(mem)
 185
 186 /*
 187  *      Each pageable resident page falls into one of three lists:
 188  *
 189  *      free
 190  *              Available for allocation now.
 191  *      inactive
 192  *              Not referenced in any map, but still has an
 193  *              object/offset-page mapping, and may be dirty.
 194  *              This is the list of pages that should be
 195  *              paged out next.
 196  *      active
 197  *              A list of pages which have been placed in
 198  *              at least one physical map.  This list is
 199  *              ordered, in LRU-like fashion.
 200  */
 201
 202 extern
 203 vm_page_t       vm_page_queue_free;     /* memory free queue */
 204 extern
 205 vm_page_t       vm_page_queue_fictitious;       /* fictitious free queue */
 206 extern
 207 queue_head_t    vm_page_queue_active;   /* active memory queue */
 208 extern
 209 queue_head_t    vm_page_queue_inactive; /* inactive memory queue */
 210
 211 extern
 212 vm_offset_t     first_phys_addr;        /* physical address for first_page */
 213 extern
 214 vm_offset_t     last_phys_addr;         /* physical address for last_page */
 215
 216 extern
 217 int     vm_page_free_count;     /* How many pages are free? */
 218 extern
 219 int     vm_page_fictitious_count;/* How many fictitious pages are free? */
 220 extern
 221 int     vm_page_active_count;   /* How many pages are active? */
 222 extern
 223 int     vm_page_inactive_count; /* How many pages are inactive? */
 224 extern
 225 int     vm_page_wire_count;     /* How many pages are wired? */
 226 extern
 227 int     vm_page_free_target;    /* How many do we want free? */
 228 extern
 229 int     vm_page_free_min;       /* When to wakeup pageout */
 230 extern
 231 int     vm_page_inactive_target;/* How many do we want inactive? */
 232 extern
 233 int     vm_page_free_reserved;  /* How many pages reserved to do pageout */
 234 extern
 235 int     vm_page_laundry_count;  /* How many pages being laundered? */
 236
 237 decl_mutex_data(,vm_page_queue_lock)
 238                                 /* lock on active and inactive page queues */
 239 decl_mutex_data(,vm_page_queue_free_lock)
 240                                 /* lock on free page queue */
 241 decl_simple_lock_data(extern,vm_page_preppin_lock)      /* lock for prep/pin */
 242
 243 extern unsigned int     vm_page_free_wanted;
 244                                 /* how many threads are waiting for memory */
 245
 246 extern vm_offset_t      vm_page_fictitious_addr;
 247                                 /* (fake) phys_addr of fictitious pages */
 248
 249 /*
 250  * Prototypes for functions exported by this module.
 251  */
 252 extern void             vm_page_bootstrap(
 253                                         vm_offset_t     *startp,
 254                                         vm_offset_t     *endp);
 255
 256 extern void             vm_page_module_init(void);
 257
 258 extern void             vm_page_create(
 259                                         vm_offset_t     start,
 260                                         vm_offset_t     end);
 261
 262 extern vm_page_t        vm_page_lookup(
 263                                         vm_object_t             object,
 264                                         vm_object_offset_t      offset);
 265
 266 extern vm_page_t        vm_page_grab_fictitious(void);
 267
 268 extern void             vm_page_release_fictitious(
 269                                         vm_page_t page);
 270
 271 extern boolean_t        vm_page_convert(
 272                                         vm_page_t       page);
 273
 274 extern void             vm_page_more_fictitious(void);
 275
 276 extern int              vm_pool_low(void);
 277
 278 extern vm_page_t        vm_page_grab(void);
 279
 280 extern void             vm_page_release(
 281                                         vm_page_t       page);
 282
 283 extern void             vm_page_release_limbo(
 284                                         vm_page_t       page);
 285
 286 extern void             vm_page_limbo_exchange(
 287                                         vm_page_t       limbo_m,
 288                                         vm_page_t       new_m);
 289
 290 extern boolean_t        vm_page_wait(
 291                                         int             interruptible );
 292
 293 extern vm_page_t        vm_page_alloc(
 294                                         vm_object_t             object,
 295                                         vm_object_offset_t      offset);
 296
 297 extern void             vm_page_init(
 298                                         vm_page_t       page,
 299                                         vm_offset_t     phys_addr);
 300
 301 extern void             vm_page_free(
 302                                         vm_page_t       page);
 303
 304 extern void             vm_page_activate(
 305                                         vm_page_t       page);
 306
 307 extern void             vm_page_deactivate(
 308                                         vm_page_t       page);
 309
 310 extern void             vm_page_rename(
 311                                         vm_page_t               page,
 312                                         vm_object_t             new_object,
 313                                         vm_object_offset_t      new_offset);
 314
 315 extern void             vm_page_insert(
 316                                         vm_page_t               page,
 317                                         vm_object_t             object,
 318                                         vm_object_offset_t      offset);
 319
 320 extern void             vm_page_replace(
 321                                         vm_page_t               mem,
 322                                         vm_object_t             object,
 323                                         vm_object_offset_t      offset);
 324
 325 extern void             vm_page_remove(
 326                                         vm_page_t       page);
 327
 328 extern void             vm_page_zero_fill(
 329                                         vm_page_t       page);
 330
 331 extern void             vm_page_part_zero_fill(
 332                                         vm_page_t       m,
 333                                         vm_offset_t     m_pa,
 334                                         vm_size_t       len);
 335
 336 extern void             vm_page_copy(
 337                                         vm_page_t       src_page,
 338                                         vm_page_t       dest_page);
 339
 340 extern void             vm_page_part_copy(
 341                                         vm_page_t       src_m,
 342                                         vm_offset_t     src_pa,
 343                                         vm_page_t       dst_m,
 344                                         vm_offset_t     dst_pa,
 345                                         vm_size_t       len);
 346
 347 extern void             vm_page_wire(
 348                                         vm_page_t       page);
 349
 350 extern void             vm_page_unwire(
 351                                         vm_page_t       page);
 352
 353 extern void             vm_set_page_size(void);
 354
 355 extern void             vm_page_gobble(
 356                                         vm_page_t      page);
 357
 358 extern kern_return_t    vm_page_prep(
 359                                         vm_page_t       m);
 360
 361 extern kern_return_t    vm_page_pin(
 362                                         vm_page_t       m);
 363
 364 extern kern_return_t    vm_page_unprep(
 365                                         vm_page_t       m);
 366
 367 extern kern_return_t    vm_page_unpin(
 368                                         vm_page_t       m);
 369
 370 extern void             cleanup_limbo_queue(void);
 371
 372 /*
 373  *      Functions implemented as macros. m->wanted and m->busy are
 374  *      protected by the object lock.
 375  */
 376
 377 #define PAGE_ASSERT_WAIT(m, interruptible)                      \
 378                 MACRO_BEGIN                                     \
 379                 (m)->wanted = TRUE;                             \
 380                 assert_wait((event_t) (m), (interruptible));    \
 381                 MACRO_END
 382
 383 #define PAGE_WAKEUP_DONE(m)                                     \
 384                 MACRO_BEGIN                                     \
 385                 (m)->busy = FALSE;                              \
 386                 if ((m)->wanted) {                              \
 387                         (m)->wanted = FALSE;                    \
 388                         thread_wakeup((event_t) (m));           \
 389                 }                                               \
 390                 MACRO_END
 391
 392 #define PAGE_WAKEUP(m)                                          \
 393                 MACRO_BEGIN                                     \
 394                 if ((m)->wanted) {                              \
 395                         (m)->wanted = FALSE;                    \
 396                         thread_wakeup((event_t) (m));           \
 397                 }                                               \
 398                 MACRO_END
 399
 400 #define VM_PAGE_FREE(p)                         \
 401                 MACRO_BEGIN                     \
 402                 vm_page_lock_queues();          \
 403                 vm_page_free(p);                \
 404                 vm_page_unlock_queues();        \
 405                 MACRO_END
 406
 407 #define VM_PAGE_GRAB_FICTITIOUS(M)                                      \
 408                 MACRO_BEGIN                                             \
 409                 while ((M = vm_page_grab_fictitious()) == VM_PAGE_NULL) \
 410                         vm_page_more_fictitious();                      \
 411                 MACRO_END
 412
 413 #define VM_PAGE_THROTTLED()                                             \
 414                 (vm_page_free_count < (vm_page_free_target -            \
 415                  ((vm_page_free_target-vm_page_free_min)>>2)))
 416
 417 #define VM_PAGE_WAIT()          ((void)vm_page_wait(THREAD_UNINT))
 418
 419 #define vm_page_lock_queues()   mutex_lock(&vm_page_queue_lock)
 420 #define vm_page_unlock_queues() mutex_unlock(&vm_page_queue_lock)
 421 #define vm_page_pin_lock()      simple_lock(&vm_page_preppin_lock)
 422 #define vm_page_pin_unlock()    simple_unlock(&vm_page_preppin_lock)
 423
 424 #define VM_PAGE_QUEUES_REMOVE(mem)                              \
 425         MACRO_BEGIN                                             \
 426         if (mem->active) {                                      \
 427                 assert(!mem->inactive);                         \
 428                 queue_remove(&vm_page_queue_active,             \
 429                         mem, vm_page_t, pageq);                 \
 430                 mem->active = FALSE;                            \
 431                 if (!mem->fictitious)                           \
 432                         vm_page_active_count--;                 \
 433         }                                                       \
 434                                                                 \
 435         if (mem->inactive) {                                    \
 436                 assert(!mem->active);                           \
 437                 queue_remove(&vm_page_queue_inactive,           \
 438                         mem, vm_page_t, pageq);                 \
 439                 mem->inactive = FALSE;                          \
 440                 if (!mem->fictitious)                           \
 441                         vm_page_inactive_count--;               \
 442         }                                                       \
 443         MACRO_END
 444
 445 #endif  /* _VM_VM_PAGE_H_ */