osfmk/vm/vm_pageout.h

   1 /*
   2  * Copyright (c) 2000-2009 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  */
  31 /*
  32  * Mach Operating System
  33  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
  34  * All Rights Reserved.
  35  *
  36  * Permission to use, copy, modify and distribute this software and its
  37  * documentation is hereby granted, provided that both the copyright
  38  * notice and this permission notice appear in all copies of the
  39  * software, derivative works or modified versions, and any portions
  40  * thereof, and that both notices appear in supporting documentation.
  41  *
  42  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  43  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  44  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  45  *
  46  * Carnegie Mellon requests users of this software to return to
  47  *
  48  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  49  *  School of Computer Science
  50  *  Carnegie Mellon University
  51  *  Pittsburgh PA 15213-3890
  52  *
  53  * any improvements or extensions that they make and grant Carnegie Mellon
  54  * the rights to redistribute these changes.
  55  */
  56 /*
  57  */
  58 /*
  59  *      File:   vm/vm_pageout.h
  60  *      Author: Avadis Tevanian, Jr.
  61  *      Date:   1986
  62  *
  63  *      Declarations for the pageout daemon interface.
  64  */
  65
  66 #ifndef _VM_VM_PAGEOUT_H_
  67 #define _VM_VM_PAGEOUT_H_
  68
  69 #ifdef  KERNEL_PRIVATE
  70
  71 #include <mach/mach_types.h>
  72 #include <mach/boolean.h>
  73 #include <mach/machine/vm_types.h>
  74 #include <mach/memory_object_types.h>
  75
  76 #include <kern/kern_types.h>
  77 #include <kern/lock.h>
  78
  79 #include <libkern/OSAtomic.h>
  80
  81
  82 #include <vm/vm_options.h>
  83
  84 #ifdef  MACH_KERNEL_PRIVATE
  85 #include <vm/vm_page.h>
  86 #endif
  87
  88
  89 extern kern_return_t vm_map_create_upl(
  90         vm_map_t                map,
  91         vm_map_address_t        offset,
  92         upl_size_t              *upl_size,
  93         upl_t                   *upl,
  94         upl_page_info_array_t   page_list,
  95         unsigned int            *count,
  96         int                     *flags);
  97
  98 extern ppnum_t upl_get_highest_page(
  99         upl_t                   upl);
 100
 101 extern upl_size_t upl_get_size(
 102         upl_t                   upl);
 103
 104
 105 #ifndef MACH_KERNEL_PRIVATE
 106 typedef struct vm_page  *vm_page_t;
 107 #endif
 108
 109
 110 extern void                vm_page_free_list(
 111                             vm_page_t   mem,
 112                             boolean_t   prepare_object);
 113
 114 extern kern_return_t      vm_page_alloc_list(
 115                             int         page_count,
 116                             int                 flags,
 117                             vm_page_t * list);
 118
 119 extern void               vm_page_set_offset(vm_page_t page, vm_object_offset_t offset);
 120 extern vm_object_offset_t vm_page_get_offset(vm_page_t page);
 121 extern ppnum_t            vm_page_get_phys_page(vm_page_t page);
 122 extern vm_page_t          vm_page_get_next(vm_page_t page);
 123
 124 #ifdef  MACH_KERNEL_PRIVATE
 125
 126 #include <vm/vm_page.h>
 127
 128 extern unsigned int     vm_pageout_scan_event_counter;
 129 extern unsigned int     vm_zf_queue_count;
 130
 131
 132 #if defined(__ppc__) /* On ppc, vm statistics are still 32-bit */
 133
 134 extern unsigned int     vm_zf_count;
 135
 136 #define VM_ZF_COUNT_INCR()                              \
 137         MACRO_BEGIN                                     \
 138         OSAddAtomic(1, (SInt32 *) &vm_zf_count);        \
 139         MACRO_END                                       \
 140
 141 #define VM_ZF_COUNT_DECR()                              \
 142         MACRO_BEGIN                                     \
 143         OSAddAtomic(-1, (SInt32 *) &vm_zf_count);       \
 144         MACRO_END                                       \
 145
 146 #else /* !(defined(__ppc__)) */
 147
 148 extern uint64_t vm_zf_count;
 149
 150 #define VM_ZF_COUNT_INCR()                              \
 151         MACRO_BEGIN                                     \
 152         OSAddAtomic64(1, (SInt64 *) &vm_zf_count);      \
 153         MACRO_END                                       \
 154
 155 #define VM_ZF_COUNT_DECR()                              \
 156         MACRO_BEGIN                                     \
 157         OSAddAtomic64(-1, (SInt64 *) &vm_zf_count);     \
 158         MACRO_END                                       \
 159
 160 #endif /* !(defined(__ppc__)) */
 161
 162 /*
 163  * must hold the page queues lock to
 164  * manipulate this structure
 165  */
 166 struct vm_pageout_queue {
 167         queue_head_t    pgo_pending;    /* laundry pages to be processed by pager's iothread */
 168         unsigned int    pgo_laundry;    /* current count of laundry pages on queue or in flight */
 169         unsigned int    pgo_maxlaundry;
 170
 171         unsigned int    pgo_idle:1,     /* iothread is blocked waiting for work to do */
 172                         pgo_busy:1,     /* iothread is currently processing request from pgo_pending */
 173                         pgo_throttled:1,/* vm_pageout_scan thread needs a wakeup when pgo_laundry drops */
 174                         pgo_draining:1,
 175                         :0;
 176 };
 177
 178 #define VM_PAGE_Q_THROTTLED(q)          \
 179         ((q)->pgo_laundry >= (q)->pgo_maxlaundry)
 180
 181 extern struct   vm_pageout_queue        vm_pageout_queue_internal;
 182 extern struct   vm_pageout_queue        vm_pageout_queue_external;
 183
 184 /*
 185  *      Routines exported to Mach.
 186  */
 187 extern void             vm_pageout(void);
 188
 189 extern kern_return_t    vm_pageout_internal_start(void);
 190
 191 extern void             vm_pageout_object_terminate(
 192                                         vm_object_t     object);
 193
 194 extern void             vm_pageout_cluster(
 195                                         vm_page_t       m);
 196
 197 extern void             vm_pageout_initialize_page(
 198                                         vm_page_t       m);
 199
 200 extern void             vm_pageclean_setup(
 201                                         vm_page_t               m,
 202                                         vm_page_t               new_m,
 203                                         vm_object_t             new_object,
 204                                         vm_object_offset_t      new_offset);
 205
 206 /* UPL exported routines and structures */
 207
 208 #define upl_lock_init(object)   lck_mtx_init(&(object)->Lock, &vm_object_lck_grp, &vm_object_lck_attr)
 209 #define upl_lock_destroy(object)        lck_mtx_destroy(&(object)->Lock, &vm_object_lck_grp)
 210 #define upl_lock(object)        lck_mtx_lock(&(object)->Lock)
 211 #define upl_unlock(object)      lck_mtx_unlock(&(object)->Lock)
 212
 213 #define MAX_VECTOR_UPL_ELEMENTS 8
 214
 215 struct _vector_upl_iostates{
 216         upl_offset_t offset;
 217         upl_size_t   size;
 218 };
 219
 220 typedef struct _vector_upl_iostates vector_upl_iostates_t;
 221
 222 struct _vector_upl {
 223         upl_size_t              size;
 224         uint32_t                num_upls;
 225         uint32_t                invalid_upls;
 226         uint32_t                _reserved;
 227         vm_map_t                submap;
 228         vm_offset_t             submap_dst_addr;
 229         vm_object_offset_t      offset;
 230         upl_t                   upl_elems[MAX_VECTOR_UPL_ELEMENTS];
 231         upl_page_info_array_t   pagelist;
 232         vector_upl_iostates_t   upl_iostates[MAX_VECTOR_UPL_ELEMENTS];
 233 };
 234
 235 typedef struct _vector_upl* vector_upl_t;
 236
 237 /* universal page list structure */
 238
 239 #if UPL_DEBUG
 240 #define UPL_DEBUG_STACK_FRAMES  16
 241 #define UPL_DEBUG_COMMIT_RECORDS 4
 242
 243 struct ucd {
 244         upl_offset_t    c_beg;
 245         upl_offset_t    c_end;
 246         int             c_aborted;
 247         void *          c_retaddr[UPL_DEBUG_STACK_FRAMES];
 248 };
 249 #endif
 250
 251
 252 struct upl {
 253         decl_lck_mtx_data(,     Lock)   /* Synchronization */
 254         int             ref_count;
 255         int             flags;
 256         vm_object_t     src_object; /* object derived from */
 257         vm_object_offset_t offset;
 258         upl_size_t      size;       /* size in bytes of the address space */
 259         vm_offset_t     kaddr;      /* secondary mapping in kernel */
 260         vm_object_t     map_object;
 261         ppnum_t         highest_page;
 262         void*           vector_upl;
 263 #if     UPL_DEBUG
 264         uintptr_t       ubc_alias1;
 265         uintptr_t       ubc_alias2;
 266         queue_chain_t   uplq;       /* List of outstanding upls on an obj */
 267
 268         thread_t        upl_creator;
 269         uint32_t        upl_state;
 270         uint32_t        upl_commit_index;
 271         void    *upl_create_retaddr[UPL_DEBUG_STACK_FRAMES];
 272
 273         struct  ucd     upl_commit_records[UPL_DEBUG_COMMIT_RECORDS];
 274 #endif  /* UPL_DEBUG */
 275 };
 276
 277 /* upl struct flags */
 278 #define UPL_PAGE_LIST_MAPPED    0x1
 279 #define UPL_KERNEL_MAPPED       0x2
 280 #define UPL_CLEAR_DIRTY         0x4
 281 #define UPL_COMPOSITE_LIST      0x8
 282 #define UPL_INTERNAL            0x10
 283 #define UPL_PAGE_SYNC_DONE      0x20
 284 #define UPL_DEVICE_MEMORY       0x40
 285 #define UPL_PAGEOUT             0x80
 286 #define UPL_LITE                0x100
 287 #define UPL_IO_WIRE             0x200
 288 #define UPL_ACCESS_BLOCKED      0x400
 289 #define UPL_ENCRYPTED           0x800
 290 #define UPL_SHADOWED            0x1000
 291 #define UPL_KERNEL_OBJECT       0x2000
 292 #define UPL_VECTOR              0x4000
 293 #define UPL_HAS_BUSY            0x10000
 294
 295 /* flags for upl_create flags parameter */
 296 #define UPL_CREATE_EXTERNAL     0
 297 #define UPL_CREATE_INTERNAL     0x1
 298 #define UPL_CREATE_LITE         0x2
 299
 300 extern upl_t vector_upl_create(vm_offset_t);
 301 extern void vector_upl_deallocate(upl_t);
 302 extern boolean_t vector_upl_is_valid(upl_t);
 303 extern boolean_t vector_upl_set_subupl(upl_t, upl_t, u_int32_t);
 304 extern void vector_upl_set_pagelist(upl_t);
 305 extern void vector_upl_set_submap(upl_t, vm_map_t, vm_offset_t);
 306 extern void vector_upl_get_submap(upl_t, vm_map_t*, vm_offset_t*);
 307 extern void vector_upl_set_iostate(upl_t, upl_t, upl_offset_t, upl_size_t);
 308 extern void vector_upl_get_iostate(upl_t, upl_t, upl_offset_t*, upl_size_t*);
 309 extern void vector_upl_get_iostate_byindex(upl_t, uint32_t, upl_offset_t*, upl_size_t*);
 310 extern upl_t vector_upl_subupl_byindex(upl_t , uint32_t);
 311 extern upl_t vector_upl_subupl_byoffset(upl_t , upl_offset_t*, upl_size_t*);
 312
 313 extern kern_return_t vm_object_iopl_request(
 314         vm_object_t             object,
 315         vm_object_offset_t      offset,
 316         upl_size_t              size,
 317         upl_t                   *upl_ptr,
 318         upl_page_info_array_t   user_page_list,
 319         unsigned int            *page_list_count,
 320         int                     cntrl_flags);
 321
 322 extern kern_return_t vm_object_super_upl_request(
 323         vm_object_t             object,
 324         vm_object_offset_t      offset,
 325         upl_size_t              size,
 326         upl_size_t              super_cluster,
 327         upl_t                   *upl,
 328         upl_page_info_t         *user_page_list,
 329         unsigned int            *page_list_count,
 330         int                     cntrl_flags);
 331
 332 /* should be just a regular vm_map_enter() */
 333 extern kern_return_t vm_map_enter_upl(
 334         vm_map_t                map,
 335         upl_t                   upl,
 336         vm_map_offset_t         *dst_addr);
 337
 338 /* should be just a regular vm_map_remove() */
 339 extern kern_return_t vm_map_remove_upl(
 340         vm_map_t                map,
 341         upl_t                   upl);
 342
 343 /* wired  page list structure */
 344 typedef uint32_t *wpl_array_t;
 345
 346 extern void vm_page_free_reserve(int pages);
 347
 348 extern void vm_pageout_throttle_down(vm_page_t page);
 349 extern void vm_pageout_throttle_up(vm_page_t page);
 350
 351 /*
 352  * ENCRYPTED SWAP:
 353  */
 354 extern void upl_encrypt(
 355         upl_t                   upl,
 356         upl_offset_t            crypt_offset,
 357         upl_size_t              crypt_size);
 358 extern void vm_page_encrypt(
 359         vm_page_t               page,
 360         vm_map_offset_t         kernel_map_offset);
 361 extern boolean_t vm_pages_encrypted; /* are there encrypted pages ? */
 362 extern void vm_page_decrypt(
 363         vm_page_t               page,
 364         vm_map_offset_t         kernel_map_offset);
 365 extern kern_return_t vm_paging_map_object(
 366         vm_map_offset_t         *address,
 367         vm_page_t               page,
 368         vm_object_t             object,
 369         vm_object_offset_t      offset,
 370         vm_map_size_t           *size,
 371         vm_prot_t               protection,
 372         boolean_t               can_unlock_object);
 373 extern void vm_paging_unmap_object(
 374         vm_object_t             object,
 375         vm_map_offset_t         start,
 376         vm_map_offset_t         end);
 377 decl_simple_lock_data(extern, vm_paging_lock)
 378
 379 /*
 380  * Backing store throttle when BS is exhausted
 381  */
 382 extern unsigned int    vm_backing_store_low;
 383
 384 extern void vm_pageout_queue_steal(
 385         vm_page_t page,
 386         boolean_t queues_locked);
 387
 388 #endif  /* MACH_KERNEL_PRIVATE */
 389
 390 #if UPL_DEBUG
 391 extern kern_return_t  upl_ubc_alias_set(
 392         upl_t upl,
 393         uintptr_t alias1,
 394         uintptr_t alias2);
 395 extern int  upl_ubc_alias_get(
 396         upl_t upl,
 397         uintptr_t * al,
 398         uintptr_t * al2);
 399 #endif /* UPL_DEBUG */
 400
 401 extern void vm_countdirtypages(void);
 402
 403 extern void vm_backing_store_disable(
 404                         boolean_t       suspend);
 405
 406 extern kern_return_t upl_transpose(
 407         upl_t   upl1,
 408         upl_t   upl2);
 409
 410 extern kern_return_t mach_vm_pressure_monitor(
 411         boolean_t       wait_for_pressure,
 412         unsigned int    nsecs_monitored,
 413         unsigned int    *pages_reclaimed_p,
 414         unsigned int    *pages_wanted_p);
 415
 416 extern kern_return_t
 417 vm_set_buffer_cleanup_callout(
 418         boolean_t       (*func)(int));
 419
 420 struct vm_page_stats_reusable {
 421         SInt32          reusable_count;
 422         uint64_t        reusable;
 423         uint64_t        reused;
 424         uint64_t        reused_wire;
 425         uint64_t        reused_remove;
 426         uint64_t        all_reusable_calls;
 427         uint64_t        partial_reusable_calls;
 428         uint64_t        all_reuse_calls;
 429         uint64_t        partial_reuse_calls;
 430         uint64_t        reusable_pages_success;
 431         uint64_t        reusable_pages_failure;
 432         uint64_t        reusable_pages_shared;
 433         uint64_t        reuse_pages_success;
 434         uint64_t        reuse_pages_failure;
 435         uint64_t        can_reuse_success;
 436         uint64_t        can_reuse_failure;
 437 };
 438 extern struct vm_page_stats_reusable vm_page_stats_reusable;
 439
 440 extern int hibernate_flush_memory(void);
 441
 442 #endif  /* KERNEL_PRIVATE */
 443
 444 #endif  /* _VM_VM_PAGEOUT_H_ */