osfmk/vm/vm_pageout.h

   1 /*
   2  * Copyright (c) 2000-2009 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  */
  31 /*
  32  * Mach Operating System
  33  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
  34  * All Rights Reserved.
  35  *
  36  * Permission to use, copy, modify and distribute this software and its
  37  * documentation is hereby granted, provided that both the copyright
  38  * notice and this permission notice appear in all copies of the
  39  * software, derivative works or modified versions, and any portions
  40  * thereof, and that both notices appear in supporting documentation.
  41  *
  42  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  43  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  44  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  45  *
  46  * Carnegie Mellon requests users of this software to return to
  47  *
  48  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  49  *  School of Computer Science
  50  *  Carnegie Mellon University
  51  *  Pittsburgh PA 15213-3890
  52  *
  53  * any improvements or extensions that they make and grant Carnegie Mellon
  54  * the rights to redistribute these changes.
  55  */
  56 /*
  57  */
  58 /*
  59  *      File:   vm/vm_pageout.h
  60  *      Author: Avadis Tevanian, Jr.
  61  *      Date:   1986
  62  *
  63  *      Declarations for the pageout daemon interface.
  64  */
  65
  66 #ifndef _VM_VM_PAGEOUT_H_
  67 #define _VM_VM_PAGEOUT_H_
  68
  69 #ifdef  KERNEL_PRIVATE
  70
  71 #include <mach/mach_types.h>
  72 #include <mach/boolean.h>
  73 #include <mach/machine/vm_types.h>
  74 #include <mach/memory_object_types.h>
  75
  76 #include <kern/kern_types.h>
  77 #include <kern/locks.h>
  78
  79 #include <libkern/OSAtomic.h>
  80
  81
  82 #include <vm/vm_options.h>
  83
  84 #ifdef  MACH_KERNEL_PRIVATE
  85 #include <vm/vm_page.h>
  86 #endif
  87
  88 #include <sys/kdebug.h>
  89
  90 #define VM_PAGE_AVAILABLE_COUNT()               ((unsigned int)(vm_page_cleaned_count))
  91
  92 /* externally manipulated counters */
  93 extern unsigned int vm_pageout_cleaned_fault_reactivated;
  94
  95 #if CONFIG_FREEZE
  96 extern boolean_t memorystatus_freeze_enabled;
  97 #endif
  98
  99 #define VM_DYNAMIC_PAGING_ENABLED() (VM_CONFIG_COMPRESSOR_IS_ACTIVE)
 100
 101 #if VM_PRESSURE_EVENTS
 102 extern boolean_t vm_pressure_events_enabled;
 103 #endif /* VM_PRESSURE_EVENTS */
 104
 105
 106 /*
 107  * the following codes are used in the DBG_MACH_WORKINGSET subclass
 108  * of the DBG_MACH class
 109  */
 110 #define VM_DISCONNECT_ALL_PAGE_MAPPINGS         0x00
 111 #define VM_DISCONNECT_TASK_PAGE_MAPPINGS        0x01
 112 #define VM_REAL_FAULT_ADDR_INTERNAL             0x02
 113 #define VM_REAL_FAULT_ADDR_PURGABLE             0x03
 114 #define VM_REAL_FAULT_ADDR_EXTERNAL             0x04
 115 #define VM_REAL_FAULT_ADDR_SHAREDCACHE          0x05
 116
 117
 118
 119 extern int      vm_debug_events;
 120
 121 #define VMF_CHECK_ZFDELAY               0x100
 122 #define VMF_COWDELAY                    0x101
 123 #define VMF_ZFDELAY                     0x102
 124 #define VMF_COMPRESSORDELAY             0x103
 125
 126 #define VM_PAGEOUT_SCAN                 0x104
 127 #define VM_PAGEOUT_BALANCE              0x105
 128 #define VM_PAGEOUT_FREELIST             0x106
 129 #define VM_PAGEOUT_PURGEONE             0x107
 130 #define VM_PAGEOUT_CACHE_EVICT          0x108
 131 #define VM_PAGEOUT_THREAD_BLOCK         0x109
 132 #define VM_PAGEOUT_JETSAM               0x10A
 133 #define VM_INFO1                        0x10B
 134 #define VM_INFO2                        0x10C
 135 #define VM_INFO3                        0x10D
 136 #define VM_INFO4                        0x10E
 137 #define VM_INFO5                        0x10F
 138 #define VM_INFO6                        0x110
 139 #define VM_INFO7                        0x111
 140 #define VM_INFO8                        0x112
 141 #define VM_INFO9                        0x113
 142
 143 #define VM_UPL_PAGE_WAIT                0x120
 144 #define VM_IOPL_PAGE_WAIT               0x121
 145 #define VM_PAGE_WAIT_BLOCK              0x122
 146
 147 #if CONFIG_IOSCHED
 148 #define VM_PAGE_SLEEP                   0x123
 149 #define VM_PAGE_EXPEDITE                0x124
 150 #define VM_PAGE_EXPEDITE_NO_MEMORY      0x125
 151 #endif
 152
 153 #define VM_PAGE_GRAB                    0x126
 154 #define VM_PAGE_RELEASE                 0x127
 155
 156 #define VM_PRESSURE_EVENT               0x130
 157 #define VM_EXECVE                       0x131
 158 #define VM_WAKEUP_COMPACTOR_SWAPPER     0x132
 159 #define VM_UPL_REQUEST                  0x133
 160 #define VM_IOPL_REQUEST                 0x134
 161 #define VM_KERN_REQUEST                 0x135
 162
 163 #define VM_DATA_WRITE                   0x140
 164
 165 #define VM_PRESSURE_LEVEL_CHANGE        0x141
 166
 167 #define VM_DEBUG_EVENT(name, event, control, arg1, arg2, arg3, arg4)    \
 168         MACRO_BEGIN                                             \
 169         if (__improbable(vm_debug_events)) {                    \
 170                 KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, event)) | control, arg1, arg2, arg3, arg4, 0); \
 171         }                                                       \
 172         MACRO_END
 173
 174 #define VM_DEBUG_CONSTANT_EVENT(name, event, control, arg1, arg2, arg3, arg4)   \
 175         MACRO_BEGIN                                             \
 176                 KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, event)) | control, arg1, arg2, arg3, arg4, 0); \
 177         MACRO_END
 178
 179 extern void memoryshot(unsigned int event, unsigned int control);
 180
 181 extern void update_vm_info(void);
 182
 183 #if CONFIG_IOSCHED
 184 extern int upl_get_cached_tier(
 185        upl_t                   upl);
 186 #endif
 187
 188 extern void upl_set_iodone(upl_t, void *);
 189 extern void upl_set_iodone_error(upl_t, int);
 190 extern void upl_callout_iodone(upl_t);
 191
 192 extern ppnum_t upl_get_highest_page(
 193         upl_t                   upl);
 194
 195 extern upl_size_t upl_get_size(
 196         upl_t                   upl);
 197
 198 extern upl_t upl_associated_upl(upl_t upl);
 199 extern void upl_set_associated_upl(upl_t upl, upl_t associated_upl);
 200
 201 #ifdef  XNU_KERNEL_PRIVATE
 202
 203 extern kern_return_t vm_map_create_upl(
 204         vm_map_t                map,
 205         vm_map_address_t        offset,
 206         upl_size_t              *upl_size,
 207         upl_t                   *upl,
 208         upl_page_info_array_t   page_list,
 209         unsigned int            *count,
 210         upl_control_flags_t     *flags,
 211         vm_tag_t            tag);
 212
 213 extern void iopl_valid_data(
 214         upl_t                   upl_ptr,
 215         vm_tag_t        tag);
 216
 217 #endif  /* XNU_KERNEL_PRIVATE */
 218
 219 extern struct vnode * upl_lookup_vnode(upl_t upl);
 220
 221 #ifndef MACH_KERNEL_PRIVATE
 222 typedef struct vm_page  *vm_page_t;
 223 #endif
 224
 225 extern void                vm_page_free_list(
 226                             vm_page_t   mem,
 227                             boolean_t   prepare_object);
 228
 229 extern kern_return_t      vm_page_alloc_list(
 230                             int         page_count,
 231                             int                 flags,
 232                             vm_page_t * list);
 233
 234 extern void               vm_page_set_offset(vm_page_t page, vm_object_offset_t offset);
 235 extern vm_object_offset_t vm_page_get_offset(vm_page_t page);
 236 extern ppnum_t            vm_page_get_phys_page(vm_page_t page);
 237 extern vm_page_t          vm_page_get_next(vm_page_t page);
 238
 239 extern kern_return_t    mach_vm_pressure_level_monitor(boolean_t wait_for_pressure, unsigned int *pressure_level);
 240
 241 #if !CONFIG_EMBEDDED
 242 extern kern_return_t    vm_pageout_wait(uint64_t deadline);
 243 #endif
 244
 245 #ifdef  MACH_KERNEL_PRIVATE
 246
 247 #include <vm/vm_page.h>
 248
 249 extern unsigned int     vm_pageout_scan_event_counter;
 250 extern unsigned int     vm_page_anonymous_count;
 251
 252
 253 /*
 254  * must hold the page queues lock to
 255  * manipulate this structure
 256  */
 257 struct vm_pageout_queue {
 258         vm_page_queue_head_t    pgo_pending;    /* laundry pages to be processed by pager's iothread */
 259         unsigned int    pgo_laundry;    /* current count of laundry pages on queue or in flight */
 260         unsigned int    pgo_maxlaundry;
 261         uint64_t        pgo_tid;        /* thread ID of I/O thread that services this queue */
 262         uint8_t         pgo_lowpriority; /* iothread is set to use low priority I/O */
 263
 264         unsigned int    pgo_idle:1,     /* iothread is blocked waiting for work to do */
 265                         pgo_busy:1,     /* iothread is currently processing request from pgo_pending */
 266                         pgo_throttled:1,/* vm_pageout_scan thread needs a wakeup when pgo_laundry drops */
 267                         pgo_draining:1,
 268                         pgo_inited:1,
 269                         :0;
 270 };
 271
 272 #define VM_PAGE_Q_THROTTLED(q)          \
 273         ((q)->pgo_laundry >= (q)->pgo_maxlaundry)
 274
 275 extern struct   vm_pageout_queue        vm_pageout_queue_internal;
 276 extern struct   vm_pageout_queue        vm_pageout_queue_external;
 277
 278
 279 /*
 280  *      Routines exported to Mach.
 281  */
 282 extern void             vm_pageout(void);
 283
 284 extern kern_return_t    vm_pageout_internal_start(void);
 285
 286 extern void             vm_pageout_object_terminate(
 287                                         vm_object_t     object);
 288
 289 extern void             vm_pageout_cluster(
 290                                         vm_page_t       m);
 291
 292 extern void             vm_pageout_initialize_page(
 293                                         vm_page_t       m);
 294
 295 /* UPL exported routines and structures */
 296
 297 #define upl_lock_init(object)   lck_mtx_init(&(object)->Lock, &vm_object_lck_grp, &vm_object_lck_attr)
 298 #define upl_lock_destroy(object)        lck_mtx_destroy(&(object)->Lock, &vm_object_lck_grp)
 299 #define upl_lock(object)        lck_mtx_lock(&(object)->Lock)
 300 #define upl_unlock(object)      lck_mtx_unlock(&(object)->Lock)
 301 #define upl_try_lock(object)    lck_mtx_try_lock(&(object)->Lock)
 302
 303 #define MAX_VECTOR_UPL_ELEMENTS 8
 304
 305 struct _vector_upl_iostates{
 306         upl_offset_t offset;
 307         upl_size_t   size;
 308 };
 309
 310 typedef struct _vector_upl_iostates vector_upl_iostates_t;
 311
 312 struct _vector_upl {
 313         upl_size_t              size;
 314         uint32_t                num_upls;
 315         uint32_t                invalid_upls;
 316         uint32_t                _reserved;
 317         vm_map_t                submap;
 318         vm_offset_t             submap_dst_addr;
 319         vm_object_offset_t      offset;
 320         upl_t                   upl_elems[MAX_VECTOR_UPL_ELEMENTS];
 321         upl_page_info_array_t   pagelist;
 322         vector_upl_iostates_t   upl_iostates[MAX_VECTOR_UPL_ELEMENTS];
 323 };
 324
 325 typedef struct _vector_upl* vector_upl_t;
 326
 327 /* universal page list structure */
 328
 329 #if UPL_DEBUG
 330 #define UPL_DEBUG_STACK_FRAMES  16
 331 #define UPL_DEBUG_COMMIT_RECORDS 4
 332
 333 struct ucd {
 334         upl_offset_t    c_beg;
 335         upl_offset_t    c_end;
 336         int             c_aborted;
 337         void *          c_retaddr[UPL_DEBUG_STACK_FRAMES];
 338 };
 339 #endif
 340
 341 struct upl_io_completion {
 342
 343         void     *io_context;
 344         void     (*io_done)(void *, int);
 345
 346         int      io_error;
 347 };
 348
 349
 350 struct upl {
 351         decl_lck_mtx_data(,     Lock)   /* Synchronization */
 352         int             ref_count;
 353         int             ext_ref_count;
 354         int             flags;
 355         vm_object_offset_t offset;
 356         upl_size_t      size;       /* size in bytes of the address space */
 357         vm_offset_t     kaddr;      /* secondary mapping in kernel */
 358         vm_object_t     map_object;
 359         ppnum_t         highest_page;
 360         void*           vector_upl;
 361         upl_t           associated_upl;
 362         struct upl_io_completion *upl_iodone;
 363 #if CONFIG_IOSCHED
 364         int             upl_priority;
 365         uint64_t        *upl_reprio_info;
 366         void            *decmp_io_upl;
 367 #endif
 368 #if CONFIG_IOSCHED || UPL_DEBUG
 369         thread_t        upl_creator;
 370         queue_chain_t   uplq;       /* List of outstanding upls on an obj */
 371 #endif
 372 #if     UPL_DEBUG
 373         uintptr_t       ubc_alias1;
 374         uintptr_t       ubc_alias2;
 375
 376         uint32_t        upl_state;
 377         uint32_t        upl_commit_index;
 378         void    *upl_create_retaddr[UPL_DEBUG_STACK_FRAMES];
 379
 380         struct  ucd     upl_commit_records[UPL_DEBUG_COMMIT_RECORDS];
 381 #endif  /* UPL_DEBUG */
 382 };
 383
 384 /* upl struct flags */
 385 #define UPL_PAGE_LIST_MAPPED    0x1
 386 #define UPL_KERNEL_MAPPED       0x2
 387 #define UPL_CLEAR_DIRTY         0x4
 388 #define UPL_COMPOSITE_LIST      0x8
 389 #define UPL_INTERNAL            0x10
 390 #define UPL_PAGE_SYNC_DONE      0x20
 391 #define UPL_DEVICE_MEMORY       0x40
 392 #define UPL_PAGEOUT             0x80
 393 #define UPL_LITE                0x100
 394 #define UPL_IO_WIRE             0x200
 395 #define UPL_ACCESS_BLOCKED      0x400
 396 #define UPL_SHADOWED            0x1000
 397 #define UPL_KERNEL_OBJECT       0x2000
 398 #define UPL_VECTOR              0x4000
 399 #define UPL_SET_DIRTY           0x8000
 400 #define UPL_HAS_BUSY            0x10000
 401 #define UPL_TRACKED_BY_OBJECT   0x20000
 402 #define UPL_EXPEDITE_SUPPORTED  0x40000
 403 #define UPL_DECMP_REQ           0x80000
 404 #define UPL_DECMP_REAL_IO       0x100000
 405
 406 /* flags for upl_create flags parameter */
 407 #define UPL_CREATE_EXTERNAL     0
 408 #define UPL_CREATE_INTERNAL     0x1
 409 #define UPL_CREATE_LITE         0x2
 410 #define UPL_CREATE_IO_TRACKING  0x4
 411 #define UPL_CREATE_EXPEDITE_SUP 0x8
 412
 413 extern upl_t vector_upl_create(vm_offset_t);
 414 extern void vector_upl_deallocate(upl_t);
 415 extern boolean_t vector_upl_is_valid(upl_t);
 416 extern boolean_t vector_upl_set_subupl(upl_t, upl_t, u_int32_t);
 417 extern void vector_upl_set_pagelist(upl_t);
 418 extern void vector_upl_set_submap(upl_t, vm_map_t, vm_offset_t);
 419 extern void vector_upl_get_submap(upl_t, vm_map_t*, vm_offset_t*);
 420 extern void vector_upl_set_iostate(upl_t, upl_t, upl_offset_t, upl_size_t);
 421 extern void vector_upl_get_iostate(upl_t, upl_t, upl_offset_t*, upl_size_t*);
 422 extern void vector_upl_get_iostate_byindex(upl_t, uint32_t, upl_offset_t*, upl_size_t*);
 423 extern upl_t vector_upl_subupl_byindex(upl_t , uint32_t);
 424 extern upl_t vector_upl_subupl_byoffset(upl_t , upl_offset_t*, upl_size_t*);
 425
 426 extern void vm_object_set_pmap_cache_attr(
 427                 vm_object_t             object,
 428                 upl_page_info_array_t   user_page_list,
 429                 unsigned int            num_pages,
 430                 boolean_t               batch_pmap_op);
 431
 432 extern kern_return_t vm_object_iopl_request(
 433         vm_object_t             object,
 434         vm_object_offset_t      offset,
 435         upl_size_t              size,
 436         upl_t                   *upl_ptr,
 437         upl_page_info_array_t   user_page_list,
 438         unsigned int            *page_list_count,
 439         upl_control_flags_t     cntrl_flags,
 440         vm_tag_t            tag);
 441
 442 extern kern_return_t vm_object_super_upl_request(
 443         vm_object_t             object,
 444         vm_object_offset_t      offset,
 445         upl_size_t              size,
 446         upl_size_t              super_cluster,
 447         upl_t                   *upl,
 448         upl_page_info_t         *user_page_list,
 449         unsigned int            *page_list_count,
 450         upl_control_flags_t     cntrl_flags,
 451         vm_tag_t            tag);
 452
 453 /* should be just a regular vm_map_enter() */
 454 extern kern_return_t vm_map_enter_upl(
 455         vm_map_t                map,
 456         upl_t                   upl,
 457         vm_map_offset_t         *dst_addr);
 458
 459 /* should be just a regular vm_map_remove() */
 460 extern kern_return_t vm_map_remove_upl(
 461         vm_map_t                map,
 462         upl_t                   upl);
 463
 464 /* wired  page list structure */
 465 typedef uint32_t *wpl_array_t;
 466
 467 extern void vm_page_free_reserve(int pages);
 468
 469 extern void vm_pageout_throttle_down(vm_page_t page);
 470 extern void vm_pageout_throttle_up(vm_page_t page);
 471
 472 extern kern_return_t vm_paging_map_object(
 473         vm_page_t               page,
 474         vm_object_t             object,
 475         vm_object_offset_t      offset,
 476         vm_prot_t               protection,
 477         boolean_t               can_unlock_object,
 478         vm_map_size_t           *size,          /* IN/OUT */
 479         vm_map_offset_t         *address,       /* OUT */
 480         boolean_t               *need_unmap);   /* OUT */
 481 extern void vm_paging_unmap_object(
 482         vm_object_t             object,
 483         vm_map_offset_t         start,
 484         vm_map_offset_t         end);
 485 decl_simple_lock_data(extern, vm_paging_lock)
 486
 487 /*
 488  * Backing store throttle when BS is exhausted
 489  */
 490 extern unsigned int    vm_backing_store_low;
 491
 492 extern void vm_pageout_steal_laundry(
 493         vm_page_t page,
 494         boolean_t queues_locked);
 495
 496 #endif  /* MACH_KERNEL_PRIVATE */
 497
 498 #if UPL_DEBUG
 499 extern kern_return_t  upl_ubc_alias_set(
 500         upl_t upl,
 501         uintptr_t alias1,
 502         uintptr_t alias2);
 503 extern int  upl_ubc_alias_get(
 504         upl_t upl,
 505         uintptr_t * al,
 506         uintptr_t * al2);
 507 #endif /* UPL_DEBUG */
 508
 509 extern void vm_countdirtypages(void);
 510
 511 extern void vm_backing_store_disable(
 512                         boolean_t       suspend);
 513
 514 extern kern_return_t upl_transpose(
 515         upl_t   upl1,
 516         upl_t   upl2);
 517
 518 extern kern_return_t mach_vm_pressure_monitor(
 519         boolean_t       wait_for_pressure,
 520         unsigned int    nsecs_monitored,
 521         unsigned int    *pages_reclaimed_p,
 522         unsigned int    *pages_wanted_p);
 523
 524 extern kern_return_t
 525 vm_set_buffer_cleanup_callout(
 526         boolean_t       (*func)(int));
 527
 528 struct vm_page_stats_reusable {
 529         SInt32          reusable_count;
 530         uint64_t        reusable;
 531         uint64_t        reused;
 532         uint64_t        reused_wire;
 533         uint64_t        reused_remove;
 534         uint64_t        all_reusable_calls;
 535         uint64_t        partial_reusable_calls;
 536         uint64_t        all_reuse_calls;
 537         uint64_t        partial_reuse_calls;
 538         uint64_t        reusable_pages_success;
 539         uint64_t        reusable_pages_failure;
 540         uint64_t        reusable_pages_shared;
 541         uint64_t        reuse_pages_success;
 542         uint64_t        reuse_pages_failure;
 543         uint64_t        can_reuse_success;
 544         uint64_t        can_reuse_failure;
 545         uint64_t        reusable_reclaimed;
 546         uint64_t        reusable_nonwritable;
 547         uint64_t        reusable_shared;
 548         uint64_t        free_shared;
 549 };
 550 extern struct vm_page_stats_reusable vm_page_stats_reusable;
 551
 552 extern int hibernate_flush_memory(void);
 553 extern void hibernate_reset_stats(void);
 554 extern void hibernate_create_paddr_map(void);
 555
 556 extern void vm_set_restrictions(void);
 557
 558 extern int vm_compressor_mode;
 559 extern kern_return_t vm_pageout_compress_page(void **, char *, vm_page_t);
 560 extern void vm_pageout_anonymous_pages(void);
 561 extern void vm_pageout_disconnect_all_pages(void);
 562
 563
 564 struct  vm_config {
 565         boolean_t       compressor_is_present;          /* compressor is initialized and can be used by the freezer, the sweep or the pager */
 566         boolean_t       compressor_is_active;           /* pager can actively compress pages...  'compressor_is_present' must be set */
 567         boolean_t       swap_is_present;                /* swap is initialized and can be used by the freezer, the sweep or the pager */
 568         boolean_t       swap_is_active;                 /* pager can actively swap out compressed segments... 'swap_is_present' must be set */
 569         boolean_t       freezer_swap_is_active;         /* freezer can swap out frozen tasks... "compressor_is_present + swap_is_present" must be set */
 570 };
 571
 572 extern  struct vm_config        vm_config;
 573
 574
 575 #define VM_PAGER_NOT_CONFIGURED                         0x0     /* no compresser or swap configured */
 576 #define VM_PAGER_DEFAULT                                0x1     /* Use default pager... DEPRECATED */
 577 #define VM_PAGER_COMPRESSOR_NO_SWAP                     0x2     /* Active in-core compressor only. */
 578 #define VM_PAGER_COMPRESSOR_WITH_SWAP                   0x4     /* Active in-core compressor + swap backend. */
 579 #define VM_PAGER_FREEZER_DEFAULT                        0x8     /* Freezer backed by default pager... DEPRECATED */
 580 #define VM_PAGER_FREEZER_COMPRESSOR_NO_SWAP             0x10    /* Freezer backed by in-core compressor only i.e. frozen data remain in-core compressed.*/
 581 #define VM_PAGER_COMPRESSOR_NO_SWAP_PLUS_FREEZER_COMPRESSOR_WITH_SWAP   0x20    /* Active in-core compressor + Freezer backed by in-core compressor with swap support too.*/
 582
 583 #define VM_PAGER_MAX_MODES                              6       /* Total number of vm compressor modes supported */
 584
 585
 586 #define VM_CONFIG_COMPRESSOR_IS_PRESENT         (vm_config.compressor_is_present == TRUE)
 587 #define VM_CONFIG_COMPRESSOR_IS_ACTIVE          (vm_config.compressor_is_active == TRUE)
 588 #define VM_CONFIG_SWAP_IS_PRESENT               (vm_config.swap_is_present == TRUE)
 589 #define VM_CONFIG_SWAP_IS_ACTIVE                (vm_config.swap_is_active == TRUE)
 590 #define VM_CONFIG_FREEZER_SWAP_IS_ACTIVE        (vm_config.freezer_swap_is_active == TRUE)
 591
 592 #endif  /* KERNEL_PRIVATE */
 593
 594 #ifdef XNU_KERNEL_PRIVATE
 595
 596 struct vm_pageout_state {
 597         boolean_t vm_pressure_thread_running;
 598         boolean_t vm_pressure_changed;
 599         boolean_t vm_restricted_to_single_processor;
 600         int vm_compressor_thread_count;
 601
 602         unsigned int vm_page_speculative_q_age_ms;
 603         unsigned int vm_page_speculative_percentage;
 604         unsigned int vm_page_speculative_target;
 605
 606         unsigned int vm_pageout_swap_wait;
 607         unsigned int vm_pageout_idle_wait;      /* milliseconds */
 608         unsigned int vm_pageout_empty_wait;     /* milliseconds */
 609         unsigned int vm_pageout_burst_wait;     /* milliseconds */
 610         unsigned int vm_pageout_deadlock_wait;  /* milliseconds */
 611         unsigned int vm_pageout_deadlock_relief;
 612         unsigned int vm_pageout_burst_inactive_throttle;
 613
 614         unsigned int vm_pageout_inactive;
 615         unsigned int vm_pageout_inactive_used;  /* debugging */
 616         unsigned int vm_pageout_inactive_clean; /* debugging */
 617
 618         uint32_t vm_page_filecache_min;
 619         uint32_t vm_page_filecache_min_divisor;
 620         uint32_t vm_page_xpmapped_min;
 621         uint32_t vm_page_xpmapped_min_divisor;
 622         uint64_t vm_pageout_considered_page_last;
 623
 624         int vm_page_free_count_init;
 625
 626         unsigned int vm_memory_pressure;
 627
 628         int memorystatus_purge_on_critical;
 629         int memorystatus_purge_on_warning;
 630         int memorystatus_purge_on_urgent;
 631
 632         thread_t vm_pageout_external_iothread;
 633         thread_t vm_pageout_internal_iothread;
 634 };
 635
 636 extern struct vm_pageout_state vm_pageout_state;
 637
 638 /*
 639  * This structure is used to track the VM_INFO instrumentation
 640  */
 641 struct vm_pageout_vminfo {
 642         unsigned long vm_pageout_considered_page;
 643         unsigned long vm_pageout_considered_bq_internal;
 644         unsigned long vm_pageout_considered_bq_external;
 645         unsigned long vm_pageout_skipped_external;
 646
 647         unsigned long vm_pageout_pages_evicted;
 648         unsigned long vm_pageout_pages_purged;;
 649         unsigned long vm_pageout_freed_cleaned;
 650         unsigned long vm_pageout_freed_speculative;
 651         unsigned long vm_pageout_freed_external;
 652         unsigned long vm_pageout_freed_internal;
 653         unsigned long vm_pageout_inactive_dirty_internal;
 654         unsigned long vm_pageout_inactive_dirty_external;
 655         unsigned long vm_pageout_inactive_referenced;
 656         unsigned long vm_pageout_reactivation_limit_exceeded;
 657         unsigned long vm_pageout_inactive_force_reclaim;
 658         unsigned long vm_pageout_inactive_nolock;
 659         unsigned long vm_pageout_filecache_min_reactivated;
 660         unsigned long vm_pageout_scan_inactive_throttled_internal;
 661         unsigned long vm_pageout_scan_inactive_throttled_external;
 662
 663         uint64_t      vm_pageout_compressions;
 664         uint64_t      vm_compressor_pages_grabbed;
 665         unsigned long vm_compressor_failed;
 666
 667         unsigned long vm_page_pages_freed;
 668
 669         unsigned long vm_phantom_cache_found_ghost;
 670         unsigned long vm_phantom_cache_added_ghost;
 671 };
 672
 673 extern struct vm_pageout_vminfo vm_pageout_vminfo;
 674
 675
 676 #if DEVELOPMENT || DEBUG
 677
 678 /*
 679  *      This structure records the pageout daemon's actions:
 680  *      how many pages it looks at and what happens to those pages.
 681  *      No locking needed because only one thread modifies the fields.
 682  */
 683 struct vm_pageout_debug {
 684         uint32_t vm_pageout_balanced;
 685         uint32_t vm_pageout_scan_event_counter;
 686         uint32_t vm_pageout_speculative_dirty;
 687
 688         uint32_t vm_pageout_inactive_busy;
 689         uint32_t vm_pageout_inactive_absent;
 690         uint32_t vm_pageout_inactive_notalive;
 691         uint32_t vm_pageout_inactive_error;
 692         uint32_t vm_pageout_inactive_deactivated;
 693
 694         uint32_t vm_pageout_enqueued_cleaned;
 695
 696         uint32_t vm_pageout_cleaned_busy;
 697         uint32_t vm_pageout_cleaned_nolock;
 698         uint32_t vm_pageout_cleaned_reference_reactivated;
 699         uint32_t vm_pageout_cleaned_volatile_reactivated;
 700         uint32_t vm_pageout_cleaned_reactivated;  /* debugging; how many cleaned pages are found to be referenced on pageout (and are therefore reactivated) */
 701         uint32_t vm_pageout_cleaned_fault_reactivated;
 702
 703         uint32_t vm_pageout_dirty_no_pager;
 704         uint32_t vm_pageout_purged_objects;
 705
 706         uint32_t vm_pageout_scan_throttle;
 707         uint32_t vm_pageout_scan_reclaimed_throttled;
 708         uint32_t vm_pageout_scan_burst_throttle;
 709         uint32_t vm_pageout_scan_empty_throttle;
 710         uint32_t vm_pageout_scan_swap_throttle;
 711         uint32_t vm_pageout_scan_deadlock_detected;
 712         uint32_t vm_pageout_scan_inactive_throttle_success;
 713         uint32_t vm_pageout_scan_throttle_deferred;
 714
 715         uint32_t vm_pageout_inactive_external_forced_jetsam_count;
 716
 717         uint32_t vm_grab_anon_overrides;
 718         uint32_t vm_grab_anon_nops;
 719
 720         uint32_t vm_pageout_no_victim;
 721         unsigned long vm_pageout_throttle_up_count;
 722         uint32_t vm_page_steal_pageout_page;
 723
 724         uint32_t vm_cs_validated_resets;
 725         uint32_t vm_object_iopl_request_sleep_for_cleaning;
 726         uint32_t vm_page_slide_counter;
 727         uint32_t vm_page_slide_errors;
 728         uint32_t vm_page_throttle_count;
 729         /*
 730          * Statistics about UPL enforcement of copy-on-write obligations.
 731          */
 732         unsigned long upl_cow;
 733         unsigned long upl_cow_again;
 734         unsigned long upl_cow_pages;
 735         unsigned long upl_cow_again_pages;
 736         unsigned long iopl_cow;
 737         unsigned long iopl_cow_pages;
 738 };
 739
 740 extern struct vm_pageout_debug vm_pageout_debug;
 741
 742 #define VM_PAGEOUT_DEBUG(member, value)                 \
 743         MACRO_BEGIN                                     \
 744                 vm_pageout_debug.member += value;       \
 745         MACRO_END
 746 #else
 747 #define VM_PAGEOUT_DEBUG(member, value)
 748 #endif
 749
 750 #define MAX_COMPRESSOR_THREAD_COUNT      8
 751
 752 #if DEVELOPMENT || DEBUG
 753 typedef struct vmct_stats_s {
 754         uint64_t vmct_runtimes[MAX_COMPRESSOR_THREAD_COUNT];
 755         uint64_t vmct_pages[MAX_COMPRESSOR_THREAD_COUNT];
 756         uint64_t vmct_iterations[MAX_COMPRESSOR_THREAD_COUNT];
 757         uint64_t vmct_cthreads_total;
 758         int32_t vmct_minpages[MAX_COMPRESSOR_THREAD_COUNT];
 759         int32_t vmct_maxpages[MAX_COMPRESSOR_THREAD_COUNT];
 760 } vmct_stats_t;
 761 #endif
 762 #endif
 763 #endif  /* _VM_VM_PAGEOUT_H_ */