osfmk/vm/vm_pageout.h

   1 /*
   2  * Copyright (c) 2000-2019 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  */
  31 /*
  32  * Mach Operating System
  33  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
  34  * All Rights Reserved.
  35  *
  36  * Permission to use, copy, modify and distribute this software and its
  37  * documentation is hereby granted, provided that both the copyright
  38  * notice and this permission notice appear in all copies of the
  39  * software, derivative works or modified versions, and any portions
  40  * thereof, and that both notices appear in supporting documentation.
  41  *
  42  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  43  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  44  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  45  *
  46  * Carnegie Mellon requests users of this software to return to
  47  *
  48  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  49  *  School of Computer Science
  50  *  Carnegie Mellon University
  51  *  Pittsburgh PA 15213-3890
  52  *
  53  * any improvements or extensions that they make and grant Carnegie Mellon
  54  * the rights to redistribute these changes.
  55  */
  56 /*
  57  */
  58 /*
  59  *      File:   vm/vm_pageout.h
  60  *      Author: Avadis Tevanian, Jr.
  61  *      Date:   1986
  62  *
  63  *      Declarations for the pageout daemon interface.
  64  */
  65
  66 #ifndef _VM_VM_PAGEOUT_H_
  67 #define _VM_VM_PAGEOUT_H_
  68
  69 #ifdef  KERNEL_PRIVATE
  70
  71 #include <mach/mach_types.h>
  72 #include <mach/boolean.h>
  73 #include <mach/machine/vm_types.h>
  74 #include <mach/memory_object_types.h>
  75
  76 #include <kern/kern_types.h>
  77 #include <kern/locks.h>
  78
  79 #include <libkern/OSAtomic.h>
  80
  81
  82 #include <vm/vm_options.h>
  83
  84 #ifdef  MACH_KERNEL_PRIVATE
  85 #include <vm/vm_page.h>
  86 #endif
  87
  88 #include <sys/kdebug.h>
  89
  90 #define VM_PAGE_AVAILABLE_COUNT()               ((unsigned int)(vm_page_cleaned_count))
  91
  92 /* externally manipulated counters */
  93 extern unsigned int vm_pageout_cleaned_fault_reactivated;
  94
  95 #if CONFIG_FREEZE
  96 extern boolean_t memorystatus_freeze_enabled;
  97 #endif
  98
  99 #define VM_DYNAMIC_PAGING_ENABLED() (VM_CONFIG_COMPRESSOR_IS_ACTIVE)
 100
 101 #if VM_PRESSURE_EVENTS
 102 extern boolean_t vm_pressure_events_enabled;
 103 #endif /* VM_PRESSURE_EVENTS */
 104
 105
 106 /*
 107  * the following codes are used in the DBG_MACH_WORKINGSET subclass
 108  * of the DBG_MACH class
 109  */
 110 #define VM_DISCONNECT_ALL_PAGE_MAPPINGS         0x00
 111 #define VM_DISCONNECT_TASK_PAGE_MAPPINGS        0x01
 112 #define VM_REAL_FAULT_ADDR_INTERNAL             0x02
 113 #define VM_REAL_FAULT_ADDR_PURGABLE             0x03
 114 #define VM_REAL_FAULT_ADDR_EXTERNAL             0x04
 115 #define VM_REAL_FAULT_ADDR_SHAREDCACHE          0x05
 116 #define VM_REAL_FAULT_FAST                      0x06
 117 #define VM_REAL_FAULT_SLOW                      0x07
 118 #define VM_MAP_LOOKUP_OBJECT                    0x08
 119
 120
 121
 122 extern int      vm_debug_events;
 123
 124 #define VMF_CHECK_ZFDELAY               0x100
 125 #define VMF_COWDELAY                    0x101
 126 #define VMF_ZFDELAY                     0x102
 127 #define VMF_COMPRESSORDELAY             0x103
 128
 129 #define VM_PAGEOUT_SCAN                 0x104
 130 #define VM_PAGEOUT_BALANCE              0x105
 131 #define VM_PAGEOUT_FREELIST             0x106
 132 #define VM_PAGEOUT_PURGEONE             0x107
 133 #define VM_PAGEOUT_CACHE_EVICT          0x108
 134 #define VM_PAGEOUT_THREAD_BLOCK         0x109
 135 #define VM_PAGEOUT_JETSAM               0x10A
 136 #define VM_INFO1                        0x10B
 137 #define VM_INFO2                        0x10C
 138 #define VM_INFO3                        0x10D
 139 #define VM_INFO4                        0x10E
 140 #define VM_INFO5                        0x10F
 141 #define VM_INFO6                        0x110
 142 #define VM_INFO7                        0x111
 143 #define VM_INFO8                        0x112
 144 #define VM_INFO9                        0x113
 145
 146 #define VM_UPL_PAGE_WAIT                0x120
 147 #define VM_IOPL_PAGE_WAIT               0x121
 148 #define VM_PAGE_WAIT_BLOCK              0x122
 149
 150 #if CONFIG_IOSCHED
 151 #define VM_PAGE_SLEEP                   0x123
 152 #define VM_PAGE_EXPEDITE                0x124
 153 #define VM_PAGE_EXPEDITE_NO_MEMORY      0x125
 154 #endif
 155
 156 #define VM_PAGE_GRAB                    0x126
 157 #define VM_PAGE_RELEASE                 0x127
 158
 159 #define VM_PRESSURE_EVENT               0x130
 160 #define VM_EXECVE                       0x131
 161 #define VM_WAKEUP_COMPACTOR_SWAPPER     0x132
 162 #define VM_UPL_REQUEST                  0x133
 163 #define VM_IOPL_REQUEST                 0x134
 164 #define VM_KERN_REQUEST                 0x135
 165
 166 #define VM_DATA_WRITE                   0x140
 167
 168 #define VM_PRESSURE_LEVEL_CHANGE        0x141
 169
 170 #define VM_DEBUG_EVENT(name, event, control, arg1, arg2, arg3, arg4)    \
 171         MACRO_BEGIN                                             \
 172         if (__improbable(vm_debug_events)) {                    \
 173                 KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, event)) | control, arg1, arg2, arg3, arg4, 0); \
 174         }                                                       \
 175         MACRO_END
 176
 177 #define VM_DEBUG_CONSTANT_EVENT(name, event, control, arg1, arg2, arg3, arg4)   \
 178         MACRO_BEGIN                                             \
 179                 KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, event)) | control, arg1, arg2, arg3, arg4, 0); \
 180         MACRO_END
 181
 182 extern void memoryshot(unsigned int event, unsigned int control);
 183
 184 extern void update_vm_info(void);
 185
 186 #if CONFIG_IOSCHED
 187 extern int upl_get_cached_tier(
 188         upl_t                   upl);
 189 #endif
 190
 191 extern void upl_set_iodone(upl_t, void *);
 192 extern void upl_set_iodone_error(upl_t, int);
 193 extern void upl_callout_iodone(upl_t);
 194
 195 extern ppnum_t upl_get_highest_page(
 196         upl_t                   upl);
 197
 198 extern upl_size_t upl_get_size(
 199         upl_t                   upl);
 200
 201 extern upl_t upl_associated_upl(upl_t upl);
 202 extern void upl_set_associated_upl(upl_t upl, upl_t associated_upl);
 203
 204 #ifdef  XNU_KERNEL_PRIVATE
 205
 206 extern kern_return_t vm_map_create_upl(
 207         vm_map_t                map,
 208         vm_map_address_t        offset,
 209         upl_size_t              *upl_size,
 210         upl_t                   *upl,
 211         upl_page_info_array_t   page_list,
 212         unsigned int            *count,
 213         upl_control_flags_t     *flags,
 214         vm_tag_t            tag);
 215
 216 extern void iopl_valid_data(
 217         upl_t                   upl_ptr,
 218         vm_tag_t        tag);
 219
 220 #endif  /* XNU_KERNEL_PRIVATE */
 221
 222 extern struct vnode * upl_lookup_vnode(upl_t upl);
 223
 224 #ifndef MACH_KERNEL_PRIVATE
 225 typedef struct vm_page  *vm_page_t;
 226 #endif
 227
 228 extern void                vm_page_free_list(
 229         vm_page_t   mem,
 230         boolean_t   prepare_object);
 231
 232 extern kern_return_t      vm_page_alloc_list(
 233         int         page_count,
 234         int                 flags,
 235         vm_page_t * list);
 236
 237 extern void               vm_page_set_offset(vm_page_t page, vm_object_offset_t offset);
 238 extern vm_object_offset_t vm_page_get_offset(vm_page_t page);
 239 extern ppnum_t            vm_page_get_phys_page(vm_page_t page);
 240 extern vm_page_t          vm_page_get_next(vm_page_t page);
 241
 242 extern kern_return_t    mach_vm_pressure_level_monitor(boolean_t wait_for_pressure, unsigned int *pressure_level);
 243
 244 #if !CONFIG_EMBEDDED
 245 extern kern_return_t    vm_pageout_wait(uint64_t deadline);
 246 #endif
 247
 248 #ifdef  MACH_KERNEL_PRIVATE
 249
 250 #include <vm/vm_page.h>
 251
 252 extern unsigned int     vm_pageout_scan_event_counter;
 253 extern unsigned int     vm_page_anonymous_count;
 254
 255
 256 /*
 257  * must hold the page queues lock to
 258  * manipulate this structure
 259  */
 260 struct vm_pageout_queue {
 261         vm_page_queue_head_t    pgo_pending;    /* laundry pages to be processed by pager's iothread */
 262         unsigned int    pgo_laundry;    /* current count of laundry pages on queue or in flight */
 263         unsigned int    pgo_maxlaundry;
 264         uint64_t        pgo_tid;        /* thread ID of I/O thread that services this queue */
 265         uint8_t         pgo_lowpriority; /* iothread is set to use low priority I/O */
 266
 267         unsigned int    pgo_idle:1,     /* iothread is blocked waiting for work to do */
 268             pgo_busy:1,                 /* iothread is currently processing request from pgo_pending */
 269             pgo_throttled:1,            /* vm_pageout_scan thread needs a wakeup when pgo_laundry drops */
 270             pgo_draining:1,
 271             pgo_inited:1,
 272         :0;
 273 };
 274
 275 #define VM_PAGE_Q_THROTTLED(q)          \
 276         ((q)->pgo_laundry >= (q)->pgo_maxlaundry)
 277
 278 extern struct   vm_pageout_queue        vm_pageout_queue_internal;
 279 extern struct   vm_pageout_queue        vm_pageout_queue_external;
 280
 281
 282 /*
 283  *      Routines exported to Mach.
 284  */
 285 extern void             vm_pageout(void);
 286
 287 extern kern_return_t    vm_pageout_internal_start(void);
 288
 289 extern void             vm_pageout_object_terminate(
 290         vm_object_t     object);
 291
 292 extern void             vm_pageout_cluster(
 293         vm_page_t       m);
 294
 295 extern void             vm_pageout_initialize_page(
 296         vm_page_t       m);
 297
 298 /* UPL exported routines and structures */
 299
 300 #define upl_lock_init(object)   lck_mtx_init(&(object)->Lock, &vm_object_lck_grp, &vm_object_lck_attr)
 301 #define upl_lock_destroy(object)        lck_mtx_destroy(&(object)->Lock, &vm_object_lck_grp)
 302 #define upl_lock(object)        lck_mtx_lock(&(object)->Lock)
 303 #define upl_unlock(object)      lck_mtx_unlock(&(object)->Lock)
 304 #define upl_try_lock(object)    lck_mtx_try_lock(&(object)->Lock)
 305
 306 #define MAX_VECTOR_UPL_ELEMENTS 8
 307
 308 struct _vector_upl_iostates {
 309         upl_offset_t offset;
 310         upl_size_t   size;
 311 };
 312
 313 typedef struct _vector_upl_iostates vector_upl_iostates_t;
 314
 315 struct _vector_upl {
 316         upl_size_t              size;
 317         uint32_t                num_upls;
 318         uint32_t                invalid_upls;
 319         uint32_t                _reserved;
 320         vm_map_t                submap;
 321         vm_offset_t             submap_dst_addr;
 322         vm_object_offset_t      offset;
 323         upl_t                   upl_elems[MAX_VECTOR_UPL_ELEMENTS];
 324         upl_page_info_array_t   pagelist;
 325         vector_upl_iostates_t   upl_iostates[MAX_VECTOR_UPL_ELEMENTS];
 326 };
 327
 328 typedef struct _vector_upl* vector_upl_t;
 329
 330 /* universal page list structure */
 331
 332 #if UPL_DEBUG
 333 #define UPL_DEBUG_STACK_FRAMES  16
 334 #define UPL_DEBUG_COMMIT_RECORDS 4
 335
 336 struct ucd {
 337         upl_offset_t    c_beg;
 338         upl_offset_t    c_end;
 339         int             c_aborted;
 340         void *          c_retaddr[UPL_DEBUG_STACK_FRAMES];
 341 };
 342 #endif
 343
 344 struct upl_io_completion {
 345         void     *io_context;
 346         void     (*io_done)(void *, int);
 347
 348         int      io_error;
 349 };
 350
 351
 352 struct upl {
 353         decl_lck_mtx_data(, Lock);      /* Synchronization */
 354         int             ref_count;
 355         int             ext_ref_count;
 356         int             flags;
 357         vm_object_offset_t offset;
 358         upl_size_t      size;       /* size in bytes of the address space */
 359         vm_offset_t     kaddr;      /* secondary mapping in kernel */
 360         vm_object_t     map_object;
 361         ppnum_t         highest_page;
 362         void*           vector_upl;
 363         upl_t           associated_upl;
 364         struct upl_io_completion *upl_iodone;
 365 #if CONFIG_IOSCHED
 366         int             upl_priority;
 367         uint64_t        *upl_reprio_info;
 368         void            *decmp_io_upl;
 369 #endif
 370 #if CONFIG_IOSCHED || UPL_DEBUG
 371         thread_t        upl_creator;
 372         queue_chain_t   uplq;       /* List of outstanding upls on an obj */
 373 #endif
 374 #if     UPL_DEBUG
 375         uintptr_t       ubc_alias1;
 376         uintptr_t       ubc_alias2;
 377
 378         uint32_t        upl_state;
 379         uint32_t        upl_commit_index;
 380         void    *upl_create_retaddr[UPL_DEBUG_STACK_FRAMES];
 381
 382         struct  ucd     upl_commit_records[UPL_DEBUG_COMMIT_RECORDS];
 383 #endif  /* UPL_DEBUG */
 384 };
 385
 386 /* upl struct flags */
 387 #define UPL_PAGE_LIST_MAPPED    0x1
 388 #define UPL_KERNEL_MAPPED       0x2
 389 #define UPL_CLEAR_DIRTY         0x4
 390 #define UPL_COMPOSITE_LIST      0x8
 391 #define UPL_INTERNAL            0x10
 392 #define UPL_PAGE_SYNC_DONE      0x20
 393 #define UPL_DEVICE_MEMORY       0x40
 394 #define UPL_PAGEOUT             0x80
 395 #define UPL_LITE                0x100
 396 #define UPL_IO_WIRE             0x200
 397 #define UPL_ACCESS_BLOCKED      0x400
 398 #define UPL_SHADOWED            0x1000
 399 #define UPL_KERNEL_OBJECT       0x2000
 400 #define UPL_VECTOR              0x4000
 401 #define UPL_SET_DIRTY           0x8000
 402 #define UPL_HAS_BUSY            0x10000
 403 #define UPL_TRACKED_BY_OBJECT   0x20000
 404 #define UPL_EXPEDITE_SUPPORTED  0x40000
 405 #define UPL_DECMP_REQ           0x80000
 406 #define UPL_DECMP_REAL_IO       0x100000
 407
 408 /* flags for upl_create flags parameter */
 409 #define UPL_CREATE_EXTERNAL     0
 410 #define UPL_CREATE_INTERNAL     0x1
 411 #define UPL_CREATE_LITE         0x2
 412 #define UPL_CREATE_IO_TRACKING  0x4
 413 #define UPL_CREATE_EXPEDITE_SUP 0x8
 414
 415 extern upl_t vector_upl_create(vm_offset_t);
 416 extern void vector_upl_deallocate(upl_t);
 417 extern boolean_t vector_upl_is_valid(upl_t);
 418 extern boolean_t vector_upl_set_subupl(upl_t, upl_t, u_int32_t);
 419 extern void vector_upl_set_pagelist(upl_t);
 420 extern void vector_upl_set_submap(upl_t, vm_map_t, vm_offset_t);
 421 extern void vector_upl_get_submap(upl_t, vm_map_t*, vm_offset_t*);
 422 extern void vector_upl_set_iostate(upl_t, upl_t, upl_offset_t, upl_size_t);
 423 extern void vector_upl_get_iostate(upl_t, upl_t, upl_offset_t*, upl_size_t*);
 424 extern void vector_upl_get_iostate_byindex(upl_t, uint32_t, upl_offset_t*, upl_size_t*);
 425 extern upl_t vector_upl_subupl_byindex(upl_t, uint32_t);
 426 extern upl_t vector_upl_subupl_byoffset(upl_t, upl_offset_t*, upl_size_t*);
 427
 428 extern void vm_object_set_pmap_cache_attr(
 429         vm_object_t             object,
 430         upl_page_info_array_t   user_page_list,
 431         unsigned int            num_pages,
 432         boolean_t               batch_pmap_op);
 433
 434 extern kern_return_t vm_object_iopl_request(
 435         vm_object_t             object,
 436         vm_object_offset_t      offset,
 437         upl_size_t              size,
 438         upl_t                   *upl_ptr,
 439         upl_page_info_array_t   user_page_list,
 440         unsigned int            *page_list_count,
 441         upl_control_flags_t     cntrl_flags,
 442         vm_tag_t            tag);
 443
 444 extern kern_return_t vm_object_super_upl_request(
 445         vm_object_t             object,
 446         vm_object_offset_t      offset,
 447         upl_size_t              size,
 448         upl_size_t              super_cluster,
 449         upl_t                   *upl,
 450         upl_page_info_t         *user_page_list,
 451         unsigned int            *page_list_count,
 452         upl_control_flags_t     cntrl_flags,
 453         vm_tag_t            tag);
 454
 455 /* should be just a regular vm_map_enter() */
 456 extern kern_return_t vm_map_enter_upl(
 457         vm_map_t                map,
 458         upl_t                   upl,
 459         vm_map_offset_t         *dst_addr);
 460
 461 /* should be just a regular vm_map_remove() */
 462 extern kern_return_t vm_map_remove_upl(
 463         vm_map_t                map,
 464         upl_t                   upl);
 465
 466 /* wired  page list structure */
 467 typedef uint32_t *wpl_array_t;
 468
 469 extern void vm_page_free_reserve(int pages);
 470
 471 extern void vm_pageout_throttle_down(vm_page_t page);
 472 extern void vm_pageout_throttle_up(vm_page_t page);
 473
 474 extern kern_return_t vm_paging_map_object(
 475         vm_page_t               page,
 476         vm_object_t             object,
 477         vm_object_offset_t      offset,
 478         vm_prot_t               protection,
 479         boolean_t               can_unlock_object,
 480         vm_map_size_t           *size,          /* IN/OUT */
 481         vm_map_offset_t         *address,       /* OUT */
 482         boolean_t               *need_unmap);   /* OUT */
 483 extern void vm_paging_unmap_object(
 484         vm_object_t             object,
 485         vm_map_offset_t         start,
 486         vm_map_offset_t         end);
 487 decl_simple_lock_data(extern, vm_paging_lock);
 488
 489 /*
 490  * Backing store throttle when BS is exhausted
 491  */
 492 extern unsigned int    vm_backing_store_low;
 493
 494 extern void vm_pageout_steal_laundry(
 495         vm_page_t page,
 496         boolean_t queues_locked);
 497
 498 #endif  /* MACH_KERNEL_PRIVATE */
 499
 500 #if UPL_DEBUG
 501 extern kern_return_t  upl_ubc_alias_set(
 502         upl_t upl,
 503         uintptr_t alias1,
 504         uintptr_t alias2);
 505 extern int  upl_ubc_alias_get(
 506         upl_t upl,
 507         uintptr_t * al,
 508         uintptr_t * al2);
 509 #endif /* UPL_DEBUG */
 510
 511 extern void vm_countdirtypages(void);
 512
 513 extern void vm_backing_store_disable(
 514         boolean_t       suspend);
 515
 516 extern kern_return_t upl_transpose(
 517         upl_t   upl1,
 518         upl_t   upl2);
 519
 520 extern kern_return_t mach_vm_pressure_monitor(
 521         boolean_t       wait_for_pressure,
 522         unsigned int    nsecs_monitored,
 523         unsigned int    *pages_reclaimed_p,
 524         unsigned int    *pages_wanted_p);
 525
 526 extern kern_return_t
 527 vm_set_buffer_cleanup_callout(
 528         boolean_t       (*func)(int));
 529
 530 struct vm_page_stats_reusable {
 531         SInt32          reusable_count;
 532         uint64_t        reusable;
 533         uint64_t        reused;
 534         uint64_t        reused_wire;
 535         uint64_t        reused_remove;
 536         uint64_t        all_reusable_calls;
 537         uint64_t        partial_reusable_calls;
 538         uint64_t        all_reuse_calls;
 539         uint64_t        partial_reuse_calls;
 540         uint64_t        reusable_pages_success;
 541         uint64_t        reusable_pages_failure;
 542         uint64_t        reusable_pages_shared;
 543         uint64_t        reuse_pages_success;
 544         uint64_t        reuse_pages_failure;
 545         uint64_t        can_reuse_success;
 546         uint64_t        can_reuse_failure;
 547         uint64_t        reusable_reclaimed;
 548         uint64_t        reusable_nonwritable;
 549         uint64_t        reusable_shared;
 550         uint64_t        free_shared;
 551 };
 552 extern struct vm_page_stats_reusable vm_page_stats_reusable;
 553
 554 extern int hibernate_flush_memory(void);
 555 extern void hibernate_reset_stats(void);
 556 extern void hibernate_create_paddr_map(void);
 557
 558 extern void vm_set_restrictions(void);
 559
 560 extern int vm_compressor_mode;
 561 extern kern_return_t vm_pageout_compress_page(void **, char *, vm_page_t);
 562 extern void vm_pageout_anonymous_pages(void);
 563 extern void vm_pageout_disconnect_all_pages(void);
 564
 565
 566 struct  vm_config {
 567         boolean_t       compressor_is_present;          /* compressor is initialized and can be used by the freezer, the sweep or the pager */
 568         boolean_t       compressor_is_active;           /* pager can actively compress pages...  'compressor_is_present' must be set */
 569         boolean_t       swap_is_present;                /* swap is initialized and can be used by the freezer, the sweep or the pager */
 570         boolean_t       swap_is_active;                 /* pager can actively swap out compressed segments... 'swap_is_present' must be set */
 571         boolean_t       freezer_swap_is_active;         /* freezer can swap out frozen tasks... "compressor_is_present + swap_is_present" must be set */
 572 };
 573
 574 extern  struct vm_config        vm_config;
 575
 576
 577 #define VM_PAGER_NOT_CONFIGURED                         0x0     /* no compresser or swap configured */
 578 #define VM_PAGER_DEFAULT                                0x1     /* Use default pager... DEPRECATED */
 579 #define VM_PAGER_COMPRESSOR_NO_SWAP                     0x2     /* Active in-core compressor only. */
 580 #define VM_PAGER_COMPRESSOR_WITH_SWAP                   0x4     /* Active in-core compressor + swap backend. */
 581 #define VM_PAGER_FREEZER_DEFAULT                        0x8     /* Freezer backed by default pager... DEPRECATED */
 582 #define VM_PAGER_FREEZER_COMPRESSOR_NO_SWAP             0x10    /* Freezer backed by in-core compressor only i.e. frozen data remain in-core compressed.*/
 583 #define VM_PAGER_COMPRESSOR_NO_SWAP_PLUS_FREEZER_COMPRESSOR_WITH_SWAP   0x20    /* Active in-core compressor + Freezer backed by in-core compressor with swap support too.*/
 584
 585 #define VM_PAGER_MAX_MODES                              6       /* Total number of vm compressor modes supported */
 586
 587
 588 #define VM_CONFIG_COMPRESSOR_IS_PRESENT         (vm_config.compressor_is_present == TRUE)
 589 #define VM_CONFIG_COMPRESSOR_IS_ACTIVE          (vm_config.compressor_is_active == TRUE)
 590 #define VM_CONFIG_SWAP_IS_PRESENT               (vm_config.swap_is_present == TRUE)
 591 #define VM_CONFIG_SWAP_IS_ACTIVE                (vm_config.swap_is_active == TRUE)
 592 #define VM_CONFIG_FREEZER_SWAP_IS_ACTIVE        (vm_config.freezer_swap_is_active == TRUE)
 593
 594 #endif  /* KERNEL_PRIVATE */
 595
 596 #ifdef XNU_KERNEL_PRIVATE
 597
 598 struct vm_pageout_state {
 599         boolean_t vm_pressure_thread_running;
 600         boolean_t vm_pressure_changed;
 601         boolean_t vm_restricted_to_single_processor;
 602         int vm_compressor_thread_count;
 603
 604         unsigned int vm_page_speculative_q_age_ms;
 605         unsigned int vm_page_speculative_percentage;
 606         unsigned int vm_page_speculative_target;
 607
 608         unsigned int vm_pageout_swap_wait;
 609         unsigned int vm_pageout_idle_wait;      /* milliseconds */
 610         unsigned int vm_pageout_empty_wait;     /* milliseconds */
 611         unsigned int vm_pageout_burst_wait;     /* milliseconds */
 612         unsigned int vm_pageout_deadlock_wait;  /* milliseconds */
 613         unsigned int vm_pageout_deadlock_relief;
 614         unsigned int vm_pageout_burst_inactive_throttle;
 615
 616         unsigned int vm_pageout_inactive;
 617         unsigned int vm_pageout_inactive_used;  /* debugging */
 618         unsigned int vm_pageout_inactive_clean; /* debugging */
 619
 620         uint32_t vm_page_filecache_min;
 621         uint32_t vm_page_filecache_min_divisor;
 622         uint32_t vm_page_xpmapped_min;
 623         uint32_t vm_page_xpmapped_min_divisor;
 624         uint64_t vm_pageout_considered_page_last;
 625
 626         int vm_page_free_count_init;
 627
 628         unsigned int vm_memory_pressure;
 629
 630         int memorystatus_purge_on_critical;
 631         int memorystatus_purge_on_warning;
 632         int memorystatus_purge_on_urgent;
 633
 634         thread_t vm_pageout_external_iothread;
 635         thread_t vm_pageout_internal_iothread;
 636 };
 637
 638 extern struct vm_pageout_state vm_pageout_state;
 639
 640 /*
 641  * This structure is used to track the VM_INFO instrumentation
 642  */
 643 struct vm_pageout_vminfo {
 644         unsigned long vm_pageout_considered_page;
 645         unsigned long vm_pageout_considered_bq_internal;
 646         unsigned long vm_pageout_considered_bq_external;
 647         unsigned long vm_pageout_skipped_external;
 648
 649         unsigned long vm_pageout_pages_evicted;
 650         unsigned long vm_pageout_pages_purged;
 651         unsigned long vm_pageout_freed_cleaned;
 652         unsigned long vm_pageout_freed_speculative;
 653         unsigned long vm_pageout_freed_external;
 654         unsigned long vm_pageout_freed_internal;
 655         unsigned long vm_pageout_inactive_dirty_internal;
 656         unsigned long vm_pageout_inactive_dirty_external;
 657         unsigned long vm_pageout_inactive_referenced;
 658         unsigned long vm_pageout_reactivation_limit_exceeded;
 659         unsigned long vm_pageout_inactive_force_reclaim;
 660         unsigned long vm_pageout_inactive_nolock;
 661         unsigned long vm_pageout_filecache_min_reactivated;
 662         unsigned long vm_pageout_scan_inactive_throttled_internal;
 663         unsigned long vm_pageout_scan_inactive_throttled_external;
 664
 665         uint64_t      vm_pageout_compressions;
 666         uint64_t      vm_compressor_pages_grabbed;
 667         unsigned long vm_compressor_failed;
 668
 669         unsigned long vm_page_pages_freed;
 670
 671         unsigned long vm_phantom_cache_found_ghost;
 672         unsigned long vm_phantom_cache_added_ghost;
 673 };
 674
 675 extern struct vm_pageout_vminfo vm_pageout_vminfo;
 676
 677
 678 #if DEVELOPMENT || DEBUG
 679
 680 /*
 681  *      This structure records the pageout daemon's actions:
 682  *      how many pages it looks at and what happens to those pages.
 683  *      No locking needed because only one thread modifies the fields.
 684  */
 685 struct vm_pageout_debug {
 686         uint32_t vm_pageout_balanced;
 687         uint32_t vm_pageout_scan_event_counter;
 688         uint32_t vm_pageout_speculative_dirty;
 689
 690         uint32_t vm_pageout_inactive_busy;
 691         uint32_t vm_pageout_inactive_absent;
 692         uint32_t vm_pageout_inactive_notalive;
 693         uint32_t vm_pageout_inactive_error;
 694         uint32_t vm_pageout_inactive_deactivated;
 695
 696         uint32_t vm_pageout_enqueued_cleaned;
 697
 698         uint32_t vm_pageout_cleaned_busy;
 699         uint32_t vm_pageout_cleaned_nolock;
 700         uint32_t vm_pageout_cleaned_reference_reactivated;
 701         uint32_t vm_pageout_cleaned_volatile_reactivated;
 702         uint32_t vm_pageout_cleaned_reactivated;  /* debugging; how many cleaned pages are found to be referenced on pageout (and are therefore reactivated) */
 703         uint32_t vm_pageout_cleaned_fault_reactivated;
 704
 705         uint32_t vm_pageout_dirty_no_pager;
 706         uint32_t vm_pageout_purged_objects;
 707
 708         uint32_t vm_pageout_scan_throttle;
 709         uint32_t vm_pageout_scan_reclaimed_throttled;
 710         uint32_t vm_pageout_scan_burst_throttle;
 711         uint32_t vm_pageout_scan_empty_throttle;
 712         uint32_t vm_pageout_scan_swap_throttle;
 713         uint32_t vm_pageout_scan_deadlock_detected;
 714         uint32_t vm_pageout_scan_inactive_throttle_success;
 715         uint32_t vm_pageout_scan_throttle_deferred;
 716
 717         uint32_t vm_pageout_inactive_external_forced_jetsam_count;
 718
 719         uint32_t vm_grab_anon_overrides;
 720         uint32_t vm_grab_anon_nops;
 721
 722         uint32_t vm_pageout_no_victim;
 723         unsigned long vm_pageout_throttle_up_count;
 724         uint32_t vm_page_steal_pageout_page;
 725
 726         uint32_t vm_cs_validated_resets;
 727         uint32_t vm_object_iopl_request_sleep_for_cleaning;
 728         uint32_t vm_page_slide_counter;
 729         uint32_t vm_page_slide_errors;
 730         uint32_t vm_page_throttle_count;
 731         /*
 732          * Statistics about UPL enforcement of copy-on-write obligations.
 733          */
 734         unsigned long upl_cow;
 735         unsigned long upl_cow_again;
 736         unsigned long upl_cow_pages;
 737         unsigned long upl_cow_again_pages;
 738         unsigned long iopl_cow;
 739         unsigned long iopl_cow_pages;
 740 };
 741
 742 extern struct vm_pageout_debug vm_pageout_debug;
 743
 744 #define VM_PAGEOUT_DEBUG(member, value)                 \
 745         MACRO_BEGIN                                     \
 746                 vm_pageout_debug.member += value;       \
 747         MACRO_END
 748 #else
 749 #define VM_PAGEOUT_DEBUG(member, value)
 750 #endif
 751
 752 #define MAX_COMPRESSOR_THREAD_COUNT      8
 753
 754 #if DEVELOPMENT || DEBUG
 755 typedef struct vmct_stats_s {
 756         uint64_t vmct_runtimes[MAX_COMPRESSOR_THREAD_COUNT];
 757         uint64_t vmct_pages[MAX_COMPRESSOR_THREAD_COUNT];
 758         uint64_t vmct_iterations[MAX_COMPRESSOR_THREAD_COUNT];
 759         uint64_t vmct_cthreads_total;
 760         int32_t vmct_minpages[MAX_COMPRESSOR_THREAD_COUNT];
 761         int32_t vmct_maxpages[MAX_COMPRESSOR_THREAD_COUNT];
 762 } vmct_stats_t;
 763 #endif
 764 #endif
 765 #endif  /* _VM_VM_PAGEOUT_H_ */