osfmk/vm/vm_pageout.h

   1 /*
   2  * Copyright (c) 2000-2019 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  */
  31 /*
  32  * Mach Operating System
  33  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
  34  * All Rights Reserved.
  35  *
  36  * Permission to use, copy, modify and distribute this software and its
  37  * documentation is hereby granted, provided that both the copyright
  38  * notice and this permission notice appear in all copies of the
  39  * software, derivative works or modified versions, and any portions
  40  * thereof, and that both notices appear in supporting documentation.
  41  *
  42  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  43  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  44  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  45  *
  46  * Carnegie Mellon requests users of this software to return to
  47  *
  48  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  49  *  School of Computer Science
  50  *  Carnegie Mellon University
  51  *  Pittsburgh PA 15213-3890
  52  *
  53  * any improvements or extensions that they make and grant Carnegie Mellon
  54  * the rights to redistribute these changes.
  55  */
  56 /*
  57  */
  58 /*
  59  *      File:   vm/vm_pageout.h
  60  *      Author: Avadis Tevanian, Jr.
  61  *      Date:   1986
  62  *
  63  *      Declarations for the pageout daemon interface.
  64  */
  65
  66 #ifndef _VM_VM_PAGEOUT_H_
  67 #define _VM_VM_PAGEOUT_H_
  68
  69 #ifdef  KERNEL_PRIVATE
  70
  71 #include <mach/mach_types.h>
  72 #include <mach/boolean.h>
  73 #include <mach/machine/vm_types.h>
  74 #include <mach/memory_object_types.h>
  75
  76 #include <kern/kern_types.h>
  77 #include <kern/locks.h>
  78
  79 #include <libkern/OSAtomic.h>
  80
  81
  82 #include <vm/vm_options.h>
  83
  84 #ifdef  MACH_KERNEL_PRIVATE
  85 #include <vm/vm_page.h>
  86 #endif
  87
  88 #include <sys/kdebug.h>
  89
  90 #define VM_PAGE_AVAILABLE_COUNT()               ((unsigned int)(vm_page_cleaned_count))
  91
  92 /* externally manipulated counters */
  93 extern unsigned int vm_pageout_cleaned_fault_reactivated;
  94
  95 #if CONFIG_FREEZE
  96 extern boolean_t memorystatus_freeze_enabled;
  97 #endif
  98
  99 #define VM_DYNAMIC_PAGING_ENABLED() (VM_CONFIG_COMPRESSOR_IS_ACTIVE)
 100
 101 #if VM_PRESSURE_EVENTS
 102 extern boolean_t vm_pressure_events_enabled;
 103 #endif /* VM_PRESSURE_EVENTS */
 104
 105
 106 /*
 107  * the following codes are used in the DBG_MACH_WORKINGSET subclass
 108  * of the DBG_MACH class
 109  */
 110 #define VM_DISCONNECT_ALL_PAGE_MAPPINGS         0x00
 111 #define VM_DISCONNECT_TASK_PAGE_MAPPINGS        0x01
 112 #define VM_REAL_FAULT_ADDR_INTERNAL             0x02
 113 #define VM_REAL_FAULT_ADDR_PURGABLE             0x03
 114 #define VM_REAL_FAULT_ADDR_EXTERNAL             0x04
 115 #define VM_REAL_FAULT_ADDR_SHAREDCACHE          0x05
 116
 117
 118
 119 extern int      vm_debug_events;
 120
 121 #define VMF_CHECK_ZFDELAY               0x100
 122 #define VMF_COWDELAY                    0x101
 123 #define VMF_ZFDELAY                     0x102
 124 #define VMF_COMPRESSORDELAY             0x103
 125
 126 #define VM_PAGEOUT_SCAN                 0x104
 127 #define VM_PAGEOUT_BALANCE              0x105
 128 #define VM_PAGEOUT_FREELIST             0x106
 129 #define VM_PAGEOUT_PURGEONE             0x107
 130 #define VM_PAGEOUT_CACHE_EVICT          0x108
 131 #define VM_PAGEOUT_THREAD_BLOCK         0x109
 132 #define VM_PAGEOUT_JETSAM               0x10A
 133 #define VM_INFO1                        0x10B
 134 #define VM_INFO2                        0x10C
 135 #define VM_INFO3                        0x10D
 136 #define VM_INFO4                        0x10E
 137 #define VM_INFO5                        0x10F
 138 #define VM_INFO6                        0x110
 139 #define VM_INFO7                        0x111
 140 #define VM_INFO8                        0x112
 141 #define VM_INFO9                        0x113
 142
 143 #define VM_UPL_PAGE_WAIT                0x120
 144 #define VM_IOPL_PAGE_WAIT               0x121
 145 #define VM_PAGE_WAIT_BLOCK              0x122
 146
 147 #if CONFIG_IOSCHED
 148 #define VM_PAGE_SLEEP                   0x123
 149 #define VM_PAGE_EXPEDITE                0x124
 150 #define VM_PAGE_EXPEDITE_NO_MEMORY      0x125
 151 #endif
 152
 153 #define VM_PAGE_GRAB                    0x126
 154 #define VM_PAGE_RELEASE                 0x127
 155
 156 #define VM_PRESSURE_EVENT               0x130
 157 #define VM_EXECVE                       0x131
 158 #define VM_WAKEUP_COMPACTOR_SWAPPER     0x132
 159 #define VM_UPL_REQUEST                  0x133
 160 #define VM_IOPL_REQUEST                 0x134
 161 #define VM_KERN_REQUEST                 0x135
 162
 163 #define VM_DATA_WRITE                   0x140
 164
 165 #define VM_PRESSURE_LEVEL_CHANGE        0x141
 166
 167 #define VM_DEBUG_EVENT(name, event, control, arg1, arg2, arg3, arg4)    \
 168         MACRO_BEGIN                                             \
 169         if (__improbable(vm_debug_events)) {                    \
 170                 KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, event)) | control, arg1, arg2, arg3, arg4, 0); \
 171         }                                                       \
 172         MACRO_END
 173
 174 #define VM_DEBUG_CONSTANT_EVENT(name, event, control, arg1, arg2, arg3, arg4)   \
 175         MACRO_BEGIN                                             \
 176                 KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, event)) | control, arg1, arg2, arg3, arg4, 0); \
 177         MACRO_END
 178
 179 extern void memoryshot(unsigned int event, unsigned int control);
 180
 181 extern void update_vm_info(void);
 182
 183 #if CONFIG_IOSCHED
 184 extern int upl_get_cached_tier(
 185         upl_t                   upl);
 186 #endif
 187
 188 extern void upl_set_iodone(upl_t, void *);
 189 extern void upl_set_iodone_error(upl_t, int);
 190 extern void upl_callout_iodone(upl_t);
 191
 192 extern ppnum_t upl_get_highest_page(
 193         upl_t                   upl);
 194
 195 extern upl_size_t upl_get_size(
 196         upl_t                   upl);
 197
 198 extern upl_t upl_associated_upl(upl_t upl);
 199 extern void upl_set_associated_upl(upl_t upl, upl_t associated_upl);
 200
 201 #ifdef  XNU_KERNEL_PRIVATE
 202
 203 extern kern_return_t vm_map_create_upl(
 204         vm_map_t                map,
 205         vm_map_address_t        offset,
 206         upl_size_t              *upl_size,
 207         upl_t                   *upl,
 208         upl_page_info_array_t   page_list,
 209         unsigned int            *count,
 210         upl_control_flags_t     *flags,
 211         vm_tag_t            tag);
 212
 213 extern void iopl_valid_data(
 214         upl_t                   upl_ptr,
 215         vm_tag_t        tag);
 216
 217 #endif  /* XNU_KERNEL_PRIVATE */
 218
 219 extern struct vnode * upl_lookup_vnode(upl_t upl);
 220
 221 #ifndef MACH_KERNEL_PRIVATE
 222 typedef struct vm_page  *vm_page_t;
 223 #endif
 224
 225 extern void                vm_page_free_list(
 226         vm_page_t   mem,
 227         boolean_t   prepare_object);
 228
 229 extern kern_return_t      vm_page_alloc_list(
 230         int         page_count,
 231         int                 flags,
 232         vm_page_t * list);
 233
 234 extern void               vm_page_set_offset(vm_page_t page, vm_object_offset_t offset);
 235 extern vm_object_offset_t vm_page_get_offset(vm_page_t page);
 236 extern ppnum_t            vm_page_get_phys_page(vm_page_t page);
 237 extern vm_page_t          vm_page_get_next(vm_page_t page);
 238
 239 extern kern_return_t    mach_vm_pressure_level_monitor(boolean_t wait_for_pressure, unsigned int *pressure_level);
 240
 241 #if !CONFIG_EMBEDDED
 242 extern kern_return_t    vm_pageout_wait(uint64_t deadline);
 243 #endif
 244
 245 #ifdef  MACH_KERNEL_PRIVATE
 246
 247 #include <vm/vm_page.h>
 248
 249 extern unsigned int     vm_pageout_scan_event_counter;
 250 extern unsigned int     vm_page_anonymous_count;
 251
 252
 253 /*
 254  * must hold the page queues lock to
 255  * manipulate this structure
 256  */
 257 struct vm_pageout_queue {
 258         vm_page_queue_head_t    pgo_pending;    /* laundry pages to be processed by pager's iothread */
 259         unsigned int    pgo_laundry;    /* current count of laundry pages on queue or in flight */
 260         unsigned int    pgo_maxlaundry;
 261         uint64_t        pgo_tid;        /* thread ID of I/O thread that services this queue */
 262         uint8_t         pgo_lowpriority; /* iothread is set to use low priority I/O */
 263
 264         unsigned int    pgo_idle:1,     /* iothread is blocked waiting for work to do */
 265             pgo_busy:1,                 /* iothread is currently processing request from pgo_pending */
 266             pgo_throttled:1,            /* vm_pageout_scan thread needs a wakeup when pgo_laundry drops */
 267             pgo_draining:1,
 268             pgo_inited:1,
 269         :0;
 270 };
 271
 272 #define VM_PAGE_Q_THROTTLED(q)          \
 273         ((q)->pgo_laundry >= (q)->pgo_maxlaundry)
 274
 275 extern struct   vm_pageout_queue        vm_pageout_queue_internal;
 276 extern struct   vm_pageout_queue        vm_pageout_queue_external;
 277
 278
 279 /*
 280  *      Routines exported to Mach.
 281  */
 282 extern void             vm_pageout(void);
 283
 284 extern kern_return_t    vm_pageout_internal_start(void);
 285
 286 extern void             vm_pageout_object_terminate(
 287         vm_object_t     object);
 288
 289 extern void             vm_pageout_cluster(
 290         vm_page_t       m);
 291
 292 extern void             vm_pageout_initialize_page(
 293         vm_page_t       m);
 294
 295 /* UPL exported routines and structures */
 296
 297 #define upl_lock_init(object)   lck_mtx_init(&(object)->Lock, &vm_object_lck_grp, &vm_object_lck_attr)
 298 #define upl_lock_destroy(object)        lck_mtx_destroy(&(object)->Lock, &vm_object_lck_grp)
 299 #define upl_lock(object)        lck_mtx_lock(&(object)->Lock)
 300 #define upl_unlock(object)      lck_mtx_unlock(&(object)->Lock)
 301 #define upl_try_lock(object)    lck_mtx_try_lock(&(object)->Lock)
 302
 303 #define MAX_VECTOR_UPL_ELEMENTS 8
 304
 305 struct _vector_upl_iostates {
 306         upl_offset_t offset;
 307         upl_size_t   size;
 308 };
 309
 310 typedef struct _vector_upl_iostates vector_upl_iostates_t;
 311
 312 struct _vector_upl {
 313         upl_size_t              size;
 314         uint32_t                num_upls;
 315         uint32_t                invalid_upls;
 316         uint32_t                _reserved;
 317         vm_map_t                submap;
 318         vm_offset_t             submap_dst_addr;
 319         vm_object_offset_t      offset;
 320         upl_t                   upl_elems[MAX_VECTOR_UPL_ELEMENTS];
 321         upl_page_info_array_t   pagelist;
 322         vector_upl_iostates_t   upl_iostates[MAX_VECTOR_UPL_ELEMENTS];
 323 };
 324
 325 typedef struct _vector_upl* vector_upl_t;
 326
 327 /* universal page list structure */
 328
 329 #if UPL_DEBUG
 330 #define UPL_DEBUG_STACK_FRAMES  16
 331 #define UPL_DEBUG_COMMIT_RECORDS 4
 332
 333 struct ucd {
 334         upl_offset_t    c_beg;
 335         upl_offset_t    c_end;
 336         int             c_aborted;
 337         void *          c_retaddr[UPL_DEBUG_STACK_FRAMES];
 338 };
 339 #endif
 340
 341 struct upl_io_completion {
 342         void     *io_context;
 343         void     (*io_done)(void *, int);
 344
 345         int      io_error;
 346 };
 347
 348
 349 struct upl {
 350         decl_lck_mtx_data(, Lock);      /* Synchronization */
 351         int             ref_count;
 352         int             ext_ref_count;
 353         int             flags;
 354         vm_object_offset_t offset;
 355         upl_size_t      size;       /* size in bytes of the address space */
 356         vm_offset_t     kaddr;      /* secondary mapping in kernel */
 357         vm_object_t     map_object;
 358         ppnum_t         highest_page;
 359         void*           vector_upl;
 360         upl_t           associated_upl;
 361         struct upl_io_completion *upl_iodone;
 362 #if CONFIG_IOSCHED
 363         int             upl_priority;
 364         uint64_t        *upl_reprio_info;
 365         void            *decmp_io_upl;
 366 #endif
 367 #if CONFIG_IOSCHED || UPL_DEBUG
 368         thread_t        upl_creator;
 369         queue_chain_t   uplq;       /* List of outstanding upls on an obj */
 370 #endif
 371 #if     UPL_DEBUG
 372         uintptr_t       ubc_alias1;
 373         uintptr_t       ubc_alias2;
 374
 375         uint32_t        upl_state;
 376         uint32_t        upl_commit_index;
 377         void    *upl_create_retaddr[UPL_DEBUG_STACK_FRAMES];
 378
 379         struct  ucd     upl_commit_records[UPL_DEBUG_COMMIT_RECORDS];
 380 #endif  /* UPL_DEBUG */
 381 };
 382
 383 /* upl struct flags */
 384 #define UPL_PAGE_LIST_MAPPED    0x1
 385 #define UPL_KERNEL_MAPPED       0x2
 386 #define UPL_CLEAR_DIRTY         0x4
 387 #define UPL_COMPOSITE_LIST      0x8
 388 #define UPL_INTERNAL            0x10
 389 #define UPL_PAGE_SYNC_DONE      0x20
 390 #define UPL_DEVICE_MEMORY       0x40
 391 #define UPL_PAGEOUT             0x80
 392 #define UPL_LITE                0x100
 393 #define UPL_IO_WIRE             0x200
 394 #define UPL_ACCESS_BLOCKED      0x400
 395 #define UPL_SHADOWED            0x1000
 396 #define UPL_KERNEL_OBJECT       0x2000
 397 #define UPL_VECTOR              0x4000
 398 #define UPL_SET_DIRTY           0x8000
 399 #define UPL_HAS_BUSY            0x10000
 400 #define UPL_TRACKED_BY_OBJECT   0x20000
 401 #define UPL_EXPEDITE_SUPPORTED  0x40000
 402 #define UPL_DECMP_REQ           0x80000
 403 #define UPL_DECMP_REAL_IO       0x100000
 404
 405 /* flags for upl_create flags parameter */
 406 #define UPL_CREATE_EXTERNAL     0
 407 #define UPL_CREATE_INTERNAL     0x1
 408 #define UPL_CREATE_LITE         0x2
 409 #define UPL_CREATE_IO_TRACKING  0x4
 410 #define UPL_CREATE_EXPEDITE_SUP 0x8
 411
 412 extern upl_t vector_upl_create(vm_offset_t);
 413 extern void vector_upl_deallocate(upl_t);
 414 extern boolean_t vector_upl_is_valid(upl_t);
 415 extern boolean_t vector_upl_set_subupl(upl_t, upl_t, u_int32_t);
 416 extern void vector_upl_set_pagelist(upl_t);
 417 extern void vector_upl_set_submap(upl_t, vm_map_t, vm_offset_t);
 418 extern void vector_upl_get_submap(upl_t, vm_map_t*, vm_offset_t*);
 419 extern void vector_upl_set_iostate(upl_t, upl_t, upl_offset_t, upl_size_t);
 420 extern void vector_upl_get_iostate(upl_t, upl_t, upl_offset_t*, upl_size_t*);
 421 extern void vector_upl_get_iostate_byindex(upl_t, uint32_t, upl_offset_t*, upl_size_t*);
 422 extern upl_t vector_upl_subupl_byindex(upl_t, uint32_t);
 423 extern upl_t vector_upl_subupl_byoffset(upl_t, upl_offset_t*, upl_size_t*);
 424
 425 extern void vm_object_set_pmap_cache_attr(
 426         vm_object_t             object,
 427         upl_page_info_array_t   user_page_list,
 428         unsigned int            num_pages,
 429         boolean_t               batch_pmap_op);
 430
 431 extern kern_return_t vm_object_iopl_request(
 432         vm_object_t             object,
 433         vm_object_offset_t      offset,
 434         upl_size_t              size,
 435         upl_t                   *upl_ptr,
 436         upl_page_info_array_t   user_page_list,
 437         unsigned int            *page_list_count,
 438         upl_control_flags_t     cntrl_flags,
 439         vm_tag_t            tag);
 440
 441 extern kern_return_t vm_object_super_upl_request(
 442         vm_object_t             object,
 443         vm_object_offset_t      offset,
 444         upl_size_t              size,
 445         upl_size_t              super_cluster,
 446         upl_t                   *upl,
 447         upl_page_info_t         *user_page_list,
 448         unsigned int            *page_list_count,
 449         upl_control_flags_t     cntrl_flags,
 450         vm_tag_t            tag);
 451
 452 /* should be just a regular vm_map_enter() */
 453 extern kern_return_t vm_map_enter_upl(
 454         vm_map_t                map,
 455         upl_t                   upl,
 456         vm_map_offset_t         *dst_addr);
 457
 458 /* should be just a regular vm_map_remove() */
 459 extern kern_return_t vm_map_remove_upl(
 460         vm_map_t                map,
 461         upl_t                   upl);
 462
 463 /* wired  page list structure */
 464 typedef uint32_t *wpl_array_t;
 465
 466 extern void vm_page_free_reserve(int pages);
 467
 468 extern void vm_pageout_throttle_down(vm_page_t page);
 469 extern void vm_pageout_throttle_up(vm_page_t page);
 470
 471 extern kern_return_t vm_paging_map_object(
 472         vm_page_t               page,
 473         vm_object_t             object,
 474         vm_object_offset_t      offset,
 475         vm_prot_t               protection,
 476         boolean_t               can_unlock_object,
 477         vm_map_size_t           *size,          /* IN/OUT */
 478         vm_map_offset_t         *address,       /* OUT */
 479         boolean_t               *need_unmap);   /* OUT */
 480 extern void vm_paging_unmap_object(
 481         vm_object_t             object,
 482         vm_map_offset_t         start,
 483         vm_map_offset_t         end);
 484 decl_simple_lock_data(extern, vm_paging_lock);
 485
 486 /*
 487  * Backing store throttle when BS is exhausted
 488  */
 489 extern unsigned int    vm_backing_store_low;
 490
 491 extern void vm_pageout_steal_laundry(
 492         vm_page_t page,
 493         boolean_t queues_locked);
 494
 495 #endif  /* MACH_KERNEL_PRIVATE */
 496
 497 #if UPL_DEBUG
 498 extern kern_return_t  upl_ubc_alias_set(
 499         upl_t upl,
 500         uintptr_t alias1,
 501         uintptr_t alias2);
 502 extern int  upl_ubc_alias_get(
 503         upl_t upl,
 504         uintptr_t * al,
 505         uintptr_t * al2);
 506 #endif /* UPL_DEBUG */
 507
 508 extern void vm_countdirtypages(void);
 509
 510 extern void vm_backing_store_disable(
 511         boolean_t       suspend);
 512
 513 extern kern_return_t upl_transpose(
 514         upl_t   upl1,
 515         upl_t   upl2);
 516
 517 extern kern_return_t mach_vm_pressure_monitor(
 518         boolean_t       wait_for_pressure,
 519         unsigned int    nsecs_monitored,
 520         unsigned int    *pages_reclaimed_p,
 521         unsigned int    *pages_wanted_p);
 522
 523 extern kern_return_t
 524 vm_set_buffer_cleanup_callout(
 525         boolean_t       (*func)(int));
 526
 527 struct vm_page_stats_reusable {
 528         SInt32          reusable_count;
 529         uint64_t        reusable;
 530         uint64_t        reused;
 531         uint64_t        reused_wire;
 532         uint64_t        reused_remove;
 533         uint64_t        all_reusable_calls;
 534         uint64_t        partial_reusable_calls;
 535         uint64_t        all_reuse_calls;
 536         uint64_t        partial_reuse_calls;
 537         uint64_t        reusable_pages_success;
 538         uint64_t        reusable_pages_failure;
 539         uint64_t        reusable_pages_shared;
 540         uint64_t        reuse_pages_success;
 541         uint64_t        reuse_pages_failure;
 542         uint64_t        can_reuse_success;
 543         uint64_t        can_reuse_failure;
 544         uint64_t        reusable_reclaimed;
 545         uint64_t        reusable_nonwritable;
 546         uint64_t        reusable_shared;
 547         uint64_t        free_shared;
 548 };
 549 extern struct vm_page_stats_reusable vm_page_stats_reusable;
 550
 551 extern int hibernate_flush_memory(void);
 552 extern void hibernate_reset_stats(void);
 553 extern void hibernate_create_paddr_map(void);
 554
 555 extern void vm_set_restrictions(void);
 556
 557 extern int vm_compressor_mode;
 558 extern kern_return_t vm_pageout_compress_page(void **, char *, vm_page_t);
 559 extern void vm_pageout_anonymous_pages(void);
 560 extern void vm_pageout_disconnect_all_pages(void);
 561
 562
 563 struct  vm_config {
 564         boolean_t       compressor_is_present;          /* compressor is initialized and can be used by the freezer, the sweep or the pager */
 565         boolean_t       compressor_is_active;           /* pager can actively compress pages...  'compressor_is_present' must be set */
 566         boolean_t       swap_is_present;                /* swap is initialized and can be used by the freezer, the sweep or the pager */
 567         boolean_t       swap_is_active;                 /* pager can actively swap out compressed segments... 'swap_is_present' must be set */
 568         boolean_t       freezer_swap_is_active;         /* freezer can swap out frozen tasks... "compressor_is_present + swap_is_present" must be set */
 569 };
 570
 571 extern  struct vm_config        vm_config;
 572
 573
 574 #define VM_PAGER_NOT_CONFIGURED                         0x0     /* no compresser or swap configured */
 575 #define VM_PAGER_DEFAULT                                0x1     /* Use default pager... DEPRECATED */
 576 #define VM_PAGER_COMPRESSOR_NO_SWAP                     0x2     /* Active in-core compressor only. */
 577 #define VM_PAGER_COMPRESSOR_WITH_SWAP                   0x4     /* Active in-core compressor + swap backend. */
 578 #define VM_PAGER_FREEZER_DEFAULT                        0x8     /* Freezer backed by default pager... DEPRECATED */
 579 #define VM_PAGER_FREEZER_COMPRESSOR_NO_SWAP             0x10    /* Freezer backed by in-core compressor only i.e. frozen data remain in-core compressed.*/
 580 #define VM_PAGER_COMPRESSOR_NO_SWAP_PLUS_FREEZER_COMPRESSOR_WITH_SWAP   0x20    /* Active in-core compressor + Freezer backed by in-core compressor with swap support too.*/
 581
 582 #define VM_PAGER_MAX_MODES                              6       /* Total number of vm compressor modes supported */
 583
 584
 585 #define VM_CONFIG_COMPRESSOR_IS_PRESENT         (vm_config.compressor_is_present == TRUE)
 586 #define VM_CONFIG_COMPRESSOR_IS_ACTIVE          (vm_config.compressor_is_active == TRUE)
 587 #define VM_CONFIG_SWAP_IS_PRESENT               (vm_config.swap_is_present == TRUE)
 588 #define VM_CONFIG_SWAP_IS_ACTIVE                (vm_config.swap_is_active == TRUE)
 589 #define VM_CONFIG_FREEZER_SWAP_IS_ACTIVE        (vm_config.freezer_swap_is_active == TRUE)
 590
 591 #endif  /* KERNEL_PRIVATE */
 592
 593 #ifdef XNU_KERNEL_PRIVATE
 594
 595 struct vm_pageout_state {
 596         boolean_t vm_pressure_thread_running;
 597         boolean_t vm_pressure_changed;
 598         boolean_t vm_restricted_to_single_processor;
 599         int vm_compressor_thread_count;
 600
 601         unsigned int vm_page_speculative_q_age_ms;
 602         unsigned int vm_page_speculative_percentage;
 603         unsigned int vm_page_speculative_target;
 604
 605         unsigned int vm_pageout_swap_wait;
 606         unsigned int vm_pageout_idle_wait;      /* milliseconds */
 607         unsigned int vm_pageout_empty_wait;     /* milliseconds */
 608         unsigned int vm_pageout_burst_wait;     /* milliseconds */
 609         unsigned int vm_pageout_deadlock_wait;  /* milliseconds */
 610         unsigned int vm_pageout_deadlock_relief;
 611         unsigned int vm_pageout_burst_inactive_throttle;
 612
 613         unsigned int vm_pageout_inactive;
 614         unsigned int vm_pageout_inactive_used;  /* debugging */
 615         unsigned int vm_pageout_inactive_clean; /* debugging */
 616
 617         uint32_t vm_page_filecache_min;
 618         uint32_t vm_page_filecache_min_divisor;
 619         uint32_t vm_page_xpmapped_min;
 620         uint32_t vm_page_xpmapped_min_divisor;
 621         uint64_t vm_pageout_considered_page_last;
 622
 623         int vm_page_free_count_init;
 624
 625         unsigned int vm_memory_pressure;
 626
 627         int memorystatus_purge_on_critical;
 628         int memorystatus_purge_on_warning;
 629         int memorystatus_purge_on_urgent;
 630
 631         thread_t vm_pageout_external_iothread;
 632         thread_t vm_pageout_internal_iothread;
 633 };
 634
 635 extern struct vm_pageout_state vm_pageout_state;
 636
 637 /*
 638  * This structure is used to track the VM_INFO instrumentation
 639  */
 640 struct vm_pageout_vminfo {
 641         unsigned long vm_pageout_considered_page;
 642         unsigned long vm_pageout_considered_bq_internal;
 643         unsigned long vm_pageout_considered_bq_external;
 644         unsigned long vm_pageout_skipped_external;
 645
 646         unsigned long vm_pageout_pages_evicted;
 647         unsigned long vm_pageout_pages_purged;
 648         unsigned long vm_pageout_freed_cleaned;
 649         unsigned long vm_pageout_freed_speculative;
 650         unsigned long vm_pageout_freed_external;
 651         unsigned long vm_pageout_freed_internal;
 652         unsigned long vm_pageout_inactive_dirty_internal;
 653         unsigned long vm_pageout_inactive_dirty_external;
 654         unsigned long vm_pageout_inactive_referenced;
 655         unsigned long vm_pageout_reactivation_limit_exceeded;
 656         unsigned long vm_pageout_inactive_force_reclaim;
 657         unsigned long vm_pageout_inactive_nolock;
 658         unsigned long vm_pageout_filecache_min_reactivated;
 659         unsigned long vm_pageout_scan_inactive_throttled_internal;
 660         unsigned long vm_pageout_scan_inactive_throttled_external;
 661
 662         uint64_t      vm_pageout_compressions;
 663         uint64_t      vm_compressor_pages_grabbed;
 664         unsigned long vm_compressor_failed;
 665
 666         unsigned long vm_page_pages_freed;
 667
 668         unsigned long vm_phantom_cache_found_ghost;
 669         unsigned long vm_phantom_cache_added_ghost;
 670 };
 671
 672 extern struct vm_pageout_vminfo vm_pageout_vminfo;
 673
 674
 675 #if DEVELOPMENT || DEBUG
 676
 677 /*
 678  *      This structure records the pageout daemon's actions:
 679  *      how many pages it looks at and what happens to those pages.
 680  *      No locking needed because only one thread modifies the fields.
 681  */
 682 struct vm_pageout_debug {
 683         uint32_t vm_pageout_balanced;
 684         uint32_t vm_pageout_scan_event_counter;
 685         uint32_t vm_pageout_speculative_dirty;
 686
 687         uint32_t vm_pageout_inactive_busy;
 688         uint32_t vm_pageout_inactive_absent;
 689         uint32_t vm_pageout_inactive_notalive;
 690         uint32_t vm_pageout_inactive_error;
 691         uint32_t vm_pageout_inactive_deactivated;
 692
 693         uint32_t vm_pageout_enqueued_cleaned;
 694
 695         uint32_t vm_pageout_cleaned_busy;
 696         uint32_t vm_pageout_cleaned_nolock;
 697         uint32_t vm_pageout_cleaned_reference_reactivated;
 698         uint32_t vm_pageout_cleaned_volatile_reactivated;
 699         uint32_t vm_pageout_cleaned_reactivated;  /* debugging; how many cleaned pages are found to be referenced on pageout (and are therefore reactivated) */
 700         uint32_t vm_pageout_cleaned_fault_reactivated;
 701
 702         uint32_t vm_pageout_dirty_no_pager;
 703         uint32_t vm_pageout_purged_objects;
 704
 705         uint32_t vm_pageout_scan_throttle;
 706         uint32_t vm_pageout_scan_reclaimed_throttled;
 707         uint32_t vm_pageout_scan_burst_throttle;
 708         uint32_t vm_pageout_scan_empty_throttle;
 709         uint32_t vm_pageout_scan_swap_throttle;
 710         uint32_t vm_pageout_scan_deadlock_detected;
 711         uint32_t vm_pageout_scan_inactive_throttle_success;
 712         uint32_t vm_pageout_scan_throttle_deferred;
 713
 714         uint32_t vm_pageout_inactive_external_forced_jetsam_count;
 715
 716         uint32_t vm_grab_anon_overrides;
 717         uint32_t vm_grab_anon_nops;
 718
 719         uint32_t vm_pageout_no_victim;
 720         unsigned long vm_pageout_throttle_up_count;
 721         uint32_t vm_page_steal_pageout_page;
 722
 723         uint32_t vm_cs_validated_resets;
 724         uint32_t vm_object_iopl_request_sleep_for_cleaning;
 725         uint32_t vm_page_slide_counter;
 726         uint32_t vm_page_slide_errors;
 727         uint32_t vm_page_throttle_count;
 728         /*
 729          * Statistics about UPL enforcement of copy-on-write obligations.
 730          */
 731         unsigned long upl_cow;
 732         unsigned long upl_cow_again;
 733         unsigned long upl_cow_pages;
 734         unsigned long upl_cow_again_pages;
 735         unsigned long iopl_cow;
 736         unsigned long iopl_cow_pages;
 737 };
 738
 739 extern struct vm_pageout_debug vm_pageout_debug;
 740
 741 #define VM_PAGEOUT_DEBUG(member, value)                 \
 742         MACRO_BEGIN                                     \
 743                 vm_pageout_debug.member += value;       \
 744         MACRO_END
 745 #else
 746 #define VM_PAGEOUT_DEBUG(member, value)
 747 #endif
 748
 749 #define MAX_COMPRESSOR_THREAD_COUNT      8
 750
 751 #if DEVELOPMENT || DEBUG
 752 typedef struct vmct_stats_s {
 753         uint64_t vmct_runtimes[MAX_COMPRESSOR_THREAD_COUNT];
 754         uint64_t vmct_pages[MAX_COMPRESSOR_THREAD_COUNT];
 755         uint64_t vmct_iterations[MAX_COMPRESSOR_THREAD_COUNT];
 756         uint64_t vmct_cthreads_total;
 757         int32_t vmct_minpages[MAX_COMPRESSOR_THREAD_COUNT];
 758         int32_t vmct_maxpages[MAX_COMPRESSOR_THREAD_COUNT];
 759 } vmct_stats_t;
 760 #endif
 761 #endif
 762 #endif  /* _VM_VM_PAGEOUT_H_ */