osfmk/vm/vm_compressor.h

   1 /*
   2  * Copyright (c) 2000-2016 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <kern/kalloc.h>
  30 #include <vm/vm_compressor_pager.h>
  31 #include <vm/vm_kern.h>
  32 #include <vm/vm_page.h>
  33 #include <vm/vm_protos.h>
  34 #include <vm/WKdm_new.h>
  35 #include <vm/vm_object.h>
  36 #include <vm/vm_map.h>
  37 #include <machine/pmap.h>
  38 #include <kern/locks.h>
  39
  40 #include <sys/kdebug.h>
  41
  42 #if defined(__arm64__)
  43 #include <arm/proc_reg.h>
  44 #endif
  45
  46 #define C_SEG_OFFSET_BITS       16
  47 #define C_SEG_BUFSIZE           (1024 * 256)
  48 #define C_SEG_MAX_PAGES         (C_SEG_BUFSIZE / PAGE_SIZE)
  49
  50 #if CONFIG_EMBEDDED
  51 #define C_SEG_OFF_LIMIT         (C_SEG_BYTES_TO_OFFSET((C_SEG_BUFSIZE - 512)))
  52 #define C_SEG_ALLOCSIZE         (C_SEG_BUFSIZE + PAGE_SIZE)
  53 #else
  54 #define C_SEG_OFF_LIMIT         (C_SEG_BYTES_TO_OFFSET((C_SEG_BUFSIZE - 128)))
  55 #define C_SEG_ALLOCSIZE         (C_SEG_BUFSIZE)
  56 #endif
  57 #define C_SEG_MAX_POPULATE_SIZE (4 * PAGE_SIZE)
  58
  59 #if defined(__arm64__)
  60
  61 #if DEVELOPMENT || DEBUG
  62
  63
  64 #endif
  65
  66 #endif
  67
  68
  69 #if DEBUG || COMPRESSOR_INTEGRITY_CHECKS
  70 #define ENABLE_SWAP_CHECKS 1
  71 #define ENABLE_COMPRESSOR_CHECKS 1
  72 #define POPCOUNT_THE_COMPRESSED_DATA (1)
  73 #else
  74 #define ENABLE_SWAP_CHECKS 0
  75 #define ENABLE_COMPRESSOR_CHECKS 0
  76 #endif
  77
  78 #define CHECKSUM_THE_SWAP               ENABLE_SWAP_CHECKS      /* Debug swap data */
  79 #define CHECKSUM_THE_DATA               ENABLE_COMPRESSOR_CHECKS        /* Debug compressor/decompressor data */
  80 #define CHECKSUM_THE_COMPRESSED_DATA    ENABLE_COMPRESSOR_CHECKS        /* Debug compressor/decompressor compressed data */
  81 #define VALIDATE_C_SEGMENTS             ENABLE_COMPRESSOR_CHECKS        /* Debug compaction */
  82
  83 #define RECORD_THE_COMPRESSED_DATA      0
  84
  85 struct c_slot {
  86         uint64_t        c_offset:C_SEG_OFFSET_BITS,
  87 #if defined(__arm64__)
  88                         c_size:14,
  89                         c_codec:1,
  90                         c_packed_ptr:33;
  91 #elif defined(__arm__)
  92                         c_size:12,
  93                         c_codec:1,
  94                         c_packed_ptr:35;
  95 #else
  96                         c_size:12,
  97                         c_packed_ptr:36;
  98 #endif
  99 #if CHECKSUM_THE_DATA
 100         unsigned int    c_hash_data;
 101 #endif
 102 #if CHECKSUM_THE_COMPRESSED_DATA
 103         unsigned int    c_hash_compressed_data;
 104 #endif
 105 #if POPCOUNT_THE_COMPRESSED_DATA
 106         unsigned int    c_pop_cdata;
 107 #endif
 108 };
 109
 110 #define C_IS_EMPTY              0
 111 #define C_IS_FREE               1
 112 #define C_IS_FILLING            2
 113 #define C_ON_AGE_Q              3
 114 #define C_ON_SWAPOUT_Q          4
 115 #define C_ON_SWAPPEDOUT_Q       5
 116 #define C_ON_SWAPPEDOUTSPARSE_Q 6
 117 #define C_ON_SWAPPEDIN_Q        7
 118 #define C_ON_MAJORCOMPACT_Q     8
 119 #define C_ON_BAD_Q              9
 120
 121
 122 struct c_segment {
 123         lck_mtx_t       c_lock;
 124         queue_chain_t   c_age_list;
 125         queue_chain_t   c_list;
 126
 127 #define C_SEG_MAX_LIMIT         (1 << 20)       /* this needs to track the size of c_mysegno */
 128         uint32_t        c_mysegno:20,
 129                         c_busy:1,
 130                         c_busy_swapping:1,
 131                         c_wanted:1,
 132                         c_on_minorcompact_q:1,  /* can also be on the age_q, the majorcompact_q or the swappedin_q */
 133
 134                         c_state:4,              /* what state is the segment in which dictates which q to find it on */
 135                         c_overage_swap:1,
 136                         c_reserved:3;
 137
 138         uint32_t        c_creation_ts;
 139         uint64_t        c_generation_id;
 140
 141         int32_t         c_bytes_used;
 142         int32_t         c_bytes_unused;
 143         uint32_t        c_slots_used;
 144
 145         uint16_t        c_firstemptyslot;
 146         uint16_t        c_nextslot;
 147         uint32_t        c_nextoffset;
 148         uint32_t        c_populated_offset;
 149
 150         uint32_t        c_swappedin_ts;
 151
 152         union {
 153                 int32_t *c_buffer;
 154                 uint64_t c_swap_handle;
 155         } c_store;
 156
 157 #if     VALIDATE_C_SEGMENTS
 158         uint32_t        c_was_minor_compacted;
 159         uint32_t        c_was_major_compacted;
 160         uint32_t        c_was_major_donor;
 161 #endif
 162 #if CHECKSUM_THE_SWAP
 163         unsigned int    cseg_hash;
 164         unsigned int    cseg_swap_size;
 165 #endif /* CHECKSUM_THE_SWAP */
 166
 167 #if MACH_ASSERT
 168         thread_t        c_busy_for_thread;
 169 #endif /* MACH_ASSERT */
 170
 171         int             c_slot_var_array_len;
 172         struct  c_slot  *c_slot_var_array;
 173         struct  c_slot  c_slot_fixed_array[0];
 174 };
 175
 176
 177 struct  c_slot_mapping {
 178         uint32_t        s_cseg:22,      /* segment number + 1 */
 179                         s_cindx:10;     /* index in the segment */
 180 };
 181 #define C_SLOT_MAX_INDEX        (1 << 10)
 182
 183 typedef struct c_slot_mapping *c_slot_mapping_t;
 184
 185
 186 #define C_SEG_SLOT_VAR_ARRAY_MIN_LEN    C_SEG_MAX_PAGES
 187
 188 extern  int             c_seg_fixed_array_len;
 189 extern  vm_offset_t     c_buffers;
 190 #define C_SEG_BUFFER_ADDRESS(c_segno)   ((c_buffers + ((uint64_t)c_segno * (uint64_t)C_SEG_ALLOCSIZE)))
 191
 192 #define C_SEG_SLOT_FROM_INDEX(cseg, index)      (index < c_seg_fixed_array_len ? &(cseg->c_slot_fixed_array[index]) : &(cseg->c_slot_var_array[index - c_seg_fixed_array_len]))
 193
 194 #define C_SEG_OFFSET_TO_BYTES(off)      ((off) * (int) sizeof(int32_t))
 195 #define C_SEG_BYTES_TO_OFFSET(bytes)    ((bytes) / (int) sizeof(int32_t))
 196
 197 #define C_SEG_UNUSED_BYTES(cseg)        (cseg->c_bytes_unused + (C_SEG_OFFSET_TO_BYTES(cseg->c_populated_offset - cseg->c_nextoffset)))
 198 //todo opensource
 199
 200 #ifndef __PLATFORM_WKDM_ALIGNMENT_MASK__
 201 #define C_SEG_OFFSET_ALIGNMENT_MASK     0x3ULL
 202 #define C_SEG_OFFSET_ALIGNMENT_BOUNDARY 0x4
 203 #else
 204 #define C_SEG_OFFSET_ALIGNMENT_MASK     __PLATFORM_WKDM_ALIGNMENT_MASK__
 205 #define C_SEG_OFFSET_ALIGNMENT_BOUNDARY __PLATFORM_WKDM_ALIGNMENT_BOUNDARY__
 206 #endif
 207
 208 #define C_SEG_SHOULD_MINORCOMPACT_NOW(cseg)     ((C_SEG_UNUSED_BYTES(cseg) >= (C_SEG_BUFSIZE / 4)) ? 1 : 0)
 209
 210 /*
 211  * the decsion to force a c_seg to be major compacted is based on 2 criteria
 212  * 1) is the c_seg buffer almost empty (i.e. we have a chance to merge it with another c_seg)
 213  * 2) are there at least a minimum number of slots unoccupied so that we have a chance
 214  *    of combining this c_seg with another one.
 215  */
 216 #define C_SEG_SHOULD_MAJORCOMPACT_NOW(cseg)                                                                                     \
 217         ((((cseg->c_bytes_unused + (C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES(c_seg->c_nextoffset))) >= (C_SEG_BUFSIZE / 8)) &&     \
 218           ((C_SLOT_MAX_INDEX - cseg->c_slots_used) > (C_SEG_BUFSIZE / PAGE_SIZE))) \
 219         ? 1 : 0)
 220
 221 #define C_SEG_ONDISK_IS_SPARSE(cseg)    ((cseg->c_bytes_used < cseg->c_bytes_unused) ? 1 : 0)
 222 #define C_SEG_IS_ONDISK(cseg)           ((cseg->c_state == C_ON_SWAPPEDOUT_Q || cseg->c_state == C_ON_SWAPPEDOUTSPARSE_Q))
 223 #define C_SEG_IS_ON_DISK_OR_SOQ(cseg)   ((cseg->c_state == C_ON_SWAPPEDOUT_Q || \
 224                                           cseg->c_state == C_ON_SWAPPEDOUTSPARSE_Q || \
 225                                           cseg->c_state == C_ON_SWAPOUT_Q))
 226
 227
 228 #define C_SEG_WAKEUP_DONE(cseg)                         \
 229         MACRO_BEGIN                                     \
 230         assert((cseg)->c_busy);                         \
 231         (cseg)->c_busy = 0;                             \
 232         assert((cseg)->c_busy_for_thread != NULL);      \
 233         assert((((cseg)->c_busy_for_thread = NULL), TRUE));     \
 234         if ((cseg)->c_wanted) {                         \
 235                 (cseg)->c_wanted = 0;                   \
 236                 thread_wakeup((event_t) (cseg));        \
 237         }                                               \
 238         MACRO_END
 239
 240 #define C_SEG_BUSY(cseg)                                \
 241         MACRO_BEGIN                                     \
 242         assert((cseg)->c_busy == 0);                    \
 243         (cseg)->c_busy = 1;                             \
 244         assert((cseg)->c_busy_for_thread == NULL);      \
 245         assert((((cseg)->c_busy_for_thread = current_thread()), TRUE)); \
 246         MACRO_END
 247
 248
 249 extern vm_map_t compressor_map;
 250
 251 #if DEVELOPMENT || DEBUG
 252 extern boolean_t write_protect_c_segs;
 253 extern int vm_compressor_test_seg_wp;
 254
 255 #define C_SEG_MAKE_WRITEABLE(cseg)                      \
 256         MACRO_BEGIN                                     \
 257         if (write_protect_c_segs) {                     \
 258                 vm_map_protect(compressor_map,                  \
 259                                (vm_map_offset_t)cseg->c_store.c_buffer,         \
 260                                (vm_map_offset_t)&cseg->c_store.c_buffer[C_SEG_BYTES_TO_OFFSET(C_SEG_ALLOCSIZE)],\
 261                                VM_PROT_READ | VM_PROT_WRITE,    \
 262                                0);                              \
 263         }                               \
 264         MACRO_END
 265
 266 #define C_SEG_WRITE_PROTECT(cseg)                       \
 267         MACRO_BEGIN                                     \
 268         if (write_protect_c_segs) {                     \
 269                 vm_map_protect(compressor_map,                  \
 270                                (vm_map_offset_t)cseg->c_store.c_buffer,         \
 271                                (vm_map_offset_t)&cseg->c_store.c_buffer[C_SEG_BYTES_TO_OFFSET(C_SEG_ALLOCSIZE)],\
 272                                VM_PROT_READ,                    \
 273                                0);                              \
 274         }                                                       \
 275         if (vm_compressor_test_seg_wp) {                                \
 276                 volatile uint32_t vmtstmp = *(volatile uint32_t *)cseg->c_store.c_buffer; \
 277                 *(volatile uint32_t *)cseg->c_store.c_buffer = 0xDEADABCD; \
 278                 (void) vmtstmp;                                         \
 279         }                                                               \
 280         MACRO_END
 281 #endif
 282
 283 typedef struct c_segment *c_segment_t;
 284 typedef struct c_slot   *c_slot_t;
 285
 286 uint64_t vm_compressor_total_compressions(void);
 287 void vm_wake_compactor_swapper(void);
 288 void vm_run_compactor(void);
 289 void vm_thrashing_jetsam_done(void);
 290 void vm_consider_waking_compactor_swapper(void);
 291 void vm_consider_swapping(void);
 292 void vm_compressor_flush(void);
 293 void c_seg_free(c_segment_t);
 294 void c_seg_free_locked(c_segment_t);
 295 void c_seg_insert_into_age_q(c_segment_t);
 296 void c_seg_need_delayed_compaction(c_segment_t, boolean_t);
 297
 298 void vm_decompressor_lock(void);
 299 void vm_decompressor_unlock(void);
 300
 301 void vm_compressor_delay_trim(void);
 302 void vm_compressor_do_warmup(void);
 303 void vm_compressor_record_warmup_start(void);
 304 void vm_compressor_record_warmup_end(void);
 305
 306 int                     vm_wants_task_throttled(task_t);
 307
 308 extern void             vm_compaction_swapper_do_init(void);
 309 extern void             vm_compressor_swap_init(void);
 310 extern void             vm_compressor_init_locks(void);
 311 extern lck_rw_t         c_master_lock;
 312
 313 #if ENCRYPTED_SWAP
 314 extern void             vm_swap_decrypt(c_segment_t);
 315 #endif /* ENCRYPTED_SWAP */
 316
 317 extern int              vm_swap_low_on_space(void);
 318 extern kern_return_t    vm_swap_get(c_segment_t, uint64_t, uint64_t);
 319 extern void             vm_swap_free(uint64_t);
 320 extern void             vm_swap_consider_defragmenting(void);
 321
 322 extern void             c_seg_swapin_requeue(c_segment_t, boolean_t, boolean_t, boolean_t);
 323 extern int              c_seg_swapin(c_segment_t, boolean_t, boolean_t);
 324 extern void             c_seg_wait_on_busy(c_segment_t);
 325 extern void             c_seg_trim_tail(c_segment_t);
 326 extern void             c_seg_switch_state(c_segment_t, int, boolean_t);
 327
 328 extern boolean_t        fastwake_recording_in_progress;
 329 extern int              compaction_swapper_inited;
 330 extern int              compaction_swapper_running;
 331 extern uint64_t         vm_swap_put_failures;
 332
 333 extern int              c_overage_swapped_count;
 334 extern int              c_overage_swapped_limit;
 335
 336 extern queue_head_t     c_minor_list_head;
 337 extern queue_head_t     c_age_list_head;
 338 extern queue_head_t     c_swapout_list_head;
 339 extern queue_head_t     c_swappedout_list_head;
 340 extern queue_head_t     c_swappedout_sparse_list_head;
 341
 342 extern uint32_t         c_age_count;
 343 extern uint32_t         c_swapout_count;
 344 extern uint32_t         c_swappedout_count;
 345 extern uint32_t         c_swappedout_sparse_count;
 346
 347 extern int64_t          compressor_bytes_used;
 348 extern uint64_t         first_c_segment_to_warm_generation_id;
 349 extern uint64_t         last_c_segment_to_warm_generation_id;
 350 extern boolean_t        hibernate_flushing;
 351 extern boolean_t        hibernate_no_swapspace;
 352 extern boolean_t        hibernate_in_progress_with_pinned_swap;
 353 extern uint32_t         swapout_target_age;
 354
 355 extern void c_seg_insert_into_q(queue_head_t *, c_segment_t);
 356
 357 extern uint32_t vm_compressor_minorcompact_threshold_divisor;
 358 extern uint32_t vm_compressor_majorcompact_threshold_divisor;
 359 extern uint32_t vm_compressor_unthrottle_threshold_divisor;
 360 extern uint32_t vm_compressor_catchup_threshold_divisor;
 361 extern uint64_t vm_compressor_compute_elapsed_msecs(clock_sec_t, clock_nsec_t, clock_sec_t, clock_nsec_t);
 362
 363 #define PAGE_REPLACEMENT_DISALLOWED(enable)     (enable == TRUE ? lck_rw_lock_shared(&c_master_lock) : lck_rw_done(&c_master_lock))
 364 #define PAGE_REPLACEMENT_ALLOWED(enable)        (enable == TRUE ? lck_rw_lock_exclusive(&c_master_lock) : lck_rw_done(&c_master_lock))
 365
 366
 367 #define AVAILABLE_NON_COMPRESSED_MEMORY         (vm_page_active_count + vm_page_inactive_count + vm_page_free_count + vm_page_speculative_count)
 368 #define AVAILABLE_MEMORY                        (AVAILABLE_NON_COMPRESSED_MEMORY + VM_PAGE_COMPRESSOR_COUNT)
 369 /* TODO, there may be a minor optimisation opportunity to replace these divisions
 370  * with multiplies and shifts
 371  */
 372
 373 #define VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD            (((AVAILABLE_MEMORY) * 10) / (vm_compressor_minorcompact_threshold_divisor ? vm_compressor_minorcompact_threshold_divisor : 1))
 374 #define VM_PAGE_COMPRESSOR_SWAP_THRESHOLD               (((AVAILABLE_MEMORY) * 10) / (vm_compressor_majorcompact_threshold_divisor ? vm_compressor_majorcompact_threshold_divisor : 1))
 375 #define VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD    (((AVAILABLE_MEMORY) * 10) / (vm_compressor_unthrottle_threshold_divisor ? vm_compressor_unthrottle_threshold_divisor : 1))
 376 #define VM_PAGE_COMPRESSOR_SWAP_CATCHUP_THRESHOLD       (((AVAILABLE_MEMORY) * 10) / (vm_compressor_catchup_threshold_divisor ? vm_compressor_catchup_threshold_divisor : 1))
 377
 378 #ifdef  CONFIG_EMBEDDED
 379 #define AVAILABLE_NON_COMPRESSED_MIN                    20000
 380 #define COMPRESSOR_NEEDS_TO_SWAP()              (((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_THRESHOLD) || \
 381                                                   (AVAILABLE_NON_COMPRESSED_MEMORY < AVAILABLE_NON_COMPRESSED_MIN)) ? 1 : 0)
 382 #else
 383 #define COMPRESSOR_NEEDS_TO_SWAP()              ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_THRESHOLD) ? 1 : 0)
 384 #endif
 385
 386 #define VM_PAGEOUT_SCAN_NEEDS_TO_THROTTLE()                             \
 387         (vm_compressor_mode == VM_PAGER_COMPRESSOR_WITH_SWAP &&         \
 388          ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_CATCHUP_THRESHOLD) ? 1 : 0))
 389 #define HARD_THROTTLE_LIMIT_REACHED()           ((AVAILABLE_NON_COMPRESSED_MEMORY < (VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) / 2) ? 1 : 0)
 390 #define SWAPPER_NEEDS_TO_UNTHROTTLE()           ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) ? 1 : 0)
 391 #define COMPRESSOR_NEEDS_TO_MINOR_COMPACT()     ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD) ? 1 : 0)
 392
 393
 394 #ifdef  CONFIG_EMBEDDED
 395 #define COMPRESSOR_FREE_RESERVED_LIMIT          28
 396 #else
 397 #define COMPRESSOR_FREE_RESERVED_LIMIT          128
 398 #endif
 399
 400 uint32_t vm_compressor_get_encode_scratch_size(void);
 401 uint32_t vm_compressor_get_decode_scratch_size(void);
 402
 403 #define COMPRESSOR_SCRATCH_BUF_SIZE vm_compressor_get_encode_scratch_size()
 404
 405 #if RECORD_THE_COMPRESSED_DATA
 406 extern void      c_compressed_record_init(void);
 407 extern void      c_compressed_record_write(char *, int);
 408 #endif
 409
 410 extern lck_mtx_t        *c_list_lock;
 411
 412 #if DEVELOPMENT || DEBUG
 413 extern uint32_t vm_ktrace_enabled;
 414
 415 #define VMKDBG(x, ...)          \
 416 MACRO_BEGIN                     \
 417 if (vm_ktrace_enabled) {        \
 418         KDBG(x, ## __VA_ARGS__);\
 419 }                               \
 420 MACRO_END
 421 #endif