osfmk/vm/vm_compressor.h

   1 /*
   2  * Copyright (c) 2000-2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <kern/kalloc.h>
  30 #include <vm/vm_compressor_pager.h>
  31 #include <vm/vm_kern.h>
  32 #include <vm/vm_page.h>
  33 #include <vm/vm_protos.h>
  34 #include <vm/WKdm_new.h>
  35 #include <vm/vm_object.h>
  36 #include <machine/pmap.h>
  37 #include <kern/locks.h>
  38
  39 #include <sys/kdebug.h>
  40
  41
  42 #define C_SEG_OFFSET_BITS       16
  43 #define C_SEG_BUFSIZE           (1024 * 256)
  44 #define C_SEG_ALLOCSIZE         (C_SEG_BUFSIZE + PAGE_SIZE)
  45 #define C_SEG_OFF_LIMIT         (C_SEG_BYTES_TO_OFFSET((C_SEG_BUFSIZE - 512)))
  46
  47 #define C_SEG_SLOT_ARRAYS       6
  48 #define C_SEG_SLOT_ARRAY_SIZE   64              /* must be a power of 2 */
  49 #define C_SEG_SLOT_ARRAY_MASK   (C_SEG_SLOT_ARRAY_SIZE - 1)
  50 #define C_SLOT_MAX              (C_SEG_SLOT_ARRAYS * C_SEG_SLOT_ARRAY_SIZE)
  51
  52
  53 #define CHECKSUM_THE_SWAP               0       /* Debug swap data */
  54 #define CHECKSUM_THE_DATA               0       /* Debug compressor/decompressor data */
  55 #define CHECKSUM_THE_COMPRESSED_DATA    0       /* Debug compressor/decompressor compressed data */
  56 #define VALIDATE_C_SEGMENTS             0       /* Debug compaction */
  57 #define TRACK_BAD_C_SEGMENTS            0       /* Debug I/O error handling */
  58
  59 struct c_segment {
  60 #if __i386__ || __x86_64__
  61         lck_mtx_t       c_lock;
  62 #else /* __i386__ || __x86_64__ */
  63         lck_spin_t      c_lock;
  64 #endif /* __i386__ || __x86_64__ */
  65         queue_chain_t   c_age_list;
  66         queue_chain_t   c_list;
  67
  68         uint64_t        c_generation_id;
  69         int32_t         c_bytes_used;
  70         int32_t         c_bytes_unused;
  71
  72 #define C_SEG_MAX_LIMIT         (1 << 19)       /* this needs to track the size of c_mysegno */
  73         uint32_t        c_mysegno:19,
  74                         c_filling:1,
  75                         c_busy:1,
  76                         c_busy_swapping:1,
  77                         c_wanted:1,
  78                         c_must_free:1,
  79                         c_ondisk:1,
  80                         c_was_swapped_in:1,
  81                         c_on_minorcompact_q:1,  /* can also be on the age_q or the swappedin_q */
  82                         c_on_age_q:1,           /* creation age ordered list of in-core segments that
  83                                                    are available to be major-compacted and swapped out */
  84                         c_on_swappedin_q:1,     /* allows us to age newly swapped in segments */
  85                         c_on_swapout_q:1,       /* this is a transient queue */
  86                         c_on_swappedout_q:1,    /* segment has been major-compacted and
  87                                                    possibly swapped out to disk (c_ondisk == 1) */
  88                         c_on_swappedout_sparse_q:1;     /* segment has become sparse and should be garbage
  89                                                            collected if too many segments reach this state */
  90         uint16_t        c_firstemptyslot;
  91         uint16_t        c_nextslot;
  92         uint32_t        c_nextoffset;
  93         uint32_t        c_populated_offset;
  94
  95         uint32_t        c_creation_ts;
  96         uint32_t        c_swappedin_ts;
  97
  98         union {
  99                 int32_t *c_buffer;
 100                 uint64_t c_swap_handle;
 101         } c_store;
 102
 103 #if TRACK_BAD_C_SEGMENTS
 104         uint32_t        c_on_bad_q;
 105 #endif
 106
 107 #if     VALIDATE_C_SEGMENTS
 108         uint32_t        c_was_minor_compacted;
 109         uint32_t        c_was_major_compacted;
 110         uint32_t        c_was_major_donor;
 111 #endif
 112 #if CHECKSUM_THE_SWAP
 113         unsigned int    cseg_hash;
 114         unsigned int    cseg_swap_size;
 115 #endif /* CHECKSUM_THE_SWAP */
 116
 117         struct c_slot   *c_slots[C_SEG_SLOT_ARRAYS];
 118 };
 119
 120
 121 #define C_SEG_SLOT_FROM_INDEX(cseg, index)      (&(cseg->c_slots[index / C_SEG_SLOT_ARRAY_SIZE])[index & C_SEG_SLOT_ARRAY_MASK])
 122 #define C_SEG_SLOTARRAY_FROM_INDEX(cseg, index) (index / C_SEG_SLOT_ARRAY_SIZE)
 123
 124 #define C_SEG_OFFSET_TO_BYTES(off)      ((off) * (int) sizeof(int32_t))
 125 #define C_SEG_BYTES_TO_OFFSET(bytes)    ((bytes) / (int) sizeof(int32_t))
 126
 127 #define C_SEG_UNUSED_BYTES(cseg)        (cseg->c_bytes_unused + (C_SEG_OFFSET_TO_BYTES(cseg->c_populated_offset - cseg->c_nextoffset)))
 128
 129 #define C_SEG_OFFSET_ALIGNMENT_MASK     0x3
 130
 131 #define C_SEG_ONDISK_IS_SPARSE(cseg)    ((cseg->c_bytes_used < (C_SEG_BUFSIZE / 2)) ? 1 : 0)
 132 #define C_SEG_INCORE_IS_SPARSE(cseg)    ((C_SEG_UNUSED_BYTES(cseg) >= (C_SEG_BUFSIZE / 2)) ? 1 : 0)
 133
 134 #define C_SEG_WAKEUP_DONE(cseg)                         \
 135         MACRO_BEGIN                                     \
 136         (cseg)->c_busy = 0;                             \
 137         if ((cseg)->c_wanted) {                         \
 138                 (cseg)->c_wanted = 0;                   \
 139                 thread_wakeup((event_t) (cseg));        \
 140         }                                               \
 141         MACRO_END
 142
 143
 144 typedef struct c_segment *c_segment_t;
 145 typedef struct c_slot   *c_slot_t;
 146
 147 uint64_t vm_compressor_total_compressions(void);
 148 void vm_wake_compactor_swapper(void);
 149 void vm_consider_waking_compactor_swapper(void);
 150 void vm_compressor_flush(void);
 151 void c_seg_free(c_segment_t);
 152 void c_seg_free_locked(c_segment_t);
 153 void c_seg_insert_into_age_q(c_segment_t);
 154
 155 void vm_decompressor_lock(void);
 156 void vm_decompressor_unlock(void);
 157
 158 void vm_compressor_do_warmup(void);
 159 void vm_compressor_record_warmup_start(void);
 160 void vm_compressor_record_warmup_end(void);
 161
 162 int                     vm_low_on_space(void);
 163 boolean_t               vm_compression_available(void);
 164
 165 extern void             vm_compressor_init_locks(void);
 166 extern lck_rw_t         c_master_lock;
 167
 168 #if CRYPTO
 169 extern void             vm_swap_decrypt(c_segment_t);
 170 #endif /* CRYPTO */
 171
 172 extern kern_return_t    vm_swap_get(vm_offset_t, uint64_t, uint64_t);
 173 extern void             vm_swap_free(uint64_t);
 174 extern void             vm_swap_consider_defragmenting(void);
 175
 176 extern void             c_seg_swapin_requeue(c_segment_t);
 177 extern void             c_seg_swapin(c_segment_t, boolean_t);
 178 extern void             c_seg_wait_on_busy(c_segment_t);
 179 extern void             c_seg_trim_tail(c_segment_t);
 180
 181 extern boolean_t        fastwake_recording_in_progress;
 182 extern int              compaction_swapper_running;
 183 extern uint64_t         vm_swap_put_failures;
 184
 185 extern queue_head_t     c_minor_list_head;
 186 extern queue_head_t     c_age_list_head;
 187 extern queue_head_t     c_swapout_list_head;
 188 extern queue_head_t     c_swappedout_list_head;
 189 extern queue_head_t     c_swappedout_sparse_list_head;
 190
 191 extern uint32_t         c_age_count;
 192 extern uint32_t         c_swapout_count;
 193 extern uint32_t         c_swappedout_count;
 194 extern uint32_t         c_swappedout_sparse_count;
 195
 196 extern int64_t          compressor_bytes_used;
 197 extern uint64_t         first_c_segment_to_warm_generation_id;
 198 extern uint64_t         last_c_segment_to_warm_generation_id;
 199 extern boolean_t        hibernate_flushing;
 200 extern boolean_t        hibernate_no_swapspace;
 201 extern uint32_t         swapout_target_age;
 202
 203 extern void c_seg_insert_into_q(queue_head_t *, c_segment_t);
 204
 205 extern uint32_t vm_compressor_minorcompact_threshold_divisor;
 206 extern uint32_t vm_compressor_majorcompact_threshold_divisor;
 207 extern uint32_t vm_compressor_unthrottle_threshold_divisor;
 208 extern uint32_t vm_compressor_catchup_threshold_divisor;
 209
 210 #define PAGE_REPLACEMENT_DISALLOWED(enable)     (enable == TRUE ? lck_rw_lock_shared(&c_master_lock) : lck_rw_done(&c_master_lock))
 211 #define PAGE_REPLACEMENT_ALLOWED(enable)        (enable == TRUE ? lck_rw_lock_exclusive(&c_master_lock) : lck_rw_done(&c_master_lock))
 212
 213
 214 #define AVAILABLE_NON_COMPRESSED_MEMORY         (vm_page_active_count + vm_page_inactive_count + vm_page_free_count + vm_page_speculative_count)
 215 #define AVAILABLE_MEMORY                        (AVAILABLE_NON_COMPRESSED_MEMORY + VM_PAGE_COMPRESSOR_COUNT)
 216
 217 #define VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD            (((AVAILABLE_MEMORY) * 10) / (vm_compressor_minorcompact_threshold_divisor ? vm_compressor_minorcompact_threshold_divisor : 1))
 218 #define VM_PAGE_COMPRESSOR_SWAP_THRESHOLD               (((AVAILABLE_MEMORY) * 10) / (vm_compressor_majorcompact_threshold_divisor ? vm_compressor_majorcompact_threshold_divisor : 1))
 219 #define VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD    (((AVAILABLE_MEMORY) * 10) / (vm_compressor_unthrottle_threshold_divisor ? vm_compressor_unthrottle_threshold_divisor : 1))
 220 #define VM_PAGE_COMPRESSOR_SWAP_CATCHUP_THRESHOLD       (((AVAILABLE_MEMORY) * 10) / (vm_compressor_catchup_threshold_divisor ? vm_compressor_catchup_threshold_divisor : 1))
 221
 222 #define COMPRESSOR_NEEDS_TO_SWAP()              ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_THRESHOLD) ? 1 : 0)
 223
 224 #define VM_PAGEOUT_SCAN_NEEDS_TO_THROTTLE()                             \
 225         ((vm_compressor_mode == VM_PAGER_COMPRESSOR_WITH_SWAP ||        \
 226           vm_compressor_mode == VM_PAGER_FREEZER_COMPRESSOR_WITH_SWAP) && \
 227          ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_CATCHUP_THRESHOLD) ? 1 : 0))
 228 #define HARD_THROTTLE_LIMIT_REACHED()           ((AVAILABLE_NON_COMPRESSED_MEMORY < (VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) / 2) ? 1 : 0)
 229 #define SWAPPER_NEEDS_TO_UNTHROTTLE()           ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) ? 1 : 0)
 230 #define COMPRESSOR_NEEDS_TO_MINOR_COMPACT()     ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD) ? 1 : 0)
 231 #define COMPRESSOR_NEEDS_TO_MAJOR_COMPACT()     ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_THRESHOLD) ? 1 : 0)
 232
 233 #define COMPRESSOR_FREE_RESERVED_LIMIT          28
 234
 235 /*
 236  * Upward trajectory.
 237  */
 238 extern boolean_t vm_compressor_low_on_space(void);
 239
 240 #define VM_PRESSURE_NORMAL_TO_WARNING()         ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD) ? 1 : 0)
 241 #define VM_PRESSURE_WARNING_TO_CRITICAL()       (vm_compressor_low_on_space() || (AVAILABLE_NON_COMPRESSED_MEMORY < ((12 * VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) / 10)) ? 1 : 0)
 242
 243 /*
 244  * Downward trajectory.
 245  */
 246 #define VM_PRESSURE_WARNING_TO_NORMAL()         ((AVAILABLE_NON_COMPRESSED_MEMORY > ((12 * VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD) / 10)) ? 1 : 0)
 247 #define VM_PRESSURE_CRITICAL_TO_WARNING()       ((AVAILABLE_NON_COMPRESSED_MEMORY > ((14 * VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) / 10)) ? 1 : 0)
 248
 249 #define COMPRESSOR_SCRATCH_BUF_SIZE WKdm_SCRATCH_BUF_SIZE
 250
 251
 252 #if __i386__ || __x86_64__
 253 extern lck_mtx_t        *c_list_lock;
 254 #else /* __i386__ || __x86_64__ */
 255 extern lck_spin_t       *c_list_lock;
 256 #endif /* __i386__ || __x86_64__ */