xnu-4570.31.3.tar.gz
[apple/xnu.git] / osfmk / vm / vm_compressor.h
1 /*
2 * Copyright (c) 2000-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <kern/kalloc.h>
30 #include <vm/vm_compressor_pager.h>
31 #include <vm/vm_kern.h>
32 #include <vm/vm_page.h>
33 #include <vm/vm_protos.h>
34 #include <vm/WKdm_new.h>
35 #include <vm/vm_object.h>
36 #include <vm/vm_map.h>
37 #include <machine/pmap.h>
38 #include <kern/locks.h>
39
40 #include <sys/kdebug.h>
41
42 #if defined(__arm64__)
43 #include <arm/proc_reg.h>
44 #endif
45
46 #define C_SEG_OFFSET_BITS 16
47 #define C_SEG_BUFSIZE (1024 * 256)
48 #define C_SEG_MAX_PAGES (C_SEG_BUFSIZE / PAGE_SIZE)
49
50 #if CONFIG_EMBEDDED
51 #define C_SEG_OFF_LIMIT (C_SEG_BYTES_TO_OFFSET((C_SEG_BUFSIZE - 512)))
52 #define C_SEG_ALLOCSIZE (C_SEG_BUFSIZE + PAGE_SIZE)
53 #else
54 #define C_SEG_OFF_LIMIT (C_SEG_BYTES_TO_OFFSET((C_SEG_BUFSIZE - 128)))
55 #define C_SEG_ALLOCSIZE (C_SEG_BUFSIZE)
56 #endif
57 #define C_SEG_MAX_POPULATE_SIZE (4 * PAGE_SIZE)
58
59 #if defined(__arm64__)
60
61 #if DEVELOPMENT || DEBUG
62
63
64 #endif
65
66 #endif
67
68 #if DEBUG || COMPRESSOR_INTEGRITY_CHECKS
69 #define ENABLE_SWAP_CHECKS 1
70 #define ENABLE_COMPRESSOR_CHECKS 1
71 #define POPCOUNT_THE_COMPRESSED_DATA (1)
72 #else
73 #define ENABLE_SWAP_CHECKS 0
74 #define ENABLE_COMPRESSOR_CHECKS 0
75 #endif
76
77 #define CHECKSUM_THE_SWAP ENABLE_SWAP_CHECKS /* Debug swap data */
78 #define CHECKSUM_THE_DATA ENABLE_COMPRESSOR_CHECKS /* Debug compressor/decompressor data */
79 #define CHECKSUM_THE_COMPRESSED_DATA ENABLE_COMPRESSOR_CHECKS /* Debug compressor/decompressor compressed data */
80 #define VALIDATE_C_SEGMENTS ENABLE_COMPRESSOR_CHECKS /* Debug compaction */
81
82 #define RECORD_THE_COMPRESSED_DATA 0
83
84 struct c_slot {
85 uint64_t c_offset:C_SEG_OFFSET_BITS,
86 #if defined(__arm64__)
87 c_size:14,
88 c_codec:1,
89 c_packed_ptr:33;
90 #elif defined(__arm__)
91 c_size:12,
92 c_codec:1,
93 c_packed_ptr:35;
94 #else
95 c_size:12,
96 c_packed_ptr:36;
97 #endif
98 #if CHECKSUM_THE_DATA
99 unsigned int c_hash_data;
100 #endif
101 #if CHECKSUM_THE_COMPRESSED_DATA
102 unsigned int c_hash_compressed_data;
103 #endif
104 #if POPCOUNT_THE_COMPRESSED_DATA
105 unsigned int c_pop_cdata;
106 #endif
107 };
108
109 #define C_IS_EMPTY 0
110 #define C_IS_FREE 1
111 #define C_IS_FILLING 2
112 #define C_ON_AGE_Q 3
113 #define C_ON_SWAPOUT_Q 4
114 #define C_ON_SWAPPEDOUT_Q 5
115 #define C_ON_SWAPPEDOUTSPARSE_Q 6
116 #define C_ON_SWAPPEDIN_Q 7
117 #define C_ON_MAJORCOMPACT_Q 8
118 #define C_ON_BAD_Q 9
119
120
121 struct c_segment {
122 lck_mtx_t c_lock;
123 queue_chain_t c_age_list;
124 queue_chain_t c_list;
125
126 #define C_SEG_MAX_LIMIT (1 << 20) /* this needs to track the size of c_mysegno */
127 uint32_t c_mysegno:20,
128 c_busy:1,
129 c_busy_swapping:1,
130 c_wanted:1,
131 c_on_minorcompact_q:1, /* can also be on the age_q, the majorcompact_q or the swappedin_q */
132
133 c_state:4, /* what state is the segment in which dictates which q to find it on */
134 c_overage_swap:1,
135 c_reserved:3;
136
137 uint32_t c_creation_ts;
138 uint64_t c_generation_id;
139
140 int32_t c_bytes_used;
141 int32_t c_bytes_unused;
142 uint32_t c_slots_used;
143
144 uint16_t c_firstemptyslot;
145 uint16_t c_nextslot;
146 uint32_t c_nextoffset;
147 uint32_t c_populated_offset;
148
149 uint32_t c_swappedin_ts;
150
151 union {
152 int32_t *c_buffer;
153 uint64_t c_swap_handle;
154 } c_store;
155
156 #if VALIDATE_C_SEGMENTS
157 uint32_t c_was_minor_compacted;
158 uint32_t c_was_major_compacted;
159 uint32_t c_was_major_donor;
160 #endif
161 #if CHECKSUM_THE_SWAP
162 unsigned int cseg_hash;
163 unsigned int cseg_swap_size;
164 #endif /* CHECKSUM_THE_SWAP */
165
166 #if MACH_ASSERT
167 thread_t c_busy_for_thread;
168 #endif /* MACH_ASSERT */
169
170 int c_slot_var_array_len;
171 struct c_slot *c_slot_var_array;
172 struct c_slot c_slot_fixed_array[0];
173 };
174
175
176 struct c_slot_mapping {
177 uint32_t s_cseg:22, /* segment number + 1 */
178 s_cindx:10; /* index in the segment */
179 };
180 #define C_SLOT_MAX_INDEX (1 << 10)
181
182 typedef struct c_slot_mapping *c_slot_mapping_t;
183
184
185 #define C_SEG_SLOT_VAR_ARRAY_MIN_LEN C_SEG_MAX_PAGES
186
187 extern int c_seg_fixed_array_len;
188 extern vm_offset_t c_buffers;
189 #define C_SEG_BUFFER_ADDRESS(c_segno) ((c_buffers + ((uint64_t)c_segno * (uint64_t)C_SEG_ALLOCSIZE)))
190
191 #define C_SEG_SLOT_FROM_INDEX(cseg, index) (index < c_seg_fixed_array_len ? &(cseg->c_slot_fixed_array[index]) : &(cseg->c_slot_var_array[index - c_seg_fixed_array_len]))
192
193 #define C_SEG_OFFSET_TO_BYTES(off) ((off) * (int) sizeof(int32_t))
194 #define C_SEG_BYTES_TO_OFFSET(bytes) ((bytes) / (int) sizeof(int32_t))
195
196 #define C_SEG_UNUSED_BYTES(cseg) (cseg->c_bytes_unused + (C_SEG_OFFSET_TO_BYTES(cseg->c_populated_offset - cseg->c_nextoffset)))
197 //todo opensource
198
199 #ifndef __PLATFORM_WKDM_ALIGNMENT_MASK__
200 #define C_SEG_OFFSET_ALIGNMENT_MASK 0x3ULL
201 #define C_SEG_OFFSET_ALIGNMENT_BOUNDARY 0x4
202 #else
203 #define C_SEG_OFFSET_ALIGNMENT_MASK __PLATFORM_WKDM_ALIGNMENT_MASK__
204 #define C_SEG_OFFSET_ALIGNMENT_BOUNDARY __PLATFORM_WKDM_ALIGNMENT_BOUNDARY__
205 #endif
206
207 #define C_SEG_SHOULD_MINORCOMPACT_NOW(cseg) ((C_SEG_UNUSED_BYTES(cseg) >= (C_SEG_BUFSIZE / 4)) ? 1 : 0)
208
209 /*
210 * the decsion to force a c_seg to be major compacted is based on 2 criteria
211 * 1) is the c_seg buffer almost empty (i.e. we have a chance to merge it with another c_seg)
212 * 2) are there at least a minimum number of slots unoccupied so that we have a chance
213 * of combining this c_seg with another one.
214 */
215 #define C_SEG_SHOULD_MAJORCOMPACT_NOW(cseg) \
216 ((((cseg->c_bytes_unused + (C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES(c_seg->c_nextoffset))) >= (C_SEG_BUFSIZE / 8)) && \
217 ((C_SLOT_MAX_INDEX - cseg->c_slots_used) > (C_SEG_BUFSIZE / PAGE_SIZE))) \
218 ? 1 : 0)
219
220 #define C_SEG_ONDISK_IS_SPARSE(cseg) ((cseg->c_bytes_used < cseg->c_bytes_unused) ? 1 : 0)
221 #define C_SEG_IS_ONDISK(cseg) ((cseg->c_state == C_ON_SWAPPEDOUT_Q || cseg->c_state == C_ON_SWAPPEDOUTSPARSE_Q))
222
223
224 #define C_SEG_WAKEUP_DONE(cseg) \
225 MACRO_BEGIN \
226 assert((cseg)->c_busy); \
227 (cseg)->c_busy = 0; \
228 assert((cseg)->c_busy_for_thread != NULL); \
229 assert((((cseg)->c_busy_for_thread = NULL), TRUE)); \
230 if ((cseg)->c_wanted) { \
231 (cseg)->c_wanted = 0; \
232 thread_wakeup((event_t) (cseg)); \
233 } \
234 MACRO_END
235
236 #define C_SEG_BUSY(cseg) \
237 MACRO_BEGIN \
238 assert((cseg)->c_busy == 0); \
239 (cseg)->c_busy = 1; \
240 assert((cseg)->c_busy_for_thread == NULL); \
241 assert((((cseg)->c_busy_for_thread = current_thread()), TRUE)); \
242 MACRO_END
243
244
245 extern vm_map_t compressor_map;
246
247 #if DEVELOPMENT || DEBUG
248 extern boolean_t write_protect_c_segs;
249 extern int vm_compressor_test_seg_wp;
250
251 #define C_SEG_MAKE_WRITEABLE(cseg) \
252 MACRO_BEGIN \
253 if (write_protect_c_segs) { \
254 vm_map_protect(compressor_map, \
255 (vm_map_offset_t)cseg->c_store.c_buffer, \
256 (vm_map_offset_t)&cseg->c_store.c_buffer[C_SEG_BYTES_TO_OFFSET(C_SEG_ALLOCSIZE)],\
257 VM_PROT_READ | VM_PROT_WRITE, \
258 0); \
259 } \
260 MACRO_END
261
262 #define C_SEG_WRITE_PROTECT(cseg) \
263 MACRO_BEGIN \
264 if (write_protect_c_segs) { \
265 vm_map_protect(compressor_map, \
266 (vm_map_offset_t)cseg->c_store.c_buffer, \
267 (vm_map_offset_t)&cseg->c_store.c_buffer[C_SEG_BYTES_TO_OFFSET(C_SEG_ALLOCSIZE)],\
268 VM_PROT_READ, \
269 0); \
270 } \
271 if (vm_compressor_test_seg_wp) { \
272 volatile uint32_t vmtstmp = *(volatile uint32_t *)cseg->c_store.c_buffer; \
273 *(volatile uint32_t *)cseg->c_store.c_buffer = 0xDEADABCD; \
274 (void) vmtstmp; \
275 } \
276 MACRO_END
277 #endif
278
279 typedef struct c_segment *c_segment_t;
280 typedef struct c_slot *c_slot_t;
281
282 uint64_t vm_compressor_total_compressions(void);
283 void vm_wake_compactor_swapper(void);
284 void vm_run_compactor(void);
285 void vm_thrashing_jetsam_done(void);
286 void vm_consider_waking_compactor_swapper(void);
287 void vm_consider_swapping(void);
288 void vm_compressor_flush(void);
289 void c_seg_free(c_segment_t);
290 void c_seg_free_locked(c_segment_t);
291 void c_seg_insert_into_age_q(c_segment_t);
292 void c_seg_need_delayed_compaction(c_segment_t, boolean_t);
293
294 void vm_decompressor_lock(void);
295 void vm_decompressor_unlock(void);
296
297 void vm_compressor_delay_trim(void);
298 void vm_compressor_do_warmup(void);
299 void vm_compressor_record_warmup_start(void);
300 void vm_compressor_record_warmup_end(void);
301
302 int vm_wants_task_throttled(task_t);
303
304 extern void vm_compaction_swapper_do_init(void);
305 extern void vm_compressor_swap_init(void);
306 extern void vm_compressor_init_locks(void);
307 extern lck_rw_t c_master_lock;
308
309 #if ENCRYPTED_SWAP
310 extern void vm_swap_decrypt(c_segment_t);
311 #endif /* ENCRYPTED_SWAP */
312
313 extern int vm_swap_low_on_space(void);
314 extern kern_return_t vm_swap_get(c_segment_t, uint64_t, uint64_t);
315 extern void vm_swap_free(uint64_t);
316 extern void vm_swap_consider_defragmenting(void);
317
318 extern void c_seg_swapin_requeue(c_segment_t, boolean_t, boolean_t, boolean_t);
319 extern int c_seg_swapin(c_segment_t, boolean_t, boolean_t);
320 extern void c_seg_wait_on_busy(c_segment_t);
321 extern void c_seg_trim_tail(c_segment_t);
322 extern void c_seg_switch_state(c_segment_t, int, boolean_t);
323
324 extern boolean_t fastwake_recording_in_progress;
325 extern int compaction_swapper_inited;
326 extern int compaction_swapper_running;
327 extern uint64_t vm_swap_put_failures;
328
329 extern int c_overage_swapped_count;
330 extern int c_overage_swapped_limit;
331
332 extern queue_head_t c_minor_list_head;
333 extern queue_head_t c_age_list_head;
334 extern queue_head_t c_swapout_list_head;
335 extern queue_head_t c_swappedout_list_head;
336 extern queue_head_t c_swappedout_sparse_list_head;
337
338 extern uint32_t c_age_count;
339 extern uint32_t c_swapout_count;
340 extern uint32_t c_swappedout_count;
341 extern uint32_t c_swappedout_sparse_count;
342
343 extern int64_t compressor_bytes_used;
344 extern uint64_t first_c_segment_to_warm_generation_id;
345 extern uint64_t last_c_segment_to_warm_generation_id;
346 extern boolean_t hibernate_flushing;
347 extern boolean_t hibernate_no_swapspace;
348 extern boolean_t hibernate_in_progress_with_pinned_swap;
349 extern uint32_t swapout_target_age;
350
351 extern void c_seg_insert_into_q(queue_head_t *, c_segment_t);
352
353 extern uint32_t vm_compressor_minorcompact_threshold_divisor;
354 extern uint32_t vm_compressor_majorcompact_threshold_divisor;
355 extern uint32_t vm_compressor_unthrottle_threshold_divisor;
356 extern uint32_t vm_compressor_catchup_threshold_divisor;
357 extern uint64_t vm_compressor_compute_elapsed_msecs(clock_sec_t, clock_nsec_t, clock_sec_t, clock_nsec_t);
358
359 #define PAGE_REPLACEMENT_DISALLOWED(enable) (enable == TRUE ? lck_rw_lock_shared(&c_master_lock) : lck_rw_done(&c_master_lock))
360 #define PAGE_REPLACEMENT_ALLOWED(enable) (enable == TRUE ? lck_rw_lock_exclusive(&c_master_lock) : lck_rw_done(&c_master_lock))
361
362
363 #define AVAILABLE_NON_COMPRESSED_MEMORY (vm_page_active_count + vm_page_inactive_count + vm_page_free_count + vm_page_speculative_count)
364 #define AVAILABLE_MEMORY (AVAILABLE_NON_COMPRESSED_MEMORY + VM_PAGE_COMPRESSOR_COUNT)
365 /* TODO, there may be a minor optimisation opportunity to replace these divisions
366 * with multiplies and shifts
367 */
368
369 #define VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD (((AVAILABLE_MEMORY) * 10) / (vm_compressor_minorcompact_threshold_divisor ? vm_compressor_minorcompact_threshold_divisor : 1))
370 #define VM_PAGE_COMPRESSOR_SWAP_THRESHOLD (((AVAILABLE_MEMORY) * 10) / (vm_compressor_majorcompact_threshold_divisor ? vm_compressor_majorcompact_threshold_divisor : 1))
371 #define VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD (((AVAILABLE_MEMORY) * 10) / (vm_compressor_unthrottle_threshold_divisor ? vm_compressor_unthrottle_threshold_divisor : 1))
372 #define VM_PAGE_COMPRESSOR_SWAP_CATCHUP_THRESHOLD (((AVAILABLE_MEMORY) * 10) / (vm_compressor_catchup_threshold_divisor ? vm_compressor_catchup_threshold_divisor : 1))
373
374 #ifdef CONFIG_EMBEDDED
375 #define AVAILABLE_NON_COMPRESSED_MIN 20000
376 #define COMPRESSOR_NEEDS_TO_SWAP() (((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_THRESHOLD) || \
377 (AVAILABLE_NON_COMPRESSED_MEMORY < AVAILABLE_NON_COMPRESSED_MIN)) ? 1 : 0)
378 #else
379 #define COMPRESSOR_NEEDS_TO_SWAP() ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_THRESHOLD) ? 1 : 0)
380 #endif
381
382 #define VM_PAGEOUT_SCAN_NEEDS_TO_THROTTLE() \
383 (vm_compressor_mode == VM_PAGER_COMPRESSOR_WITH_SWAP && \
384 ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_CATCHUP_THRESHOLD) ? 1 : 0))
385 #define HARD_THROTTLE_LIMIT_REACHED() ((AVAILABLE_NON_COMPRESSED_MEMORY < (VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) / 2) ? 1 : 0)
386 #define SWAPPER_NEEDS_TO_UNTHROTTLE() ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) ? 1 : 0)
387 #define COMPRESSOR_NEEDS_TO_MINOR_COMPACT() ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD) ? 1 : 0)
388
389
390 #ifdef CONFIG_EMBEDDED
391 #define COMPRESSOR_FREE_RESERVED_LIMIT 28
392 #else
393 #define COMPRESSOR_FREE_RESERVED_LIMIT 128
394 #endif
395
396 uint32_t vm_compressor_get_encode_scratch_size(void);
397 uint32_t vm_compressor_get_decode_scratch_size(void);
398
399 #define COMPRESSOR_SCRATCH_BUF_SIZE vm_compressor_get_encode_scratch_size()
400
401 #if RECORD_THE_COMPRESSED_DATA
402 extern void c_compressed_record_init(void);
403 extern void c_compressed_record_write(char *, int);
404 #endif
405
406 extern lck_mtx_t *c_list_lock;
407
408 #if DEVELOPMENT || DEBUG
409 extern uint32_t vm_ktrace_enabled;
410
411 #define VMKDBG(x, ...) \
412 MACRO_BEGIN \
413 if (vm_ktrace_enabled) { \
414 KDBG(x, ## __VA_ARGS__);\
415 } \
416 MACRO_END
417 #endif