]> git.saurik.com Git - apple/xnu.git/blob - osfmk/vm/vm_compressor.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-3789.60.24.tar.gz
[apple/xnu.git] / osfmk / vm / vm_compressor.c
1 /*
2 * Copyright (c) 2000-2013 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <vm/vm_compressor.h>
30
31 #if CONFIG_PHANTOM_CACHE
32 #include <vm/vm_phantom_cache.h>
33 #endif
34
35 #include <vm/vm_map.h>
36 #include <vm/vm_pageout.h>
37 #include <vm/memory_object.h>
38 #include <vm/vm_compressor_algorithms.h>
39 #include <vm/vm_fault.h>
40 #include <mach/mach_host.h> /* for host_info() */
41 #include <kern/ledger.h>
42 #include <kern/policy_internal.h>
43
44 #include <i386/misc_protos.h>
45
46 #include <IOKit/IOHibernatePrivate.h>
47
48 /*
49 * vm_compressor_mode has a heirarchy of control to set its value.
50 * boot-args are checked first, then device-tree, and finally
51 * the default value that is defined below. See vm_fault_init() for
52 * the boot-arg & device-tree code.
53 */
54
55 int vm_compressor_mode = VM_PAGER_COMPRESSOR_WITH_SWAP;
56 int vm_scale = 16;
57
58
59 int vm_compressor_is_active = 0;
60 int vm_compression_limit = 0;
61 int vm_compressor_available = 0;
62
63 extern void vm_pageout_io_throttle(void);
64
65 #if CHECKSUM_THE_DATA || CHECKSUM_THE_SWAP || CHECKSUM_THE_COMPRESSED_DATA
66 extern unsigned int hash_string(char *cp, int len);
67 #endif
68
69 #define UNPACK_C_SIZE(cs) ((cs->c_size == (PAGE_SIZE-1)) ? PAGE_SIZE : cs->c_size)
70 #define PACK_C_SIZE(cs, size) (cs->c_size = ((size == PAGE_SIZE) ? PAGE_SIZE - 1 : size))
71
72
73 struct c_sv_hash_entry {
74 union {
75 struct {
76 uint32_t c_sv_he_ref;
77 uint32_t c_sv_he_data;
78 } c_sv_he;
79 uint64_t c_sv_he_record;
80
81 } c_sv_he_un;
82 };
83
84 #define he_ref c_sv_he_un.c_sv_he.c_sv_he_ref
85 #define he_data c_sv_he_un.c_sv_he.c_sv_he_data
86 #define he_record c_sv_he_un.c_sv_he_record
87
88 #define C_SV_HASH_MAX_MISS 32
89 #define C_SV_HASH_SIZE ((1 << 10))
90 #define C_SV_HASH_MASK ((1 << 10) - 1)
91 #define C_SV_CSEG_ID ((1 << 22) - 1)
92
93
94 struct c_slot_mapping {
95 uint32_t s_cseg:22, /* segment number + 1 */
96 s_cindx:10; /* index in the segment */
97 };
98 #define C_SLOT_MAX_INDEX (1 << 10)
99
100 typedef struct c_slot_mapping *c_slot_mapping_t;
101
102
103 union c_segu {
104 c_segment_t c_seg;
105 uintptr_t c_segno;
106 };
107
108
109
110 #define C_SLOT_PACK_PTR(ptr) (((uintptr_t)ptr - (uintptr_t) KERNEL_PMAP_HEAP_RANGE_START) >> 2)
111 #define C_SLOT_UNPACK_PTR(cslot) ((uintptr_t)(cslot->c_packed_ptr << 2) + (uintptr_t) KERNEL_PMAP_HEAP_RANGE_START)
112
113
114 uint32_t c_segment_count = 0;
115 uint32_t c_segment_count_max = 0;
116
117 uint64_t c_generation_id = 0;
118 uint64_t c_generation_id_flush_barrier;
119
120
121 #define HIBERNATE_FLUSHING_SECS_TO_COMPLETE 120
122
123 boolean_t hibernate_no_swapspace = FALSE;
124 clock_sec_t hibernate_flushing_deadline = 0;
125
126
127 #if RECORD_THE_COMPRESSED_DATA
128 char *c_compressed_record_sbuf;
129 char *c_compressed_record_ebuf;
130 char *c_compressed_record_cptr;
131 #endif
132
133
134 queue_head_t c_age_list_head;
135 queue_head_t c_swapout_list_head;
136 queue_head_t c_swappedin_list_head;
137 queue_head_t c_swappedout_list_head;
138 queue_head_t c_swappedout_sparse_list_head;
139 queue_head_t c_major_list_head;
140 queue_head_t c_filling_list_head;
141 queue_head_t c_bad_list_head;
142
143 uint32_t c_age_count = 0;
144 uint32_t c_swapout_count = 0;
145 uint32_t c_swappedin_count = 0;
146 uint32_t c_swappedout_count = 0;
147 uint32_t c_swappedout_sparse_count = 0;
148 uint32_t c_major_count = 0;
149 uint32_t c_filling_count = 0;
150 uint32_t c_empty_count = 0;
151 uint32_t c_bad_count = 0;
152
153
154 queue_head_t c_minor_list_head;
155 uint32_t c_minor_count = 0;
156
157 int c_overage_swapped_count = 0;
158 int c_overage_swapped_limit = 0;
159
160 int c_seg_fixed_array_len;
161 union c_segu *c_segments;
162 vm_offset_t c_buffers;
163 vm_size_t c_buffers_size;
164 caddr_t c_segments_next_page;
165 boolean_t c_segments_busy;
166 uint32_t c_segments_available;
167 uint32_t c_segments_limit;
168 uint32_t c_segments_nearing_limit;
169
170 uint32_t c_segment_svp_in_hash;
171 uint32_t c_segment_svp_hash_succeeded;
172 uint32_t c_segment_svp_hash_failed;
173 uint32_t c_segment_svp_zero_compressions;
174 uint32_t c_segment_svp_nonzero_compressions;
175 uint32_t c_segment_svp_zero_decompressions;
176 uint32_t c_segment_svp_nonzero_decompressions;
177
178 uint32_t c_segment_noncompressible_pages;
179
180 uint32_t c_segment_pages_compressed;
181 uint32_t c_segment_pages_compressed_limit;
182 uint32_t c_segment_pages_compressed_nearing_limit;
183 uint32_t c_free_segno_head = (uint32_t)-1;
184
185 uint32_t vm_compressor_minorcompact_threshold_divisor = 10;
186 uint32_t vm_compressor_majorcompact_threshold_divisor = 10;
187 uint32_t vm_compressor_unthrottle_threshold_divisor = 10;
188 uint32_t vm_compressor_catchup_threshold_divisor = 10;
189
190 #define C_SEGMENTS_PER_PAGE (PAGE_SIZE / sizeof(union c_segu))
191
192
193 lck_grp_attr_t vm_compressor_lck_grp_attr;
194 lck_attr_t vm_compressor_lck_attr;
195 lck_grp_t vm_compressor_lck_grp;
196 lck_mtx_t *c_list_lock;
197 lck_rw_t c_master_lock;
198 boolean_t decompressions_blocked = FALSE;
199
200 zone_t compressor_segment_zone;
201 int c_compressor_swap_trigger = 0;
202
203 uint32_t compressor_cpus;
204 char *compressor_scratch_bufs;
205 char *kdp_compressor_scratch_buf;
206 char *kdp_compressor_decompressed_page;
207 addr64_t kdp_compressor_decompressed_page_paddr;
208 ppnum_t kdp_compressor_decompressed_page_ppnum;
209
210 clock_sec_t start_of_sample_period_sec = 0;
211 clock_nsec_t start_of_sample_period_nsec = 0;
212 clock_sec_t start_of_eval_period_sec = 0;
213 clock_nsec_t start_of_eval_period_nsec = 0;
214 uint32_t sample_period_decompression_count = 0;
215 uint32_t sample_period_compression_count = 0;
216 uint32_t last_eval_decompression_count = 0;
217 uint32_t last_eval_compression_count = 0;
218
219 #define DECOMPRESSION_SAMPLE_MAX_AGE (60 * 30)
220
221 boolean_t vm_swapout_ripe_segments = FALSE;
222 uint32_t vm_ripe_target_age = (60 * 60 * 48);
223
224 uint32_t swapout_target_age = 0;
225 uint32_t age_of_decompressions_during_sample_period[DECOMPRESSION_SAMPLE_MAX_AGE];
226 uint32_t overage_decompressions_during_sample_period = 0;
227
228 void do_fastwake_warmup(queue_head_t *, boolean_t);
229 boolean_t fastwake_warmup = FALSE;
230 boolean_t fastwake_recording_in_progress = FALSE;
231 clock_sec_t dont_trim_until_ts = 0;
232
233 uint64_t c_segment_warmup_count;
234 uint64_t first_c_segment_to_warm_generation_id = 0;
235 uint64_t last_c_segment_to_warm_generation_id = 0;
236 boolean_t hibernate_flushing = FALSE;
237
238 int64_t c_segment_input_bytes __attribute__((aligned(8))) = 0;
239 int64_t c_segment_compressed_bytes __attribute__((aligned(8))) = 0;
240 int64_t compressor_bytes_used __attribute__((aligned(8))) = 0;
241
242
243 struct c_sv_hash_entry c_segment_sv_hash_table[C_SV_HASH_SIZE] __attribute__ ((aligned (8)));
244
245
246 static boolean_t compressor_needs_to_swap(void);
247 static void vm_compressor_swap_trigger_thread(void);
248 static void vm_compressor_do_delayed_compactions(boolean_t);
249 static void vm_compressor_compact_and_swap(boolean_t);
250 static void vm_compressor_age_swapped_in_segments(boolean_t);
251
252 static void vm_compressor_take_paging_space_action(void);
253
254 boolean_t vm_compressor_low_on_space(void);
255
256 void compute_swapout_target_age(void);
257
258 boolean_t c_seg_major_compact(c_segment_t, c_segment_t);
259 boolean_t c_seg_major_compact_ok(c_segment_t, c_segment_t);
260
261 int c_seg_minor_compaction_and_unlock(c_segment_t, boolean_t);
262 int c_seg_do_minor_compaction_and_unlock(c_segment_t, boolean_t, boolean_t, boolean_t);
263 void c_seg_try_minor_compaction_and_unlock(c_segment_t c_seg);
264
265 void c_seg_move_to_sparse_list(c_segment_t);
266 void c_seg_insert_into_q(queue_head_t *, c_segment_t);
267
268 uint64_t vm_available_memory(void);
269 uint64_t vm_compressor_pages_compressed(void);
270
271 /*
272 * indicate the need to do a major compaction if
273 * the overall set of in-use compression segments
274 * becomes sparse... on systems that support pressure
275 * driven swapping, this will also cause swapouts to
276 * be initiated.
277 */
278 static inline boolean_t vm_compressor_needs_to_major_compact()
279 {
280 uint32_t incore_seg_count;
281
282 incore_seg_count = c_segment_count - c_swappedout_count - c_swappedout_sparse_count;
283
284 if ((c_segment_count >= (c_segments_nearing_limit / 8)) &&
285 ((incore_seg_count * C_SEG_MAX_PAGES) - VM_PAGE_COMPRESSOR_COUNT) >
286 ((incore_seg_count / 8) * C_SEG_MAX_PAGES))
287 return (1);
288 return (0);
289 }
290
291
292 uint64_t
293 vm_available_memory(void)
294 {
295 return (((uint64_t)AVAILABLE_NON_COMPRESSED_MEMORY) * PAGE_SIZE_64);
296 }
297
298
299 uint64_t
300 vm_compressor_pages_compressed(void)
301 {
302 return (c_segment_pages_compressed * PAGE_SIZE_64);
303 }
304
305
306 boolean_t
307 vm_compressor_low_on_space(void)
308 {
309 if ((c_segment_pages_compressed > c_segment_pages_compressed_nearing_limit) ||
310 (c_segment_count > c_segments_nearing_limit))
311 return (TRUE);
312
313 return (FALSE);
314 }
315
316
317 int
318 vm_wants_task_throttled(task_t task)
319 {
320 if (task == kernel_task)
321 return (0);
322
323 if (VM_CONFIG_SWAP_IS_ACTIVE) {
324 if ((vm_compressor_low_on_space() || HARD_THROTTLE_LIMIT_REACHED()) &&
325 (unsigned int)pmap_compressed(task->map->pmap) > (c_segment_pages_compressed / 4))
326 return (1);
327 }
328 return (0);
329 }
330
331
332 #if DEVELOPMENT || DEBUG
333 boolean_t kill_on_no_paging_space = FALSE; /* On compressor/swap exhaustion, kill the largest process regardless of
334 * its chosen process policy. Controlled by a boot-arg of the same name. */
335 #endif /* DEVELOPMENT || DEBUG */
336
337
338 static uint32_t no_paging_space_action_in_progress = 0;
339 extern void memorystatus_send_low_swap_note(void);
340
341 static void
342 vm_compressor_take_paging_space_action(void)
343 {
344 if (no_paging_space_action_in_progress == 0) {
345
346 if (OSCompareAndSwap(0, 1, (UInt32 *)&no_paging_space_action_in_progress)) {
347
348 if (no_paging_space_action()) {
349 #if DEVELOPMENT || DEBUG
350 if (kill_on_no_paging_space == TRUE) {
351 /*
352 * Since we are choosing to always kill a process, we don't need the
353 * "out of application memory" dialog box in this mode. And, hence we won't
354 * send the knote.
355 */
356 no_paging_space_action_in_progress = 0;
357 return;
358 }
359 #endif /* DEVELOPMENT || DEBUG */
360 memorystatus_send_low_swap_note();
361 }
362
363 no_paging_space_action_in_progress = 0;
364 }
365 }
366 }
367
368
369 void
370 vm_compressor_init_locks(void)
371 {
372 lck_grp_attr_setdefault(&vm_compressor_lck_grp_attr);
373 lck_grp_init(&vm_compressor_lck_grp, "vm_compressor", &vm_compressor_lck_grp_attr);
374 lck_attr_setdefault(&vm_compressor_lck_attr);
375
376 lck_rw_init(&c_master_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
377 }
378
379
380 void
381 vm_decompressor_lock(void)
382 {
383 PAGE_REPLACEMENT_ALLOWED(TRUE);
384
385 decompressions_blocked = TRUE;
386
387 PAGE_REPLACEMENT_ALLOWED(FALSE);
388 }
389
390 void
391 vm_decompressor_unlock(void)
392 {
393 PAGE_REPLACEMENT_ALLOWED(TRUE);
394
395 decompressions_blocked = FALSE;
396
397 PAGE_REPLACEMENT_ALLOWED(FALSE);
398
399 thread_wakeup((event_t)&decompressions_blocked);
400 }
401
402 static inline void cslot_copy(c_slot_t cdst, c_slot_t csrc) {
403 #if CHECKSUM_THE_DATA
404 cdst->c_hash_data = csrc->c_hash_data;
405 #endif
406 #if CHECKSUM_THE_COMPRESSED_DATA
407 cdst->c_hash_compressed_data = csrc->c_hash_compressed_data;
408 #endif
409 cdst->c_size = csrc->c_size;
410 cdst->c_packed_ptr = csrc->c_packed_ptr;
411 }
412
413 vm_map_t compressor_map;
414
415 void
416 vm_compressor_init(void)
417 {
418 thread_t thread;
419 struct c_slot cs_dummy;
420 c_slot_t cs = &cs_dummy;
421 int c_segment_min_size;
422 int c_segment_padded_size;
423 kern_return_t retval = KERN_SUCCESS;
424 vm_offset_t start_addr = 0;
425 vm_size_t c_segments_arr_size = 0, compressor_submap_size = 0;
426 #if RECORD_THE_COMPRESSED_DATA
427 vm_size_t c_compressed_record_sbuf_size = 0;
428 #endif /* RECORD_THE_COMPRESSED_DATA */
429
430 #if DEVELOPMENT || DEBUG
431 char bootarg_name[32];
432 if (PE_parse_boot_argn("-kill_on_no_paging_space", bootarg_name, sizeof (bootarg_name))) {
433 kill_on_no_paging_space = TRUE;
434 }
435 #endif /* DEVELOPMENT || DEBUG */
436
437 /*
438 * ensure that any pointer that gets created from
439 * the vm_page zone can be packed properly
440 */
441 cs->c_packed_ptr = C_SLOT_PACK_PTR(zone_map_min_address);
442
443 if (C_SLOT_UNPACK_PTR(cs) != (uintptr_t)zone_map_min_address)
444 panic("C_SLOT_UNPACK_PTR failed on zone_map_min_address - %p", (void *)zone_map_min_address);
445
446 cs->c_packed_ptr = C_SLOT_PACK_PTR(zone_map_max_address);
447
448 if (C_SLOT_UNPACK_PTR(cs) != (uintptr_t)zone_map_max_address)
449 panic("C_SLOT_UNPACK_PTR failed on zone_map_max_address - %p", (void *)zone_map_max_address);
450
451
452 assert((C_SEGMENTS_PER_PAGE * sizeof(union c_segu)) == PAGE_SIZE);
453
454 PE_parse_boot_argn("vm_compression_limit", &vm_compression_limit, sizeof (vm_compression_limit));
455
456 if (max_mem <= (3ULL * 1024ULL * 1024ULL * 1024ULL)) {
457 vm_compressor_minorcompact_threshold_divisor = 11;
458 vm_compressor_majorcompact_threshold_divisor = 13;
459 vm_compressor_unthrottle_threshold_divisor = 20;
460 vm_compressor_catchup_threshold_divisor = 35;
461 } else {
462 vm_compressor_minorcompact_threshold_divisor = 20;
463 vm_compressor_majorcompact_threshold_divisor = 25;
464 vm_compressor_unthrottle_threshold_divisor = 35;
465 vm_compressor_catchup_threshold_divisor = 50;
466 }
467 /*
468 * vm_page_init_lck_grp is now responsible for calling vm_compressor_init_locks
469 * c_master_lock needs to be available early so that "vm_page_find_contiguous" can
470 * use PAGE_REPLACEMENT_ALLOWED to coordinate with the compressor.
471 */
472
473 c_list_lock = lck_mtx_alloc_init(&vm_compressor_lck_grp, &vm_compressor_lck_attr);
474
475 queue_init(&c_bad_list_head);
476 queue_init(&c_age_list_head);
477 queue_init(&c_minor_list_head);
478 queue_init(&c_major_list_head);
479 queue_init(&c_filling_list_head);
480 queue_init(&c_swapout_list_head);
481 queue_init(&c_swappedin_list_head);
482 queue_init(&c_swappedout_list_head);
483 queue_init(&c_swappedout_sparse_list_head);
484
485 c_segment_min_size = sizeof(struct c_segment) + (C_SEG_SLOT_VAR_ARRAY_MIN_LEN * sizeof(struct c_slot));
486
487 for (c_segment_padded_size = 128; c_segment_padded_size < c_segment_min_size; c_segment_padded_size = c_segment_padded_size << 1);
488
489 compressor_segment_zone = zinit(c_segment_padded_size, 128000 * c_segment_padded_size, PAGE_SIZE, "compressor_segment");
490 zone_change(compressor_segment_zone, Z_CALLERACCT, FALSE);
491 zone_change(compressor_segment_zone, Z_NOENCRYPT, TRUE);
492
493 c_seg_fixed_array_len = (c_segment_padded_size - sizeof(struct c_segment)) / sizeof(struct c_slot);
494
495 c_free_segno_head = -1;
496 c_segments_available = 0;
497
498 if (vm_compression_limit == 0) {
499 c_segment_pages_compressed_limit = (uint32_t)((max_mem / PAGE_SIZE)) * vm_scale;
500
501 #define OLD_SWAP_LIMIT (1024 * 1024 * 16)
502 #define MAX_SWAP_LIMIT (1024 * 1024 * 128)
503
504 if (c_segment_pages_compressed_limit > (OLD_SWAP_LIMIT))
505 c_segment_pages_compressed_limit = OLD_SWAP_LIMIT;
506
507 if (c_segment_pages_compressed_limit < (uint32_t)(max_mem / PAGE_SIZE_64))
508 c_segment_pages_compressed_limit = (uint32_t)(max_mem / PAGE_SIZE_64);
509 } else {
510 if (vm_compression_limit < MAX_SWAP_LIMIT)
511 c_segment_pages_compressed_limit = vm_compression_limit;
512 else
513 c_segment_pages_compressed_limit = MAX_SWAP_LIMIT;
514 }
515 if ((c_segments_limit = c_segment_pages_compressed_limit / (C_SEG_BUFSIZE / PAGE_SIZE)) > C_SEG_MAX_LIMIT)
516 c_segments_limit = C_SEG_MAX_LIMIT;
517
518 c_segment_pages_compressed_nearing_limit = (c_segment_pages_compressed_limit * 98) / 100;
519 c_segments_nearing_limit = (c_segments_limit * 98) / 100;
520
521 c_segments_busy = FALSE;
522
523 /*
524 * Submap needs space for:
525 * - c_segments
526 * - c_buffers
527 * - swap reclaimations -- C_SEG_BUFSIZE
528 */
529 c_segments_arr_size = vm_map_round_page((sizeof(union c_segu) * c_segments_limit),VM_MAP_PAGE_MASK(kernel_map));
530 c_buffers_size = vm_map_round_page(((vm_size_t)C_SEG_ALLOCSIZE * (vm_size_t)c_segments_limit), VM_MAP_PAGE_MASK(kernel_map));
531
532 compressor_submap_size = c_segments_arr_size + c_buffers_size + C_SEG_BUFSIZE;
533
534 #if RECORD_THE_COMPRESSED_DATA
535 c_compressed_record_sbuf_size = (vm_size_t)C_SEG_ALLOCSIZE + (PAGE_SIZE * 2);
536 compressor_submap_size += c_compressed_record_sbuf_size;
537 #endif /* RECORD_THE_COMPRESSED_DATA */
538
539 retval = kmem_suballoc(kernel_map, &start_addr, compressor_submap_size,
540 FALSE, VM_FLAGS_ANYWHERE | VM_FLAGS_PERMANENT | VM_MAKE_TAG(0),
541 &compressor_map);
542
543 if (retval != KERN_SUCCESS)
544 panic("vm_compressor_init: kmem_suballoc failed");
545
546 if (kernel_memory_allocate(compressor_map, (vm_offset_t *)(&c_segments), (sizeof(union c_segu) * c_segments_limit), 0, KMA_KOBJECT | KMA_VAONLY | KMA_PERMANENT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS)
547 panic("vm_compressor_init: kernel_memory_allocate failed - c_segments\n");
548 if (kernel_memory_allocate(compressor_map, &c_buffers, c_buffers_size, 0, KMA_COMPRESSOR | KMA_VAONLY | KMA_PERMANENT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS)
549 panic("vm_compressor_init: kernel_memory_allocate failed - c_buffers\n");
550
551 c_segments_next_page = (caddr_t)c_segments;
552 vm_compressor_algorithm_init();
553
554 {
555 host_basic_info_data_t hinfo;
556 mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
557
558 #define BSD_HOST 1
559 host_info((host_t)BSD_HOST, HOST_BASIC_INFO, (host_info_t)&hinfo, &count);
560
561 compressor_cpus = hinfo.max_cpus;
562 compressor_scratch_bufs = kalloc_tag(compressor_cpus * vm_compressor_get_decode_scratch_size(), VM_KERN_MEMORY_COMPRESSOR);
563
564 kdp_compressor_scratch_buf = kalloc_tag(vm_compressor_get_decode_scratch_size(), VM_KERN_MEMORY_COMPRESSOR);
565 kdp_compressor_decompressed_page = kalloc_tag(PAGE_SIZE, VM_KERN_MEMORY_COMPRESSOR);
566 kdp_compressor_decompressed_page_paddr = kvtophys((vm_offset_t)kdp_compressor_decompressed_page);
567 kdp_compressor_decompressed_page_ppnum = (ppnum_t) atop(kdp_compressor_decompressed_page_paddr);
568 }
569 #if CONFIG_FREEZE
570 freezer_compressor_scratch_buf = kalloc_tag(vm_compressor_get_encode_scratch_size(), VM_KERN_MEMORY_COMPRESSOR);
571 #endif
572
573 #if RECORD_THE_COMPRESSED_DATA
574 if (kernel_memory_allocate(compressor_map, (vm_offset_t *)&c_compressed_record_sbuf, c_compressed_record_sbuf_size, 0, KMA_KOBJECT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS)
575 panic("vm_compressor_init: kernel_memory_allocate failed - c_compressed_record_sbuf\n");
576
577 c_compressed_record_cptr = c_compressed_record_sbuf;
578 c_compressed_record_ebuf = c_compressed_record_sbuf + c_compressed_record_sbuf_size;
579 #endif
580
581 if (kernel_thread_start_priority((thread_continue_t)vm_compressor_swap_trigger_thread, NULL,
582 BASEPRI_PREEMPT - 1, &thread) != KERN_SUCCESS) {
583 panic("vm_compressor_swap_trigger_thread: create failed");
584 }
585 thread_deallocate(thread);
586
587 if (vm_pageout_internal_start() != KERN_SUCCESS) {
588 panic("vm_compressor_init: Failed to start the internal pageout thread.\n");
589 }
590 if (VM_CONFIG_SWAP_IS_PRESENT)
591 vm_compressor_swap_init();
592
593 if (VM_CONFIG_COMPRESSOR_IS_ACTIVE)
594 vm_compressor_is_active = 1;
595
596 #if CONFIG_FREEZE
597 memorystatus_freeze_enabled = TRUE;
598 #endif /* CONFIG_FREEZE */
599
600 vm_compressor_available = 1;
601
602 vm_page_reactivate_all_throttled();
603 }
604
605
606 #if VALIDATE_C_SEGMENTS
607
608 static void
609 c_seg_validate(c_segment_t c_seg, boolean_t must_be_compact)
610 {
611 int c_indx;
612 int32_t bytes_used;
613 int32_t bytes_unused;
614 uint32_t c_rounded_size;
615 uint32_t c_size;
616 c_slot_t cs;
617
618 if (c_seg->c_firstemptyslot < c_seg->c_nextslot) {
619 c_indx = c_seg->c_firstemptyslot;
620 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
621
622 if (cs == NULL)
623 panic("c_seg_validate: no slot backing c_firstemptyslot");
624
625 if (cs->c_size)
626 panic("c_seg_validate: c_firstemptyslot has non-zero size (%d)\n", cs->c_size);
627 }
628 bytes_used = 0;
629 bytes_unused = 0;
630
631 for (c_indx = 0; c_indx < c_seg->c_nextslot; c_indx++) {
632
633 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
634
635 c_size = UNPACK_C_SIZE(cs);
636
637 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
638
639 bytes_used += c_rounded_size;
640
641 #if CHECKSUM_THE_COMPRESSED_DATA
642 if (c_size && cs->c_hash_compressed_data != hash_string((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size))
643 panic("compressed data doesn't match original");
644 #endif
645 }
646
647 if (bytes_used != c_seg->c_bytes_used)
648 panic("c_seg_validate: bytes_used mismatch - found %d, segment has %d\n", bytes_used, c_seg->c_bytes_used);
649
650 if (c_seg->c_bytes_used > C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset))
651 panic("c_seg_validate: c_bytes_used > c_nextoffset - c_nextoffset = %d, c_bytes_used = %d\n",
652 (int32_t)C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset), c_seg->c_bytes_used);
653
654 if (must_be_compact) {
655 if (c_seg->c_bytes_used != C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset))
656 panic("c_seg_validate: c_bytes_used doesn't match c_nextoffset - c_nextoffset = %d, c_bytes_used = %d\n",
657 (int32_t)C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset), c_seg->c_bytes_used);
658 }
659 }
660
661 #endif
662
663
664 void
665 c_seg_need_delayed_compaction(c_segment_t c_seg, boolean_t c_list_lock_held)
666 {
667 boolean_t clear_busy = FALSE;
668
669 if (c_list_lock_held == FALSE) {
670 if ( !lck_mtx_try_lock_spin_always(c_list_lock)) {
671 C_SEG_BUSY(c_seg);
672
673 lck_mtx_unlock_always(&c_seg->c_lock);
674 lck_mtx_lock_spin_always(c_list_lock);
675 lck_mtx_lock_spin_always(&c_seg->c_lock);
676
677 clear_busy = TRUE;
678 }
679 }
680 assert(c_seg->c_state != C_IS_FILLING);
681
682 if (!c_seg->c_on_minorcompact_q && !(C_SEG_IS_ONDISK(c_seg))) {
683 queue_enter(&c_minor_list_head, c_seg, c_segment_t, c_list);
684 c_seg->c_on_minorcompact_q = 1;
685 c_minor_count++;
686 }
687 if (c_list_lock_held == FALSE)
688 lck_mtx_unlock_always(c_list_lock);
689
690 if (clear_busy == TRUE)
691 C_SEG_WAKEUP_DONE(c_seg);
692 }
693
694
695 unsigned int c_seg_moved_to_sparse_list = 0;
696
697 void
698 c_seg_move_to_sparse_list(c_segment_t c_seg)
699 {
700 boolean_t clear_busy = FALSE;
701
702 if ( !lck_mtx_try_lock_spin_always(c_list_lock)) {
703 C_SEG_BUSY(c_seg);
704
705 lck_mtx_unlock_always(&c_seg->c_lock);
706 lck_mtx_lock_spin_always(c_list_lock);
707 lck_mtx_lock_spin_always(&c_seg->c_lock);
708
709 clear_busy = TRUE;
710 }
711 c_seg_switch_state(c_seg, C_ON_SWAPPEDOUTSPARSE_Q, FALSE);
712
713 c_seg_moved_to_sparse_list++;
714
715 lck_mtx_unlock_always(c_list_lock);
716
717 if (clear_busy == TRUE)
718 C_SEG_WAKEUP_DONE(c_seg);
719 }
720
721
722 void
723 c_seg_insert_into_q(queue_head_t *qhead, c_segment_t c_seg)
724 {
725 c_segment_t c_seg_next;
726
727 if (queue_empty(qhead)) {
728 queue_enter(qhead, c_seg, c_segment_t, c_age_list);
729 } else {
730 c_seg_next = (c_segment_t)queue_first(qhead);
731
732 while (TRUE) {
733
734 if (c_seg->c_generation_id < c_seg_next->c_generation_id) {
735 queue_insert_before(qhead, c_seg, c_seg_next, c_segment_t, c_age_list);
736 break;
737 }
738 c_seg_next = (c_segment_t) queue_next(&c_seg_next->c_age_list);
739
740 if (queue_end(qhead, (queue_entry_t) c_seg_next)) {
741 queue_enter(qhead, c_seg, c_segment_t, c_age_list);
742 break;
743 }
744 }
745 }
746 }
747
748
749 int try_minor_compaction_failed = 0;
750 int try_minor_compaction_succeeded = 0;
751
752 void
753 c_seg_try_minor_compaction_and_unlock(c_segment_t c_seg)
754 {
755
756 assert(c_seg->c_on_minorcompact_q);
757 /*
758 * c_seg is currently on the delayed minor compaction
759 * queue and we have c_seg locked... if we can get the
760 * c_list_lock w/o blocking (if we blocked we could deadlock
761 * because the lock order is c_list_lock then c_seg's lock)
762 * we'll pull it from the delayed list and free it directly
763 */
764 if ( !lck_mtx_try_lock_spin_always(c_list_lock)) {
765 /*
766 * c_list_lock is held, we need to bail
767 */
768 try_minor_compaction_failed++;
769
770 lck_mtx_unlock_always(&c_seg->c_lock);
771 } else {
772 try_minor_compaction_succeeded++;
773
774 C_SEG_BUSY(c_seg);
775 c_seg_do_minor_compaction_and_unlock(c_seg, TRUE, FALSE, FALSE);
776 }
777 }
778
779
780 int
781 c_seg_do_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy, boolean_t need_list_lock, boolean_t disallow_page_replacement)
782 {
783 int c_seg_freed;
784
785 assert(c_seg->c_busy);
786
787 /*
788 * check for the case that can occur when we are not swapping
789 * and this segment has been major compacted in the past
790 * and moved to the majorcompact q to remove it from further
791 * consideration... if the occupancy falls too low we need
792 * to put it back on the age_q so that it will be considered
793 * in the next major compaction sweep... if we don't do this
794 * we will eventually run into the c_segments_limit
795 */
796 if (c_seg->c_state == C_ON_MAJORCOMPACT_Q && C_SEG_SHOULD_MAJORCOMPACT(c_seg)) {
797
798 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
799 }
800 if (!c_seg->c_on_minorcompact_q) {
801 if (clear_busy == TRUE)
802 C_SEG_WAKEUP_DONE(c_seg);
803
804 lck_mtx_unlock_always(&c_seg->c_lock);
805
806 return (0);
807 }
808 queue_remove(&c_minor_list_head, c_seg, c_segment_t, c_list);
809 c_seg->c_on_minorcompact_q = 0;
810 c_minor_count--;
811
812 lck_mtx_unlock_always(c_list_lock);
813
814 if (disallow_page_replacement == TRUE) {
815 lck_mtx_unlock_always(&c_seg->c_lock);
816
817 PAGE_REPLACEMENT_DISALLOWED(TRUE);
818
819 lck_mtx_lock_spin_always(&c_seg->c_lock);
820 }
821 c_seg_freed = c_seg_minor_compaction_and_unlock(c_seg, clear_busy);
822
823 if (disallow_page_replacement == TRUE)
824 PAGE_REPLACEMENT_DISALLOWED(FALSE);
825
826 if (need_list_lock == TRUE)
827 lck_mtx_lock_spin_always(c_list_lock);
828
829 return (c_seg_freed);
830 }
831
832
833 void
834 c_seg_wait_on_busy(c_segment_t c_seg)
835 {
836 c_seg->c_wanted = 1;
837 assert_wait((event_t) (c_seg), THREAD_UNINT);
838
839 lck_mtx_unlock_always(&c_seg->c_lock);
840 thread_block(THREAD_CONTINUE_NULL);
841 }
842
843
844 void
845 c_seg_switch_state(c_segment_t c_seg, int new_state, boolean_t insert_head)
846 {
847 int old_state = c_seg->c_state;
848
849 #if __i386__ || __x86_64__
850 if (new_state != C_IS_FILLING)
851 LCK_MTX_ASSERT(&c_seg->c_lock, LCK_MTX_ASSERT_OWNED);
852 LCK_MTX_ASSERT(c_list_lock, LCK_MTX_ASSERT_OWNED);
853 #endif
854 switch (old_state) {
855
856 case C_IS_EMPTY:
857 assert(new_state == C_IS_FILLING || new_state == C_IS_FREE);
858
859 c_empty_count--;
860 break;
861
862 case C_IS_FILLING:
863 assert(new_state == C_ON_AGE_Q || new_state == C_ON_SWAPOUT_Q);
864
865 queue_remove(&c_filling_list_head, c_seg, c_segment_t, c_age_list);
866 c_filling_count--;
867 break;
868
869 case C_ON_AGE_Q:
870 assert(new_state == C_ON_SWAPOUT_Q || new_state == C_ON_MAJORCOMPACT_Q ||
871 new_state == C_IS_FREE);
872
873 queue_remove(&c_age_list_head, c_seg, c_segment_t, c_age_list);
874 c_age_count--;
875 break;
876
877 case C_ON_SWAPPEDIN_Q:
878 assert(new_state == C_ON_AGE_Q || new_state == C_IS_FREE);
879
880 queue_remove(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
881 c_swappedin_count--;
882 break;
883
884 case C_ON_SWAPOUT_Q:
885 assert(new_state == C_ON_SWAPPEDOUT_Q || new_state == C_ON_SWAPPEDOUTSPARSE_Q ||
886 new_state == C_ON_AGE_Q || new_state == C_IS_FREE || new_state == C_IS_EMPTY);
887
888 queue_remove(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
889 thread_wakeup((event_t)&compaction_swapper_running);
890 c_swapout_count--;
891 break;
892
893 case C_ON_SWAPPEDOUT_Q:
894 assert(new_state == C_ON_SWAPPEDIN_Q || new_state == C_ON_AGE_Q ||
895 new_state == C_ON_SWAPPEDOUTSPARSE_Q ||
896 new_state == C_ON_BAD_Q || new_state == C_IS_EMPTY || new_state == C_IS_FREE);
897
898 queue_remove(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
899 c_swappedout_count--;
900 break;
901
902 case C_ON_SWAPPEDOUTSPARSE_Q:
903 assert(new_state == C_ON_SWAPPEDIN_Q || new_state == C_ON_AGE_Q ||
904 new_state == C_ON_BAD_Q || new_state == C_IS_EMPTY || new_state == C_IS_FREE);
905
906 queue_remove(&c_swappedout_sparse_list_head, c_seg, c_segment_t, c_age_list);
907 c_swappedout_sparse_count--;
908 break;
909
910 case C_ON_MAJORCOMPACT_Q:
911 assert(new_state == C_ON_AGE_Q || new_state == C_IS_FREE);
912
913 queue_remove(&c_major_list_head, c_seg, c_segment_t, c_age_list);
914 c_major_count--;
915 break;
916
917 case C_ON_BAD_Q:
918 assert(new_state == C_IS_FREE);
919
920 queue_remove(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
921 c_bad_count--;
922 break;
923
924 default:
925 panic("c_seg %p has bad c_state = %d\n", c_seg, old_state);
926 }
927
928 switch(new_state) {
929 case C_IS_FREE:
930 assert(old_state != C_IS_FILLING);
931
932 break;
933
934 case C_IS_EMPTY:
935 assert(old_state == C_ON_SWAPOUT_Q || old_state == C_ON_SWAPPEDOUT_Q || old_state == C_ON_SWAPPEDOUTSPARSE_Q);
936
937 c_empty_count++;
938 break;
939
940 case C_IS_FILLING:
941 assert(old_state == C_IS_EMPTY);
942
943 queue_enter(&c_filling_list_head, c_seg, c_segment_t, c_age_list);
944 c_filling_count++;
945 break;
946
947 case C_ON_AGE_Q:
948 assert(old_state == C_IS_FILLING || old_state == C_ON_SWAPPEDIN_Q || old_state == C_ON_SWAPOUT_Q ||
949 old_state == C_ON_MAJORCOMPACT_Q || old_state == C_ON_SWAPPEDOUT_Q || old_state == C_ON_SWAPPEDOUTSPARSE_Q);
950
951 if (old_state == C_IS_FILLING)
952 queue_enter(&c_age_list_head, c_seg, c_segment_t, c_age_list);
953 else {
954 if (!queue_empty(&c_age_list_head)) {
955 c_segment_t c_first;
956
957 c_first = (c_segment_t)queue_first(&c_age_list_head);
958 c_seg->c_creation_ts = c_first->c_creation_ts;
959 }
960 queue_enter_first(&c_age_list_head, c_seg, c_segment_t, c_age_list);
961 }
962 c_age_count++;
963 break;
964
965 case C_ON_SWAPPEDIN_Q:
966 assert(c_seg->c_state == C_ON_SWAPPEDOUT_Q || c_seg->c_state == C_ON_SWAPPEDOUTSPARSE_Q);
967
968 if (insert_head == TRUE)
969 queue_enter_first(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
970 else
971 queue_enter(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
972 c_swappedin_count++;
973 break;
974
975 case C_ON_SWAPOUT_Q:
976 assert(old_state == C_ON_AGE_Q || old_state == C_IS_FILLING);
977
978 if (insert_head == TRUE)
979 queue_enter_first(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
980 else
981 queue_enter(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
982 c_swapout_count++;
983 break;
984
985 case C_ON_SWAPPEDOUT_Q:
986 assert(c_seg->c_state == C_ON_SWAPOUT_Q);
987
988 if (insert_head == TRUE)
989 queue_enter_first(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
990 else
991 queue_enter(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
992 c_swappedout_count++;
993 break;
994
995 case C_ON_SWAPPEDOUTSPARSE_Q:
996 assert(c_seg->c_state == C_ON_SWAPOUT_Q || c_seg->c_state == C_ON_SWAPPEDOUT_Q);
997
998 if (insert_head == TRUE)
999 queue_enter_first(&c_swappedout_sparse_list_head, c_seg, c_segment_t, c_age_list);
1000 else
1001 queue_enter(&c_swappedout_sparse_list_head, c_seg, c_segment_t, c_age_list);
1002
1003 c_swappedout_sparse_count++;
1004 break;
1005
1006 case C_ON_MAJORCOMPACT_Q:
1007 assert(c_seg->c_state == C_ON_AGE_Q);
1008
1009 if (insert_head == TRUE)
1010 queue_enter_first(&c_major_list_head, c_seg, c_segment_t, c_age_list);
1011 else
1012 queue_enter(&c_major_list_head, c_seg, c_segment_t, c_age_list);
1013 c_major_count++;
1014 break;
1015
1016 case C_ON_BAD_Q:
1017 assert(c_seg->c_state == C_ON_SWAPPEDOUT_Q || c_seg->c_state == C_ON_SWAPPEDOUTSPARSE_Q);
1018
1019 if (insert_head == TRUE)
1020 queue_enter_first(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
1021 else
1022 queue_enter(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
1023 c_bad_count++;
1024 break;
1025
1026 default:
1027 panic("c_seg %p requesting bad c_state = %d\n", c_seg, new_state);
1028 }
1029 c_seg->c_state = new_state;
1030 }
1031
1032
1033
1034 void
1035 c_seg_free(c_segment_t c_seg)
1036 {
1037 assert(c_seg->c_busy);
1038
1039 lck_mtx_unlock_always(&c_seg->c_lock);
1040 lck_mtx_lock_spin_always(c_list_lock);
1041 lck_mtx_lock_spin_always(&c_seg->c_lock);
1042
1043 c_seg_free_locked(c_seg);
1044 }
1045
1046
1047 void
1048 c_seg_free_locked(c_segment_t c_seg)
1049 {
1050 int segno;
1051 int pages_populated = 0;
1052 int32_t *c_buffer = NULL;
1053 uint64_t c_swap_handle = 0;
1054
1055 assert(c_seg->c_busy);
1056 assert(!c_seg->c_on_minorcompact_q);
1057 assert(!c_seg->c_busy_swapping);
1058
1059 if (c_seg->c_overage_swap == TRUE) {
1060 c_overage_swapped_count--;
1061 c_seg->c_overage_swap = FALSE;
1062 }
1063 if ( !(C_SEG_IS_ONDISK(c_seg)))
1064 c_buffer = c_seg->c_store.c_buffer;
1065 else
1066 c_swap_handle = c_seg->c_store.c_swap_handle;
1067
1068 c_seg_switch_state(c_seg, C_IS_FREE, FALSE);
1069
1070 lck_mtx_unlock_always(c_list_lock);
1071
1072 if (c_buffer) {
1073 pages_populated = (round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE;
1074 c_seg->c_store.c_buffer = NULL;
1075 } else
1076 c_seg->c_store.c_swap_handle = (uint64_t)-1;
1077
1078 lck_mtx_unlock_always(&c_seg->c_lock);
1079
1080 if (c_buffer) {
1081 if (pages_populated)
1082 kernel_memory_depopulate(compressor_map, (vm_offset_t) c_buffer, pages_populated * PAGE_SIZE, KMA_COMPRESSOR);
1083
1084 } else if (c_swap_handle) {
1085 /*
1086 * Free swap space on disk.
1087 */
1088 vm_swap_free(c_swap_handle);
1089 }
1090 lck_mtx_lock_spin_always(&c_seg->c_lock);
1091
1092 C_SEG_WAKEUP_DONE(c_seg);
1093 lck_mtx_unlock_always(&c_seg->c_lock);
1094
1095 segno = c_seg->c_mysegno;
1096
1097 lck_mtx_lock_spin_always(c_list_lock);
1098 /*
1099 * because the c_buffer is now associated with the segno,
1100 * we can't put the segno back on the free list until
1101 * after we have depopulated the c_buffer range, or
1102 * we run the risk of depopulating a range that is
1103 * now being used in one of the compressor heads
1104 */
1105 c_segments[segno].c_segno = c_free_segno_head;
1106 c_free_segno_head = segno;
1107 c_segment_count--;
1108
1109 lck_mtx_unlock_always(c_list_lock);
1110
1111 lck_mtx_destroy(&c_seg->c_lock, &vm_compressor_lck_grp);
1112
1113 if (c_seg->c_slot_var_array_len)
1114 kfree(c_seg->c_slot_var_array, sizeof(struct c_slot) * c_seg->c_slot_var_array_len);
1115
1116 zfree(compressor_segment_zone, c_seg);
1117 }
1118
1119
1120 int c_seg_trim_page_count = 0;
1121
1122 void
1123 c_seg_trim_tail(c_segment_t c_seg)
1124 {
1125 c_slot_t cs;
1126 uint32_t c_size;
1127 uint32_t c_offset;
1128 uint32_t c_rounded_size;
1129 uint16_t current_nextslot;
1130 uint32_t current_populated_offset;
1131
1132 if (c_seg->c_bytes_used == 0)
1133 return;
1134 current_nextslot = c_seg->c_nextslot;
1135 current_populated_offset = c_seg->c_populated_offset;
1136
1137 while (c_seg->c_nextslot) {
1138
1139 cs = C_SEG_SLOT_FROM_INDEX(c_seg, (c_seg->c_nextslot - 1));
1140
1141 c_size = UNPACK_C_SIZE(cs);
1142
1143 if (c_size) {
1144 if (current_nextslot != c_seg->c_nextslot) {
1145 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
1146 c_offset = cs->c_offset + C_SEG_BYTES_TO_OFFSET(c_rounded_size);
1147
1148 c_seg->c_nextoffset = c_offset;
1149 c_seg->c_populated_offset = (c_offset + (C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1)) & ~(C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1);
1150
1151 if (c_seg->c_firstemptyslot > c_seg->c_nextslot)
1152 c_seg->c_firstemptyslot = c_seg->c_nextslot;
1153
1154 c_seg_trim_page_count += ((round_page_32(C_SEG_OFFSET_TO_BYTES(current_populated_offset)) -
1155 round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE);
1156 }
1157 break;
1158 }
1159 c_seg->c_nextslot--;
1160 }
1161 assert(c_seg->c_nextslot);
1162 }
1163
1164
1165 int
1166 c_seg_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy)
1167 {
1168 c_slot_mapping_t slot_ptr;
1169 uint32_t c_offset = 0;
1170 uint32_t old_populated_offset;
1171 uint32_t c_rounded_size;
1172 uint32_t c_size;
1173 int c_indx = 0;
1174 int i;
1175 c_slot_t c_dst;
1176 c_slot_t c_src;
1177
1178 assert(c_seg->c_busy);
1179
1180 #if VALIDATE_C_SEGMENTS
1181 c_seg_validate(c_seg, FALSE);
1182 #endif
1183 if (c_seg->c_bytes_used == 0) {
1184 c_seg_free(c_seg);
1185 return (1);
1186 }
1187 lck_mtx_unlock_always(&c_seg->c_lock);
1188
1189 if (c_seg->c_firstemptyslot >= c_seg->c_nextslot || C_SEG_UNUSED_BYTES(c_seg) < PAGE_SIZE)
1190 goto done;
1191
1192 #if DEVELOPMENT || DEBUG
1193 C_SEG_MAKE_WRITEABLE(c_seg);
1194 #endif
1195
1196 #if VALIDATE_C_SEGMENTS
1197 c_seg->c_was_minor_compacted++;
1198 #endif
1199 c_indx = c_seg->c_firstemptyslot;
1200 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
1201
1202 old_populated_offset = c_seg->c_populated_offset;
1203 c_offset = c_dst->c_offset;
1204
1205 for (i = c_indx + 1; i < c_seg->c_nextslot && c_offset < c_seg->c_nextoffset; i++) {
1206
1207 c_src = C_SEG_SLOT_FROM_INDEX(c_seg, i);
1208
1209 c_size = UNPACK_C_SIZE(c_src);
1210
1211 if (c_size == 0)
1212 continue;
1213
1214 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
1215
1216 memcpy(&c_seg->c_store.c_buffer[c_offset], &c_seg->c_store.c_buffer[c_src->c_offset], c_rounded_size);
1217
1218 cslot_copy(c_dst, c_src);
1219 c_dst->c_offset = c_offset;
1220
1221 slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
1222 slot_ptr->s_cindx = c_indx;
1223
1224 c_offset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
1225 PACK_C_SIZE(c_src, 0);
1226 c_indx++;
1227
1228 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
1229 }
1230 c_seg->c_firstemptyslot = c_indx;
1231 c_seg->c_nextslot = c_indx;
1232 c_seg->c_nextoffset = c_offset;
1233 c_seg->c_populated_offset = (c_offset + (C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1)) & ~(C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1);
1234 c_seg->c_bytes_unused = 0;
1235
1236 #if VALIDATE_C_SEGMENTS
1237 c_seg_validate(c_seg, TRUE);
1238 #endif
1239 if (old_populated_offset > c_seg->c_populated_offset) {
1240 uint32_t gc_size;
1241 int32_t *gc_ptr;
1242
1243 gc_size = C_SEG_OFFSET_TO_BYTES(old_populated_offset - c_seg->c_populated_offset);
1244 gc_ptr = &c_seg->c_store.c_buffer[c_seg->c_populated_offset];
1245
1246 kernel_memory_depopulate(compressor_map, (vm_offset_t)gc_ptr, gc_size, KMA_COMPRESSOR);
1247 }
1248
1249 #if DEVELOPMENT || DEBUG
1250 C_SEG_WRITE_PROTECT(c_seg);
1251 #endif
1252
1253 done:
1254 if (clear_busy == TRUE) {
1255 lck_mtx_lock_spin_always(&c_seg->c_lock);
1256 C_SEG_WAKEUP_DONE(c_seg);
1257 lck_mtx_unlock_always(&c_seg->c_lock);
1258 }
1259 return (0);
1260 }
1261
1262
1263 static void
1264 c_seg_alloc_nextslot(c_segment_t c_seg)
1265 {
1266 struct c_slot *old_slot_array = NULL;
1267 struct c_slot *new_slot_array = NULL;
1268 int newlen;
1269 int oldlen;
1270
1271 if (c_seg->c_nextslot < c_seg_fixed_array_len)
1272 return;
1273
1274 if ((c_seg->c_nextslot - c_seg_fixed_array_len) >= c_seg->c_slot_var_array_len) {
1275
1276 oldlen = c_seg->c_slot_var_array_len;
1277 old_slot_array = c_seg->c_slot_var_array;
1278
1279 if (oldlen == 0)
1280 newlen = C_SEG_SLOT_VAR_ARRAY_MIN_LEN;
1281 else
1282 newlen = oldlen * 2;
1283
1284 new_slot_array = (struct c_slot *)kalloc(sizeof(struct c_slot) * newlen);
1285
1286 lck_mtx_lock_spin_always(&c_seg->c_lock);
1287
1288 if (old_slot_array)
1289 memcpy((char *)new_slot_array, (char *)old_slot_array, sizeof(struct c_slot) * oldlen);
1290
1291 c_seg->c_slot_var_array_len = newlen;
1292 c_seg->c_slot_var_array = new_slot_array;
1293
1294 lck_mtx_unlock_always(&c_seg->c_lock);
1295
1296 if (old_slot_array)
1297 kfree(old_slot_array, sizeof(struct c_slot) * oldlen);
1298 }
1299 }
1300
1301
1302
1303 struct {
1304 uint64_t asked_permission;
1305 uint64_t compactions;
1306 uint64_t moved_slots;
1307 uint64_t moved_bytes;
1308 uint64_t wasted_space_in_swapouts;
1309 uint64_t count_of_swapouts;
1310 uint64_t count_of_freed_segs;
1311 } c_seg_major_compact_stats;
1312
1313
1314 #define C_MAJOR_COMPACTION_SIZE_APPROPRIATE ((C_SEG_BUFSIZE * 90) / 100)
1315
1316
1317 boolean_t
1318 c_seg_major_compact_ok(
1319 c_segment_t c_seg_dst,
1320 c_segment_t c_seg_src)
1321 {
1322
1323 c_seg_major_compact_stats.asked_permission++;
1324
1325 if (c_seg_src->c_bytes_used >= C_MAJOR_COMPACTION_SIZE_APPROPRIATE &&
1326 c_seg_dst->c_bytes_used >= C_MAJOR_COMPACTION_SIZE_APPROPRIATE)
1327 return (FALSE);
1328
1329 if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX_INDEX) {
1330 /*
1331 * destination segment is full... can't compact
1332 */
1333 return (FALSE);
1334 }
1335
1336 return (TRUE);
1337 }
1338
1339
1340 boolean_t
1341 c_seg_major_compact(
1342 c_segment_t c_seg_dst,
1343 c_segment_t c_seg_src)
1344 {
1345 c_slot_mapping_t slot_ptr;
1346 uint32_t c_rounded_size;
1347 uint32_t c_size;
1348 uint16_t dst_slot;
1349 int i;
1350 c_slot_t c_dst;
1351 c_slot_t c_src;
1352 boolean_t keep_compacting = TRUE;
1353
1354 /*
1355 * segments are not locked but they are both marked c_busy
1356 * which keeps c_decompress from working on them...
1357 * we can safely allocate new pages, move compressed data
1358 * from c_seg_src to c_seg_dst and update both c_segment's
1359 * state w/o holding the master lock
1360 */
1361 #if DEVELOPMENT || DEBUG
1362 C_SEG_MAKE_WRITEABLE(c_seg_dst);
1363 #endif
1364
1365 #if VALIDATE_C_SEGMENTS
1366 c_seg_dst->c_was_major_compacted++;
1367 c_seg_src->c_was_major_donor++;
1368 #endif
1369 c_seg_major_compact_stats.compactions++;
1370
1371 dst_slot = c_seg_dst->c_nextslot;
1372
1373 for (i = 0; i < c_seg_src->c_nextslot; i++) {
1374
1375 c_src = C_SEG_SLOT_FROM_INDEX(c_seg_src, i);
1376
1377 c_size = UNPACK_C_SIZE(c_src);
1378
1379 if (c_size == 0) {
1380 /* BATCH: move what we have so far; */
1381 continue;
1382 }
1383
1384 if (C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset - c_seg_dst->c_nextoffset) < (unsigned) c_size) {
1385 int size_to_populate;
1386
1387 /* doesn't fit */
1388 size_to_populate = C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset);
1389
1390 if (size_to_populate == 0) {
1391 /* can't fit */
1392 keep_compacting = FALSE;
1393 break;
1394 }
1395 if (size_to_populate > C_SEG_MAX_POPULATE_SIZE)
1396 size_to_populate = C_SEG_MAX_POPULATE_SIZE;
1397
1398 kernel_memory_populate(compressor_map,
1399 (vm_offset_t) &c_seg_dst->c_store.c_buffer[c_seg_dst->c_populated_offset],
1400 size_to_populate,
1401 KMA_COMPRESSOR,
1402 VM_KERN_MEMORY_COMPRESSOR);
1403
1404 c_seg_dst->c_populated_offset += C_SEG_BYTES_TO_OFFSET(size_to_populate);
1405 assert(C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset) <= C_SEG_BUFSIZE);
1406 }
1407 c_seg_alloc_nextslot(c_seg_dst);
1408
1409 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, c_seg_dst->c_nextslot);
1410
1411 memcpy(&c_seg_dst->c_store.c_buffer[c_seg_dst->c_nextoffset], &c_seg_src->c_store.c_buffer[c_src->c_offset], c_size);
1412
1413 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
1414
1415 c_seg_major_compact_stats.moved_slots++;
1416 c_seg_major_compact_stats.moved_bytes += c_size;
1417
1418 cslot_copy(c_dst, c_src);
1419 c_dst->c_offset = c_seg_dst->c_nextoffset;
1420
1421 if (c_seg_dst->c_firstemptyslot == c_seg_dst->c_nextslot)
1422 c_seg_dst->c_firstemptyslot++;
1423 c_seg_dst->c_nextslot++;
1424 c_seg_dst->c_bytes_used += c_rounded_size;
1425 c_seg_dst->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
1426
1427 PACK_C_SIZE(c_src, 0);
1428
1429 c_seg_src->c_bytes_used -= c_rounded_size;
1430 c_seg_src->c_bytes_unused += c_rounded_size;
1431 c_seg_src->c_firstemptyslot = 0;
1432
1433 if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX_INDEX) {
1434 /* dest segment is now full */
1435 keep_compacting = FALSE;
1436 break;
1437 }
1438 }
1439 #if DEVELOPMENT || DEBUG
1440 C_SEG_WRITE_PROTECT(c_seg_dst);
1441 #endif
1442 if (dst_slot < c_seg_dst->c_nextslot) {
1443
1444 PAGE_REPLACEMENT_ALLOWED(TRUE);
1445 /*
1446 * we've now locked out c_decompress from
1447 * converting the slot passed into it into
1448 * a c_segment_t which allows us to use
1449 * the backptr to change which c_segment and
1450 * index the slot points to
1451 */
1452 while (dst_slot < c_seg_dst->c_nextslot) {
1453
1454 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, dst_slot);
1455
1456 slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
1457 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
1458 slot_ptr->s_cseg = c_seg_dst->c_mysegno + 1;
1459 slot_ptr->s_cindx = dst_slot++;
1460 }
1461 PAGE_REPLACEMENT_ALLOWED(FALSE);
1462 }
1463 return (keep_compacting);
1464 }
1465
1466
1467 uint64_t
1468 vm_compressor_compute_elapsed_msecs(clock_sec_t end_sec, clock_nsec_t end_nsec, clock_sec_t start_sec, clock_nsec_t start_nsec)
1469 {
1470 uint64_t end_msecs;
1471 uint64_t start_msecs;
1472
1473 end_msecs = (end_sec * 1000) + end_nsec / 1000000;
1474 start_msecs = (start_sec * 1000) + start_nsec / 1000000;
1475
1476 return (end_msecs - start_msecs);
1477 }
1478
1479
1480
1481 uint32_t compressor_eval_period_in_msecs = 250;
1482 uint32_t compressor_sample_min_in_msecs = 500;
1483 uint32_t compressor_sample_max_in_msecs = 10000;
1484 uint32_t compressor_thrashing_threshold_per_10msecs = 50;
1485 uint32_t compressor_thrashing_min_per_10msecs = 20;
1486
1487 /* When true, reset sample data next chance we get. */
1488 static boolean_t compressor_need_sample_reset = FALSE;
1489
1490 extern uint32_t vm_page_filecache_min;
1491
1492
1493 void
1494 compute_swapout_target_age(void)
1495 {
1496 clock_sec_t cur_ts_sec;
1497 clock_nsec_t cur_ts_nsec;
1498 uint32_t min_operations_needed_in_this_sample;
1499 uint64_t elapsed_msecs_in_eval;
1500 uint64_t elapsed_msecs_in_sample;
1501 boolean_t need_eval_reset = FALSE;
1502
1503 clock_get_system_nanotime(&cur_ts_sec, &cur_ts_nsec);
1504
1505 elapsed_msecs_in_sample = vm_compressor_compute_elapsed_msecs(cur_ts_sec, cur_ts_nsec, start_of_sample_period_sec, start_of_sample_period_nsec);
1506
1507 if (compressor_need_sample_reset ||
1508 elapsed_msecs_in_sample >= compressor_sample_max_in_msecs) {
1509 compressor_need_sample_reset = TRUE;
1510 need_eval_reset = TRUE;
1511 goto done;
1512 }
1513 elapsed_msecs_in_eval = vm_compressor_compute_elapsed_msecs(cur_ts_sec, cur_ts_nsec, start_of_eval_period_sec, start_of_eval_period_nsec);
1514
1515 if (elapsed_msecs_in_eval < compressor_eval_period_in_msecs)
1516 goto done;
1517 need_eval_reset = TRUE;
1518
1519 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_START, elapsed_msecs_in_eval, sample_period_compression_count, sample_period_decompression_count, 0, 0);
1520
1521 min_operations_needed_in_this_sample = (compressor_thrashing_min_per_10msecs * (uint32_t)elapsed_msecs_in_eval) / 10;
1522
1523 if ((sample_period_compression_count - last_eval_compression_count) < min_operations_needed_in_this_sample ||
1524 (sample_period_decompression_count - last_eval_decompression_count) < min_operations_needed_in_this_sample) {
1525
1526 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, sample_period_compression_count - last_eval_compression_count,
1527 sample_period_decompression_count - last_eval_decompression_count, 0, 1, 0);
1528
1529 swapout_target_age = 0;
1530
1531 compressor_need_sample_reset = TRUE;
1532 need_eval_reset = TRUE;
1533 goto done;
1534 }
1535 last_eval_compression_count = sample_period_compression_count;
1536 last_eval_decompression_count = sample_period_decompression_count;
1537
1538 if (elapsed_msecs_in_sample < compressor_sample_min_in_msecs) {
1539
1540 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, swapout_target_age, 0, 0, 5, 0);
1541 goto done;
1542 }
1543 if (sample_period_decompression_count > ((compressor_thrashing_threshold_per_10msecs * elapsed_msecs_in_sample) / 10)) {
1544
1545 uint64_t running_total;
1546 uint64_t working_target;
1547 uint64_t aging_target;
1548 uint32_t oldest_age_of_csegs_sampled = 0;
1549 uint64_t working_set_approximation = 0;
1550
1551 swapout_target_age = 0;
1552
1553 working_target = (sample_period_decompression_count / 100) * 95; /* 95 percent */
1554 aging_target = (sample_period_decompression_count / 100) * 1; /* 1 percent */
1555 running_total = 0;
1556
1557 for (oldest_age_of_csegs_sampled = 0; oldest_age_of_csegs_sampled < DECOMPRESSION_SAMPLE_MAX_AGE; oldest_age_of_csegs_sampled++) {
1558
1559 running_total += age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
1560
1561 working_set_approximation += oldest_age_of_csegs_sampled * age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
1562
1563 if (running_total >= working_target)
1564 break;
1565 }
1566 if (oldest_age_of_csegs_sampled < DECOMPRESSION_SAMPLE_MAX_AGE) {
1567
1568 working_set_approximation = (working_set_approximation * 1000) / elapsed_msecs_in_sample;
1569
1570 if (working_set_approximation < VM_PAGE_COMPRESSOR_COUNT) {
1571
1572 running_total = overage_decompressions_during_sample_period;
1573
1574 for (oldest_age_of_csegs_sampled = DECOMPRESSION_SAMPLE_MAX_AGE - 1; oldest_age_of_csegs_sampled; oldest_age_of_csegs_sampled--) {
1575 running_total += age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
1576
1577 if (running_total >= aging_target)
1578 break;
1579 }
1580 swapout_target_age = (uint32_t)cur_ts_sec - oldest_age_of_csegs_sampled;
1581
1582 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, swapout_target_age, working_set_approximation, VM_PAGE_COMPRESSOR_COUNT, 2, 0);
1583 } else {
1584 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, working_set_approximation, VM_PAGE_COMPRESSOR_COUNT, 0, 3, 0);
1585 }
1586 } else
1587 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, working_target, running_total, 0, 4, 0);
1588
1589 compressor_need_sample_reset = TRUE;
1590 need_eval_reset = TRUE;
1591 } else
1592 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, sample_period_decompression_count, (compressor_thrashing_threshold_per_10msecs * elapsed_msecs_in_sample) / 10, 0, 6, 0);
1593 done:
1594 if (compressor_need_sample_reset == TRUE) {
1595 bzero(age_of_decompressions_during_sample_period, sizeof(age_of_decompressions_during_sample_period));
1596 overage_decompressions_during_sample_period = 0;
1597
1598 start_of_sample_period_sec = cur_ts_sec;
1599 start_of_sample_period_nsec = cur_ts_nsec;
1600 sample_period_decompression_count = 0;
1601 sample_period_compression_count = 0;
1602 last_eval_decompression_count = 0;
1603 last_eval_compression_count = 0;
1604 compressor_need_sample_reset = FALSE;
1605 }
1606 if (need_eval_reset == TRUE) {
1607 start_of_eval_period_sec = cur_ts_sec;
1608 start_of_eval_period_nsec = cur_ts_nsec;
1609 }
1610 }
1611
1612
1613 int compaction_swapper_init_now = 0;
1614 int compaction_swapper_running = 0;
1615 int compaction_swapper_awakened = 0;
1616 int compaction_swapper_abort = 0;
1617
1618
1619 #if CONFIG_JETSAM
1620 boolean_t memorystatus_kill_on_VM_thrashing(boolean_t);
1621 boolean_t memorystatus_kill_on_FC_thrashing(boolean_t);
1622 int compressor_thrashing_induced_jetsam = 0;
1623 int filecache_thrashing_induced_jetsam = 0;
1624 static boolean_t vm_compressor_thrashing_detected = FALSE;
1625 #endif /* CONFIG_JETSAM */
1626
1627 static boolean_t
1628 compressor_needs_to_swap(void)
1629 {
1630 boolean_t should_swap = FALSE;
1631
1632 if (vm_swapout_ripe_segments == TRUE && c_overage_swapped_count < c_overage_swapped_limit) {
1633 c_segment_t c_seg;
1634 clock_sec_t now;
1635 clock_sec_t age;
1636 clock_nsec_t nsec;
1637
1638 clock_get_system_nanotime(&now, &nsec);
1639 age = 0;
1640
1641 lck_mtx_lock_spin_always(c_list_lock);
1642
1643 if ( !queue_empty(&c_age_list_head)) {
1644 c_seg = (c_segment_t) queue_first(&c_age_list_head);
1645
1646 age = now - c_seg->c_creation_ts;
1647 }
1648 lck_mtx_unlock_always(c_list_lock);
1649
1650 if (age >= vm_ripe_target_age)
1651 return (TRUE);
1652 }
1653 if (VM_CONFIG_SWAP_IS_ACTIVE) {
1654 if (COMPRESSOR_NEEDS_TO_SWAP()) {
1655 return (TRUE);
1656 }
1657 if (VM_PAGE_Q_THROTTLED(&vm_pageout_queue_external) && vm_page_anonymous_count < (vm_page_inactive_count / 20)) {
1658 return (TRUE);
1659 }
1660 if (vm_page_free_count < (vm_page_free_reserved - (COMPRESSOR_FREE_RESERVED_LIMIT * 2)))
1661 return (TRUE);
1662 }
1663 compute_swapout_target_age();
1664
1665 if (swapout_target_age) {
1666 c_segment_t c_seg;
1667
1668 lck_mtx_lock_spin_always(c_list_lock);
1669
1670 if (!queue_empty(&c_age_list_head)) {
1671
1672 c_seg = (c_segment_t) queue_first(&c_age_list_head);
1673
1674 if (c_seg->c_creation_ts > swapout_target_age)
1675 swapout_target_age = 0;
1676 }
1677 lck_mtx_unlock_always(c_list_lock);
1678 }
1679 #if CONFIG_PHANTOM_CACHE
1680 if (vm_phantom_cache_check_pressure())
1681 should_swap = TRUE;
1682 #endif
1683 if (swapout_target_age)
1684 should_swap = TRUE;
1685
1686 #if CONFIG_JETSAM
1687 if (should_swap || c_segment_pages_compressed > c_segment_pages_compressed_nearing_limit) {
1688
1689 if (vm_compressor_thrashing_detected == FALSE) {
1690 vm_compressor_thrashing_detected = TRUE;
1691
1692 if (swapout_target_age || c_segment_pages_compressed > c_segment_pages_compressed_nearing_limit) {
1693 memorystatus_kill_on_VM_thrashing(TRUE /* async */);
1694 compressor_thrashing_induced_jetsam++;
1695 } else {
1696 memorystatus_kill_on_FC_thrashing(TRUE /* async */);
1697 filecache_thrashing_induced_jetsam++;
1698 }
1699 }
1700 /*
1701 * let the jetsam take precedence over
1702 * any major compactions we might have
1703 * been able to do... otherwise we run
1704 * the risk of doing major compactions
1705 * on segments we're about to free up
1706 * due to the jetsam activity.
1707 */
1708 should_swap = FALSE;
1709 }
1710
1711 #endif /* CONFIG_JETSAM */
1712
1713 if (should_swap == FALSE) {
1714 /*
1715 * vm_compressor_needs_to_major_compact returns true only if we're
1716 * about to run out of available compressor segments... in this
1717 * case, we absolutely need to run a major compaction even if
1718 * we've just kicked off a jetsam or we don't otherwise need to
1719 * swap... terminating objects releases
1720 * pages back to the uncompressed cache, but does not guarantee
1721 * that we will free up even a single compression segment
1722 */
1723 should_swap = vm_compressor_needs_to_major_compact();
1724 }
1725
1726 /*
1727 * returning TRUE when swap_supported == FALSE
1728 * will cause the major compaction engine to
1729 * run, but will not trigger any swapping...
1730 * segments that have been major compacted
1731 * will be moved to the majorcompact queue
1732 */
1733 return (should_swap);
1734 }
1735
1736 #if CONFIG_JETSAM
1737 /*
1738 * This function is called from the jetsam thread after killing something to
1739 * mitigate thrashing.
1740 *
1741 * We need to restart our thrashing detection heuristics since memory pressure
1742 * has potentially changed significantly, and we don't want to detect on old
1743 * data from before the jetsam.
1744 */
1745 void
1746 vm_thrashing_jetsam_done(void)
1747 {
1748 vm_compressor_thrashing_detected = FALSE;
1749
1750 /* Were we compressor-thrashing or filecache-thrashing? */
1751 if (swapout_target_age) {
1752 swapout_target_age = 0;
1753 compressor_need_sample_reset = TRUE;
1754 }
1755 #if CONFIG_PHANTOM_CACHE
1756 else {
1757 vm_phantom_cache_restart_sample();
1758 }
1759 #endif
1760 }
1761 #endif /* CONFIG_JETSAM */
1762
1763 uint32_t vm_wake_compactor_swapper_calls = 0;
1764 uint32_t vm_run_compactor_already_running = 0;
1765 uint32_t vm_run_compactor_empty_minor_q = 0;
1766 uint32_t vm_run_compactor_did_compact = 0;
1767 uint32_t vm_run_compactor_waited = 0;
1768
1769 void
1770 vm_run_compactor(void)
1771 {
1772 if (c_segment_count == 0)
1773 return;
1774
1775 lck_mtx_lock_spin_always(c_list_lock);
1776
1777 if (c_minor_count == 0) {
1778 vm_run_compactor_empty_minor_q++;
1779
1780 lck_mtx_unlock_always(c_list_lock);
1781 return;
1782 }
1783 if (compaction_swapper_running) {
1784
1785 if (vm_restricted_to_single_processor == FALSE) {
1786 vm_run_compactor_already_running++;
1787
1788 lck_mtx_unlock_always(c_list_lock);
1789 return;
1790 }
1791 vm_run_compactor_waited++;
1792
1793 assert_wait((event_t)&compaction_swapper_running, THREAD_UNINT);
1794
1795 lck_mtx_unlock_always(c_list_lock);
1796
1797 thread_block(THREAD_CONTINUE_NULL);
1798
1799 return;
1800 }
1801 vm_run_compactor_did_compact++;
1802
1803 fastwake_warmup = FALSE;
1804 compaction_swapper_running = 1;
1805
1806 vm_compressor_do_delayed_compactions(FALSE);
1807
1808 compaction_swapper_running = 0;
1809
1810 lck_mtx_unlock_always(c_list_lock);
1811
1812 thread_wakeup((event_t)&compaction_swapper_running);
1813 }
1814
1815
1816 void
1817 vm_wake_compactor_swapper(void)
1818 {
1819 if (compaction_swapper_running || compaction_swapper_awakened || c_segment_count == 0)
1820 return;
1821
1822 if (c_minor_count || vm_compressor_needs_to_major_compact()) {
1823
1824 lck_mtx_lock_spin_always(c_list_lock);
1825
1826 fastwake_warmup = FALSE;
1827
1828 if (compaction_swapper_running == 0 && compaction_swapper_awakened == 0) {
1829
1830 vm_wake_compactor_swapper_calls++;
1831
1832 compaction_swapper_awakened = 1;
1833 thread_wakeup((event_t)&c_compressor_swap_trigger);
1834 }
1835 lck_mtx_unlock_always(c_list_lock);
1836 }
1837 }
1838
1839
1840 void
1841 vm_consider_swapping()
1842 {
1843 c_segment_t c_seg, c_seg_next;
1844 clock_sec_t now;
1845 clock_nsec_t nsec;
1846
1847 assert(VM_CONFIG_SWAP_IS_PRESENT);
1848
1849 lck_mtx_lock_spin_always(c_list_lock);
1850
1851 compaction_swapper_abort = 1;
1852
1853 while (compaction_swapper_running) {
1854 assert_wait((event_t)&compaction_swapper_running, THREAD_UNINT);
1855
1856 lck_mtx_unlock_always(c_list_lock);
1857
1858 thread_block(THREAD_CONTINUE_NULL);
1859
1860 lck_mtx_lock_spin_always(c_list_lock);
1861 }
1862 compaction_swapper_abort = 0;
1863 compaction_swapper_running = 1;
1864
1865 vm_swapout_ripe_segments = TRUE;
1866
1867 if (!queue_empty(&c_major_list_head)) {
1868
1869 clock_get_system_nanotime(&now, &nsec);
1870
1871 c_seg = (c_segment_t)queue_first(&c_major_list_head);
1872
1873 while (!queue_end(&c_major_list_head, (queue_entry_t)c_seg)) {
1874
1875 if (c_overage_swapped_count >= c_overage_swapped_limit)
1876 break;
1877
1878 c_seg_next = (c_segment_t) queue_next(&c_seg->c_age_list);
1879
1880 if ((now - c_seg->c_creation_ts) >= vm_ripe_target_age) {
1881
1882 lck_mtx_lock_spin_always(&c_seg->c_lock);
1883
1884 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
1885
1886 lck_mtx_unlock_always(&c_seg->c_lock);
1887 }
1888 c_seg = c_seg_next;
1889 }
1890 }
1891 vm_compressor_compact_and_swap(FALSE);
1892
1893 compaction_swapper_running = 0;
1894
1895 vm_swapout_ripe_segments = FALSE;
1896
1897 lck_mtx_unlock_always(c_list_lock);
1898
1899 thread_wakeup((event_t)&compaction_swapper_running);
1900 }
1901
1902
1903 void
1904 vm_consider_waking_compactor_swapper(void)
1905 {
1906 boolean_t need_wakeup = FALSE;
1907
1908 if (c_segment_count == 0)
1909 return;
1910
1911 if (compaction_swapper_running || compaction_swapper_awakened)
1912 return;
1913
1914 if (!compaction_swapper_inited && !compaction_swapper_init_now) {
1915 compaction_swapper_init_now = 1;
1916 need_wakeup = TRUE;
1917 }
1918
1919 if (c_minor_count && (COMPRESSOR_NEEDS_TO_MINOR_COMPACT())) {
1920
1921 need_wakeup = TRUE;
1922
1923 } else if (compressor_needs_to_swap()) {
1924
1925 need_wakeup = TRUE;
1926
1927 } else if (c_minor_count) {
1928 uint64_t total_bytes;
1929
1930 total_bytes = compressor_object->resident_page_count * PAGE_SIZE_64;
1931
1932 if ((total_bytes - compressor_bytes_used) > total_bytes / 10)
1933 need_wakeup = TRUE;
1934 }
1935 if (need_wakeup == TRUE) {
1936
1937 lck_mtx_lock_spin_always(c_list_lock);
1938
1939 fastwake_warmup = FALSE;
1940
1941 if (compaction_swapper_running == 0 && compaction_swapper_awakened == 0) {
1942 memoryshot(VM_WAKEUP_COMPACTOR_SWAPPER, DBG_FUNC_NONE);
1943
1944 compaction_swapper_awakened = 1;
1945 thread_wakeup((event_t)&c_compressor_swap_trigger);
1946 }
1947 lck_mtx_unlock_always(c_list_lock);
1948 }
1949 }
1950
1951
1952 #define C_SWAPOUT_LIMIT 4
1953 #define DELAYED_COMPACTIONS_PER_PASS 30
1954
1955 void
1956 vm_compressor_do_delayed_compactions(boolean_t flush_all)
1957 {
1958 c_segment_t c_seg;
1959 int number_compacted = 0;
1960 boolean_t needs_to_swap = FALSE;
1961
1962
1963 LCK_MTX_ASSERT(c_list_lock, LCK_MTX_ASSERT_OWNED);
1964
1965 while (!queue_empty(&c_minor_list_head) && needs_to_swap == FALSE) {
1966
1967 c_seg = (c_segment_t)queue_first(&c_minor_list_head);
1968
1969 lck_mtx_lock_spin_always(&c_seg->c_lock);
1970
1971 if (c_seg->c_busy) {
1972
1973 lck_mtx_unlock_always(c_list_lock);
1974 c_seg_wait_on_busy(c_seg);
1975 lck_mtx_lock_spin_always(c_list_lock);
1976
1977 continue;
1978 }
1979 C_SEG_BUSY(c_seg);
1980
1981 c_seg_do_minor_compaction_and_unlock(c_seg, TRUE, FALSE, TRUE);
1982
1983 if (VM_CONFIG_SWAP_IS_ACTIVE && (number_compacted++ > DELAYED_COMPACTIONS_PER_PASS)) {
1984
1985 if ((flush_all == TRUE || compressor_needs_to_swap() == TRUE) && c_swapout_count < C_SWAPOUT_LIMIT)
1986 needs_to_swap = TRUE;
1987
1988 number_compacted = 0;
1989 }
1990 lck_mtx_lock_spin_always(c_list_lock);
1991 }
1992 }
1993
1994
1995 #define C_SEGMENT_SWAPPEDIN_AGE_LIMIT 10
1996
1997 static void
1998 vm_compressor_age_swapped_in_segments(boolean_t flush_all)
1999 {
2000 c_segment_t c_seg;
2001 clock_sec_t now;
2002 clock_nsec_t nsec;
2003
2004 clock_get_system_nanotime(&now, &nsec);
2005
2006 while (!queue_empty(&c_swappedin_list_head)) {
2007
2008 c_seg = (c_segment_t)queue_first(&c_swappedin_list_head);
2009
2010 if (flush_all == FALSE && (now - c_seg->c_swappedin_ts) < C_SEGMENT_SWAPPEDIN_AGE_LIMIT)
2011 break;
2012
2013 lck_mtx_lock_spin_always(&c_seg->c_lock);
2014
2015 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
2016
2017 lck_mtx_unlock_always(&c_seg->c_lock);
2018 }
2019 }
2020
2021
2022 extern int vm_num_swap_files;
2023 extern int vm_num_pinned_swap_files;
2024 extern int vm_swappin_enabled;
2025
2026 extern unsigned int vm_swapfile_total_segs_used;
2027 extern unsigned int vm_swapfile_total_segs_alloced;
2028
2029
2030 void
2031 vm_compressor_flush(void)
2032 {
2033 uint64_t vm_swap_put_failures_at_start;
2034 wait_result_t wait_result = 0;
2035 AbsoluteTime startTime, endTime;
2036 clock_sec_t now_sec;
2037 clock_nsec_t now_nsec;
2038 uint64_t nsec;
2039
2040 HIBLOG("vm_compressor_flush - starting\n");
2041
2042 clock_get_uptime(&startTime);
2043
2044 lck_mtx_lock_spin_always(c_list_lock);
2045
2046 fastwake_warmup = FALSE;
2047 compaction_swapper_abort = 1;
2048
2049 while (compaction_swapper_running) {
2050 assert_wait((event_t)&compaction_swapper_running, THREAD_UNINT);
2051
2052 lck_mtx_unlock_always(c_list_lock);
2053
2054 thread_block(THREAD_CONTINUE_NULL);
2055
2056 lck_mtx_lock_spin_always(c_list_lock);
2057 }
2058 compaction_swapper_abort = 0;
2059 compaction_swapper_running = 1;
2060
2061 hibernate_flushing = TRUE;
2062 hibernate_no_swapspace = FALSE;
2063 c_generation_id_flush_barrier = c_generation_id + 1000;
2064
2065 clock_get_system_nanotime(&now_sec, &now_nsec);
2066 hibernate_flushing_deadline = now_sec + HIBERNATE_FLUSHING_SECS_TO_COMPLETE;
2067
2068 vm_swap_put_failures_at_start = vm_swap_put_failures;
2069
2070 vm_compressor_compact_and_swap(TRUE);
2071
2072 while (!queue_empty(&c_swapout_list_head)) {
2073
2074 assert_wait_timeout((event_t) &compaction_swapper_running, THREAD_INTERRUPTIBLE, 5000, 1000*NSEC_PER_USEC);
2075
2076 lck_mtx_unlock_always(c_list_lock);
2077
2078 wait_result = thread_block(THREAD_CONTINUE_NULL);
2079
2080 lck_mtx_lock_spin_always(c_list_lock);
2081
2082 if (wait_result == THREAD_TIMED_OUT)
2083 break;
2084 }
2085 hibernate_flushing = FALSE;
2086 compaction_swapper_running = 0;
2087
2088 if (vm_swap_put_failures > vm_swap_put_failures_at_start)
2089 HIBLOG("vm_compressor_flush failed to clean %llu segments - vm_page_compressor_count(%d)\n",
2090 vm_swap_put_failures - vm_swap_put_failures_at_start, VM_PAGE_COMPRESSOR_COUNT);
2091
2092 lck_mtx_unlock_always(c_list_lock);
2093
2094 thread_wakeup((event_t)&compaction_swapper_running);
2095
2096 clock_get_uptime(&endTime);
2097 SUB_ABSOLUTETIME(&endTime, &startTime);
2098 absolutetime_to_nanoseconds(endTime, &nsec);
2099
2100 HIBLOG("vm_compressor_flush completed - took %qd msecs - vm_num_swap_files = %d, vm_num_pinned_swap_files = %d, vm_swappin_enabled = %d\n",
2101 nsec / 1000000ULL, vm_num_swap_files, vm_num_pinned_swap_files, vm_swappin_enabled);
2102 }
2103
2104
2105 int compaction_swap_trigger_thread_awakened = 0;
2106
2107 static void
2108 vm_compressor_swap_trigger_thread(void)
2109 {
2110 current_thread()->options |= TH_OPT_VMPRIV;
2111
2112 /*
2113 * compaction_swapper_init_now is set when the first call to
2114 * vm_consider_waking_compactor_swapper is made from
2115 * vm_pageout_scan... since this function is called upon
2116 * thread creation, we want to make sure to delay adjusting
2117 * the tuneables until we are awakened via vm_pageout_scan
2118 * so that we are at a point where the vm_swapfile_open will
2119 * be operating on the correct directory (in case the default
2120 * of /var/vm/ is overridden by the dymanic_pager
2121 */
2122 if (compaction_swapper_init_now) {
2123 vm_compaction_swapper_do_init();
2124
2125 if (vm_restricted_to_single_processor == TRUE)
2126 thread_vm_bind_group_add();
2127
2128 compaction_swapper_init_now = 0;
2129 }
2130 lck_mtx_lock_spin_always(c_list_lock);
2131
2132 compaction_swap_trigger_thread_awakened++;
2133 compaction_swapper_awakened = 0;
2134
2135 if (compaction_swapper_running == 0) {
2136
2137 compaction_swapper_running = 1;
2138
2139 vm_compressor_compact_and_swap(FALSE);
2140
2141 compaction_swapper_running = 0;
2142 }
2143 assert_wait((event_t)&c_compressor_swap_trigger, THREAD_UNINT);
2144
2145 if (compaction_swapper_running == 0)
2146 thread_wakeup((event_t)&compaction_swapper_running);
2147
2148 lck_mtx_unlock_always(c_list_lock);
2149
2150 thread_block((thread_continue_t)vm_compressor_swap_trigger_thread);
2151
2152 /* NOTREACHED */
2153 }
2154
2155
2156 void
2157 vm_compressor_record_warmup_start(void)
2158 {
2159 c_segment_t c_seg;
2160
2161 lck_mtx_lock_spin_always(c_list_lock);
2162
2163 if (first_c_segment_to_warm_generation_id == 0) {
2164 if (!queue_empty(&c_age_list_head)) {
2165
2166 c_seg = (c_segment_t)queue_last(&c_age_list_head);
2167
2168 first_c_segment_to_warm_generation_id = c_seg->c_generation_id;
2169 } else
2170 first_c_segment_to_warm_generation_id = 0;
2171
2172 fastwake_recording_in_progress = TRUE;
2173 }
2174 lck_mtx_unlock_always(c_list_lock);
2175 }
2176
2177
2178 void
2179 vm_compressor_record_warmup_end(void)
2180 {
2181 c_segment_t c_seg;
2182
2183 lck_mtx_lock_spin_always(c_list_lock);
2184
2185 if (fastwake_recording_in_progress == TRUE) {
2186
2187 if (!queue_empty(&c_age_list_head)) {
2188
2189 c_seg = (c_segment_t)queue_last(&c_age_list_head);
2190
2191 last_c_segment_to_warm_generation_id = c_seg->c_generation_id;
2192 } else
2193 last_c_segment_to_warm_generation_id = first_c_segment_to_warm_generation_id;
2194
2195 fastwake_recording_in_progress = FALSE;
2196
2197 HIBLOG("vm_compressor_record_warmup (%qd - %qd)\n", first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id);
2198 }
2199 lck_mtx_unlock_always(c_list_lock);
2200 }
2201
2202
2203 #define DELAY_TRIM_ON_WAKE_SECS 25
2204
2205 void
2206 vm_compressor_delay_trim(void)
2207 {
2208 clock_sec_t sec;
2209 clock_nsec_t nsec;
2210
2211 clock_get_system_nanotime(&sec, &nsec);
2212 dont_trim_until_ts = sec + DELAY_TRIM_ON_WAKE_SECS;
2213 }
2214
2215
2216 void
2217 vm_compressor_do_warmup(void)
2218 {
2219 lck_mtx_lock_spin_always(c_list_lock);
2220
2221 if (first_c_segment_to_warm_generation_id == last_c_segment_to_warm_generation_id) {
2222 first_c_segment_to_warm_generation_id = last_c_segment_to_warm_generation_id = 0;
2223
2224 lck_mtx_unlock_always(c_list_lock);
2225 return;
2226 }
2227
2228 if (compaction_swapper_running == 0 && compaction_swapper_awakened == 0) {
2229
2230 fastwake_warmup = TRUE;
2231
2232 compaction_swapper_awakened = 1;
2233 thread_wakeup((event_t)&c_compressor_swap_trigger);
2234 }
2235 lck_mtx_unlock_always(c_list_lock);
2236 }
2237
2238 void
2239 do_fastwake_warmup_all(void)
2240 {
2241
2242 lck_mtx_lock_spin_always(c_list_lock);
2243
2244 if (queue_empty(&c_swappedout_list_head) && queue_empty(&c_swappedout_sparse_list_head)) {
2245
2246 lck_mtx_unlock_always(c_list_lock);
2247 return;
2248 }
2249
2250 fastwake_warmup = TRUE;
2251
2252 do_fastwake_warmup(&c_swappedout_list_head, TRUE);
2253
2254 do_fastwake_warmup(&c_swappedout_sparse_list_head, TRUE);
2255
2256 fastwake_warmup = FALSE;
2257
2258 lck_mtx_unlock_always(c_list_lock);
2259
2260 }
2261
2262 void
2263 do_fastwake_warmup(queue_head_t *c_queue, boolean_t consider_all_cseg)
2264 {
2265 c_segment_t c_seg = NULL;
2266 AbsoluteTime startTime, endTime;
2267 uint64_t nsec;
2268
2269
2270 HIBLOG("vm_compressor_fastwake_warmup (%qd - %qd) - starting\n", first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id);
2271
2272 clock_get_uptime(&startTime);
2273
2274 lck_mtx_unlock_always(c_list_lock);
2275
2276 proc_set_thread_policy(current_thread(),
2277 TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER2);
2278
2279 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2280
2281 lck_mtx_lock_spin_always(c_list_lock);
2282
2283 while (!queue_empty(c_queue) && fastwake_warmup == TRUE) {
2284
2285 c_seg = (c_segment_t) queue_first(c_queue);
2286
2287 if (consider_all_cseg == FALSE) {
2288 if (c_seg->c_generation_id < first_c_segment_to_warm_generation_id ||
2289 c_seg->c_generation_id > last_c_segment_to_warm_generation_id)
2290 break;
2291
2292 if (vm_page_free_count < (AVAILABLE_MEMORY / 4))
2293 break;
2294 }
2295
2296 lck_mtx_lock_spin_always(&c_seg->c_lock);
2297 lck_mtx_unlock_always(c_list_lock);
2298
2299 if (c_seg->c_busy) {
2300 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2301 c_seg_wait_on_busy(c_seg);
2302 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2303 } else {
2304 if (c_seg_swapin(c_seg, TRUE, FALSE) == 0)
2305 lck_mtx_unlock_always(&c_seg->c_lock);
2306 c_segment_warmup_count++;
2307
2308 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2309 vm_pageout_io_throttle();
2310 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2311 }
2312 lck_mtx_lock_spin_always(c_list_lock);
2313 }
2314 lck_mtx_unlock_always(c_list_lock);
2315
2316 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2317
2318 proc_set_thread_policy(current_thread(),
2319 TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER0);
2320
2321 clock_get_uptime(&endTime);
2322 SUB_ABSOLUTETIME(&endTime, &startTime);
2323 absolutetime_to_nanoseconds(endTime, &nsec);
2324
2325 HIBLOG("vm_compressor_fastwake_warmup completed - took %qd msecs\n", nsec / 1000000ULL);
2326
2327 lck_mtx_lock_spin_always(c_list_lock);
2328
2329 if (consider_all_cseg == FALSE) {
2330 first_c_segment_to_warm_generation_id = last_c_segment_to_warm_generation_id = 0;
2331 }
2332 }
2333
2334
2335 void
2336 vm_compressor_compact_and_swap(boolean_t flush_all)
2337 {
2338 c_segment_t c_seg, c_seg_next;
2339 boolean_t keep_compacting;
2340 clock_sec_t now;
2341 clock_nsec_t nsec;
2342
2343
2344 if (fastwake_warmup == TRUE) {
2345 uint64_t starting_warmup_count;
2346
2347 starting_warmup_count = c_segment_warmup_count;
2348
2349 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 11) | DBG_FUNC_START, c_segment_warmup_count,
2350 first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id, 0, 0);
2351 do_fastwake_warmup(&c_swappedout_list_head, FALSE);
2352 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 11) | DBG_FUNC_END, c_segment_warmup_count, c_segment_warmup_count - starting_warmup_count, 0, 0, 0);
2353
2354 fastwake_warmup = FALSE;
2355 }
2356
2357 /*
2358 * it's possible for the c_age_list_head to be empty if we
2359 * hit our limits for growing the compressor pool and we subsequently
2360 * hibernated... on the next hibernation we could see the queue as
2361 * empty and not proceeed even though we have a bunch of segments on
2362 * the swapped in queue that need to be dealt with.
2363 */
2364 vm_compressor_do_delayed_compactions(flush_all);
2365
2366 vm_compressor_age_swapped_in_segments(flush_all);
2367
2368 /*
2369 * we only need to grab the timestamp once per
2370 * invocation of this function since the
2371 * timescale we're interested in is measured
2372 * in days
2373 */
2374 clock_get_system_nanotime(&now, &nsec);
2375
2376 while (!queue_empty(&c_age_list_head) && compaction_swapper_abort == 0) {
2377
2378 if (hibernate_flushing == TRUE) {
2379 clock_sec_t sec;
2380
2381 if (hibernate_should_abort()) {
2382 HIBLOG("vm_compressor_flush - hibernate_should_abort returned TRUE\n");
2383 break;
2384 }
2385 if (hibernate_no_swapspace == TRUE) {
2386 HIBLOG("vm_compressor_flush - out of swap space\n");
2387 break;
2388 }
2389 if (vm_swap_files_pinned() == FALSE) {
2390 HIBLOG("vm_compressor_flush - unpinned swap files\n");
2391 break;
2392 }
2393 if (hibernate_in_progress_with_pinned_swap == TRUE &&
2394 (vm_swapfile_total_segs_alloced == vm_swapfile_total_segs_used)) {
2395 HIBLOG("vm_compressor_flush - out of pinned swap space\n");
2396 break;
2397 }
2398 clock_get_system_nanotime(&sec, &nsec);
2399
2400 if (sec > hibernate_flushing_deadline) {
2401 HIBLOG("vm_compressor_flush - failed to finish before deadline\n");
2402 break;
2403 }
2404 }
2405 if (c_swapout_count >= C_SWAPOUT_LIMIT) {
2406
2407 assert_wait_timeout((event_t) &compaction_swapper_running, THREAD_INTERRUPTIBLE, 100, 1000*NSEC_PER_USEC);
2408
2409 lck_mtx_unlock_always(c_list_lock);
2410
2411 thread_block(THREAD_CONTINUE_NULL);
2412
2413 lck_mtx_lock_spin_always(c_list_lock);
2414 }
2415 /*
2416 * Minor compactions
2417 */
2418 vm_compressor_do_delayed_compactions(flush_all);
2419
2420 vm_compressor_age_swapped_in_segments(flush_all);
2421
2422 if (c_swapout_count >= C_SWAPOUT_LIMIT) {
2423 /*
2424 * we timed out on the above thread_block
2425 * let's loop around and try again
2426 * the timeout allows us to continue
2427 * to do minor compactions to make
2428 * more memory available
2429 */
2430 continue;
2431 }
2432
2433 /*
2434 * Swap out segments?
2435 */
2436 if (flush_all == FALSE) {
2437 boolean_t needs_to_swap;
2438
2439 lck_mtx_unlock_always(c_list_lock);
2440
2441 needs_to_swap = compressor_needs_to_swap();
2442
2443 if (needs_to_swap == TRUE && vm_swap_low_on_space())
2444 vm_compressor_take_paging_space_action();
2445
2446 lck_mtx_lock_spin_always(c_list_lock);
2447
2448 if (needs_to_swap == FALSE)
2449 break;
2450 }
2451 if (queue_empty(&c_age_list_head))
2452 break;
2453 c_seg = (c_segment_t) queue_first(&c_age_list_head);
2454
2455 assert(c_seg->c_state == C_ON_AGE_Q);
2456
2457 if (flush_all == TRUE && c_seg->c_generation_id > c_generation_id_flush_barrier)
2458 break;
2459
2460 lck_mtx_lock_spin_always(&c_seg->c_lock);
2461
2462 if (c_seg->c_busy) {
2463
2464 lck_mtx_unlock_always(c_list_lock);
2465 c_seg_wait_on_busy(c_seg);
2466 lck_mtx_lock_spin_always(c_list_lock);
2467
2468 continue;
2469 }
2470 C_SEG_BUSY(c_seg);
2471
2472 if (c_seg_do_minor_compaction_and_unlock(c_seg, FALSE, TRUE, TRUE)) {
2473 /*
2474 * found an empty c_segment and freed it
2475 * so go grab the next guy in the queue
2476 */
2477 c_seg_major_compact_stats.count_of_freed_segs++;
2478 continue;
2479 }
2480 /*
2481 * Major compaction
2482 */
2483 keep_compacting = TRUE;
2484
2485 while (keep_compacting == TRUE) {
2486
2487 assert(c_seg->c_busy);
2488
2489 /* look for another segment to consolidate */
2490
2491 c_seg_next = (c_segment_t) queue_next(&c_seg->c_age_list);
2492
2493 if (queue_end(&c_age_list_head, (queue_entry_t)c_seg_next))
2494 break;
2495
2496 assert(c_seg_next->c_state == C_ON_AGE_Q);
2497
2498 if (c_seg_major_compact_ok(c_seg, c_seg_next) == FALSE)
2499 break;
2500
2501 lck_mtx_lock_spin_always(&c_seg_next->c_lock);
2502
2503 if (c_seg_next->c_busy) {
2504
2505 lck_mtx_unlock_always(c_list_lock);
2506 c_seg_wait_on_busy(c_seg_next);
2507 lck_mtx_lock_spin_always(c_list_lock);
2508
2509 continue;
2510 }
2511 /* grab that segment */
2512 C_SEG_BUSY(c_seg_next);
2513
2514 if (c_seg_do_minor_compaction_and_unlock(c_seg_next, FALSE, TRUE, TRUE)) {
2515 /*
2516 * found an empty c_segment and freed it
2517 * so we can't continue to use c_seg_next
2518 */
2519 c_seg_major_compact_stats.count_of_freed_segs++;
2520 continue;
2521 }
2522
2523 /* unlock the list ... */
2524 lck_mtx_unlock_always(c_list_lock);
2525
2526 /* do the major compaction */
2527
2528 keep_compacting = c_seg_major_compact(c_seg, c_seg_next);
2529
2530 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2531
2532 lck_mtx_lock_spin_always(&c_seg_next->c_lock);
2533 /*
2534 * run a minor compaction on the donor segment
2535 * since we pulled at least some of it's
2536 * data into our target... if we've emptied
2537 * it, now is a good time to free it which
2538 * c_seg_minor_compaction_and_unlock also takes care of
2539 *
2540 * by passing TRUE, we ask for c_busy to be cleared
2541 * and c_wanted to be taken care of
2542 */
2543 if (c_seg_minor_compaction_and_unlock(c_seg_next, TRUE))
2544 c_seg_major_compact_stats.count_of_freed_segs++;
2545
2546 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2547
2548 /* relock the list */
2549 lck_mtx_lock_spin_always(c_list_lock);
2550
2551 } /* major compaction */
2552
2553 lck_mtx_lock_spin_always(&c_seg->c_lock);
2554
2555 assert(c_seg->c_busy);
2556 assert(!c_seg->c_on_minorcompact_q);
2557
2558 if (VM_CONFIG_SWAP_IS_ACTIVE) {
2559 /*
2560 * This mode of putting a generic c_seg on the swapout list is
2561 * only supported when we have general swapping enabled
2562 */
2563 c_seg_switch_state(c_seg, C_ON_SWAPOUT_Q, FALSE);
2564 } else {
2565 if ((vm_swapout_ripe_segments == TRUE && c_overage_swapped_count < c_overage_swapped_limit)) {
2566
2567 assert(VM_CONFIG_SWAP_IS_PRESENT);
2568 /*
2569 * we are running compressor sweeps with swap-behind
2570 * make sure the c_seg has aged enough before swapping it
2571 * out...
2572 */
2573 if ((now - c_seg->c_creation_ts) >= vm_ripe_target_age) {
2574 c_seg->c_overage_swap = TRUE;
2575 c_overage_swapped_count++;
2576 c_seg_switch_state(c_seg, C_ON_SWAPOUT_Q, FALSE);
2577 }
2578 }
2579 }
2580 if (c_seg->c_state == C_ON_AGE_Q) {
2581 /*
2582 * this c_seg didn't get moved to the swapout queue
2583 * so we need to move it out of the way...
2584 * we just did a major compaction on it so put it
2585 * on that queue
2586 */
2587 c_seg_switch_state(c_seg, C_ON_MAJORCOMPACT_Q, FALSE);
2588 } else {
2589 c_seg_major_compact_stats.wasted_space_in_swapouts += C_SEG_BUFSIZE - c_seg->c_bytes_used;
2590 c_seg_major_compact_stats.count_of_swapouts++;
2591 }
2592 C_SEG_WAKEUP_DONE(c_seg);
2593
2594 lck_mtx_unlock_always(&c_seg->c_lock);
2595
2596 if (c_swapout_count) {
2597 lck_mtx_unlock_always(c_list_lock);
2598
2599 thread_wakeup((event_t)&c_swapout_list_head);
2600
2601 lck_mtx_lock_spin_always(c_list_lock);
2602 }
2603 }
2604 }
2605
2606
2607 static c_segment_t
2608 c_seg_allocate(c_segment_t *current_chead)
2609 {
2610 c_segment_t c_seg;
2611 int min_needed;
2612 int size_to_populate;
2613
2614 if (vm_compressor_low_on_space())
2615 vm_compressor_take_paging_space_action();
2616
2617 if ( (c_seg = *current_chead) == NULL ) {
2618 uint32_t c_segno;
2619
2620 lck_mtx_lock_spin_always(c_list_lock);
2621
2622 while (c_segments_busy == TRUE) {
2623 assert_wait((event_t) (&c_segments_busy), THREAD_UNINT);
2624
2625 lck_mtx_unlock_always(c_list_lock);
2626
2627 thread_block(THREAD_CONTINUE_NULL);
2628
2629 lck_mtx_lock_spin_always(c_list_lock);
2630 }
2631 if (c_free_segno_head == (uint32_t)-1) {
2632 uint32_t c_segments_available_new;
2633
2634 if (c_segments_available >= c_segments_limit || c_segment_pages_compressed >= c_segment_pages_compressed_limit) {
2635 lck_mtx_unlock_always(c_list_lock);
2636
2637 return (NULL);
2638 }
2639 c_segments_busy = TRUE;
2640 lck_mtx_unlock_always(c_list_lock);
2641
2642 kernel_memory_populate(compressor_map, (vm_offset_t)c_segments_next_page,
2643 PAGE_SIZE, KMA_KOBJECT, VM_KERN_MEMORY_COMPRESSOR);
2644 c_segments_next_page += PAGE_SIZE;
2645
2646 c_segments_available_new = c_segments_available + C_SEGMENTS_PER_PAGE;
2647
2648 if (c_segments_available_new > c_segments_limit)
2649 c_segments_available_new = c_segments_limit;
2650
2651 for (c_segno = c_segments_available + 1; c_segno < c_segments_available_new; c_segno++)
2652 c_segments[c_segno - 1].c_segno = c_segno;
2653
2654 lck_mtx_lock_spin_always(c_list_lock);
2655
2656 c_segments[c_segno - 1].c_segno = c_free_segno_head;
2657 c_free_segno_head = c_segments_available;
2658 c_segments_available = c_segments_available_new;
2659
2660 c_segments_busy = FALSE;
2661 thread_wakeup((event_t) (&c_segments_busy));
2662 }
2663 c_segno = c_free_segno_head;
2664 assert(c_segno >= 0 && c_segno < c_segments_limit);
2665
2666 c_free_segno_head = (uint32_t)c_segments[c_segno].c_segno;
2667
2668 /*
2669 * do the rest of the bookkeeping now while we're still behind
2670 * the list lock and grab our generation id now into a local
2671 * so that we can install it once we have the c_seg allocated
2672 */
2673 c_segment_count++;
2674 if (c_segment_count > c_segment_count_max)
2675 c_segment_count_max = c_segment_count;
2676
2677 lck_mtx_unlock_always(c_list_lock);
2678
2679 c_seg = (c_segment_t)zalloc(compressor_segment_zone);
2680 bzero((char *)c_seg, sizeof(struct c_segment));
2681
2682 c_seg->c_store.c_buffer = (int32_t *)C_SEG_BUFFER_ADDRESS(c_segno);
2683
2684 lck_mtx_init(&c_seg->c_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
2685
2686 c_seg->c_state = C_IS_EMPTY;
2687 c_seg->c_firstemptyslot = C_SLOT_MAX_INDEX;
2688 c_seg->c_mysegno = c_segno;
2689
2690 lck_mtx_lock_spin_always(c_list_lock);
2691 c_empty_count++;
2692 c_seg_switch_state(c_seg, C_IS_FILLING, FALSE);
2693 c_segments[c_segno].c_seg = c_seg;
2694 assert(c_segments[c_segno].c_segno > c_segments_available);
2695 lck_mtx_unlock_always(c_list_lock);
2696
2697 *current_chead = c_seg;
2698
2699 #if DEVELOPMENT || DEBUG
2700 C_SEG_MAKE_WRITEABLE(c_seg);
2701 #endif
2702
2703 }
2704 c_seg_alloc_nextslot(c_seg);
2705
2706 size_to_populate = C_SEG_ALLOCSIZE - C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset);
2707
2708 if (size_to_populate) {
2709
2710 min_needed = PAGE_SIZE + (C_SEG_ALLOCSIZE - C_SEG_BUFSIZE);
2711
2712 if (C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset - c_seg->c_nextoffset) < (unsigned) min_needed) {
2713
2714 if (size_to_populate > C_SEG_MAX_POPULATE_SIZE)
2715 size_to_populate = C_SEG_MAX_POPULATE_SIZE;
2716
2717 kernel_memory_populate(compressor_map,
2718 (vm_offset_t) &c_seg->c_store.c_buffer[c_seg->c_populated_offset],
2719 size_to_populate,
2720 KMA_COMPRESSOR,
2721 VM_KERN_MEMORY_COMPRESSOR);
2722 } else
2723 size_to_populate = 0;
2724 }
2725 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2726
2727 lck_mtx_lock_spin_always(&c_seg->c_lock);
2728
2729 if (size_to_populate)
2730 c_seg->c_populated_offset += C_SEG_BYTES_TO_OFFSET(size_to_populate);
2731
2732 return (c_seg);
2733 }
2734
2735
2736 static void
2737 c_current_seg_filled(c_segment_t c_seg, c_segment_t *current_chead)
2738 {
2739 uint32_t unused_bytes;
2740 uint32_t offset_to_depopulate;
2741 int new_state = C_ON_AGE_Q;
2742 clock_sec_t sec;
2743 clock_nsec_t nsec;
2744
2745 unused_bytes = trunc_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset - c_seg->c_nextoffset));
2746
2747 if (unused_bytes) {
2748
2749 offset_to_depopulate = C_SEG_BYTES_TO_OFFSET(round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_nextoffset)));
2750
2751 /*
2752 * release the extra physical page(s) at the end of the segment
2753 */
2754 lck_mtx_unlock_always(&c_seg->c_lock);
2755
2756 kernel_memory_depopulate(
2757 compressor_map,
2758 (vm_offset_t) &c_seg->c_store.c_buffer[offset_to_depopulate],
2759 unused_bytes,
2760 KMA_COMPRESSOR);
2761
2762 lck_mtx_lock_spin_always(&c_seg->c_lock);
2763
2764 c_seg->c_populated_offset = offset_to_depopulate;
2765 }
2766 assert(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset) <= C_SEG_BUFSIZE);
2767
2768 #if DEVELOPMENT || DEBUG
2769 {
2770 boolean_t c_seg_was_busy = FALSE;
2771
2772 if ( !c_seg->c_busy)
2773 C_SEG_BUSY(c_seg);
2774 else
2775 c_seg_was_busy = TRUE;
2776
2777 lck_mtx_unlock_always(&c_seg->c_lock);
2778
2779 C_SEG_WRITE_PROTECT(c_seg);
2780
2781 lck_mtx_lock_spin_always(&c_seg->c_lock);
2782
2783 if (c_seg_was_busy == FALSE)
2784 C_SEG_WAKEUP_DONE(c_seg);
2785 }
2786 #endif
2787
2788 #if CONFIG_FREEZE
2789 if (current_chead == (c_segment_t*)&freezer_chead &&
2790 VM_CONFIG_SWAP_IS_PRESENT &&
2791 VM_CONFIG_FREEZER_SWAP_IS_ACTIVE &&
2792 c_freezer_swapout_count < VM_MAX_FREEZER_CSEG_SWAP_COUNT) {
2793 new_state = C_ON_SWAPOUT_Q;
2794 }
2795 #endif /* CONFIG_FREEZE */
2796
2797 clock_get_system_nanotime(&sec, &nsec);
2798 c_seg->c_creation_ts = (uint32_t)sec;
2799
2800 lck_mtx_lock_spin_always(c_list_lock);
2801
2802 #if CONFIG_FREEZE
2803 if (c_seg->c_state == C_ON_SWAPOUT_Q)
2804 c_freezer_swapout_count++;
2805 #endif /* CONFIG_FREEZE */
2806
2807 c_seg->c_generation_id = c_generation_id++;
2808 c_seg_switch_state(c_seg, new_state, FALSE);
2809
2810 if (c_seg->c_state == C_ON_AGE_Q && C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE)
2811 c_seg_need_delayed_compaction(c_seg, TRUE);
2812
2813 lck_mtx_unlock_always(c_list_lock);
2814
2815 #if CONFIG_FREEZE
2816 if (c_seg->c_state == C_ON_SWAPOUT_Q)
2817 thread_wakeup((event_t)&c_swapout_list_head);
2818 #endif /* CONFIG_FREEZE */
2819
2820 *current_chead = NULL;
2821 }
2822
2823
2824 /*
2825 * returns with c_seg locked
2826 */
2827 void
2828 c_seg_swapin_requeue(c_segment_t c_seg, boolean_t has_data, boolean_t minor_compact_ok, boolean_t age_on_swapin_q)
2829 {
2830 clock_sec_t sec;
2831 clock_nsec_t nsec;
2832
2833 clock_get_system_nanotime(&sec, &nsec);
2834
2835 lck_mtx_lock_spin_always(c_list_lock);
2836 lck_mtx_lock_spin_always(&c_seg->c_lock);
2837
2838 assert(c_seg->c_busy_swapping);
2839 assert(c_seg->c_busy);
2840
2841 c_seg->c_busy_swapping = 0;
2842
2843 if (c_seg->c_overage_swap == TRUE) {
2844 c_overage_swapped_count--;
2845 c_seg->c_overage_swap = FALSE;
2846 }
2847 if (has_data == TRUE) {
2848 if (age_on_swapin_q == TRUE)
2849 c_seg_switch_state(c_seg, C_ON_SWAPPEDIN_Q, FALSE);
2850 else
2851 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
2852
2853 if (minor_compact_ok == TRUE && !c_seg->c_on_minorcompact_q && C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE)
2854 c_seg_need_delayed_compaction(c_seg, TRUE);
2855 } else {
2856 c_seg->c_store.c_buffer = (int32_t*) NULL;
2857 c_seg->c_populated_offset = C_SEG_BYTES_TO_OFFSET(0);
2858
2859 c_seg_switch_state(c_seg, C_ON_BAD_Q, FALSE);
2860 }
2861 c_seg->c_swappedin_ts = (uint32_t)sec;
2862
2863 lck_mtx_unlock_always(c_list_lock);
2864 }
2865
2866
2867
2868 /*
2869 * c_seg has to be locked and is returned locked if the c_seg isn't freed
2870 * PAGE_REPLACMENT_DISALLOWED has to be TRUE on entry and is returned TRUE
2871 * c_seg_swapin returns 1 if the c_seg was freed, 0 otherwise
2872 */
2873
2874 int
2875 c_seg_swapin(c_segment_t c_seg, boolean_t force_minor_compaction, boolean_t age_on_swapin_q)
2876 {
2877 vm_offset_t addr = 0;
2878 uint32_t io_size = 0;
2879 uint64_t f_offset;
2880
2881 assert(C_SEG_IS_ONDISK(c_seg));
2882
2883 #if !CHECKSUM_THE_SWAP
2884 c_seg_trim_tail(c_seg);
2885 #endif
2886 io_size = round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset));
2887 f_offset = c_seg->c_store.c_swap_handle;
2888
2889 C_SEG_BUSY(c_seg);
2890 c_seg->c_busy_swapping = 1;
2891
2892 /*
2893 * This thread is likely going to block for I/O.
2894 * Make sure it is ready to run when the I/O completes because
2895 * it needs to clear the busy bit on the c_seg so that other
2896 * waiting threads can make progress too. To do that, boost
2897 * the rwlock_count so that the priority is boosted.
2898 */
2899 set_thread_rwlock_boost();
2900 lck_mtx_unlock_always(&c_seg->c_lock);
2901
2902 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2903
2904 addr = (vm_offset_t)C_SEG_BUFFER_ADDRESS(c_seg->c_mysegno);
2905 c_seg->c_store.c_buffer = (int32_t*) addr;
2906
2907 kernel_memory_populate(compressor_map, addr, io_size, KMA_COMPRESSOR, VM_KERN_MEMORY_COMPRESSOR);
2908
2909 if (vm_swap_get(c_seg, f_offset, io_size) != KERN_SUCCESS) {
2910 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2911
2912 c_seg->c_store.c_swap_handle = f_offset;
2913
2914 kernel_memory_depopulate(compressor_map, addr, io_size, KMA_COMPRESSOR);
2915
2916 c_seg_swapin_requeue(c_seg, FALSE, TRUE, age_on_swapin_q);
2917 } else {
2918 c_seg->c_store.c_buffer = (int32_t*) addr;
2919 #if ENCRYPTED_SWAP
2920 vm_swap_decrypt(c_seg);
2921 #endif /* ENCRYPTED_SWAP */
2922
2923 #if CHECKSUM_THE_SWAP
2924 if (c_seg->cseg_swap_size != io_size)
2925 panic("swapin size doesn't match swapout size");
2926
2927 if (c_seg->cseg_hash != hash_string((char*) c_seg->c_store.c_buffer, (int)io_size)) {
2928 panic("c_seg_swapin - Swap hash mismatch\n");
2929 }
2930 #endif /* CHECKSUM_THE_SWAP */
2931
2932 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2933
2934 c_seg_swapin_requeue(c_seg, TRUE, force_minor_compaction == TRUE ? FALSE : TRUE, age_on_swapin_q);
2935
2936 OSAddAtomic64(c_seg->c_bytes_used, &compressor_bytes_used);
2937
2938 if (force_minor_compaction == TRUE) {
2939 if (c_seg_minor_compaction_and_unlock(c_seg, FALSE)) {
2940 /*
2941 * Drop the rwlock_count so that the thread priority
2942 * is returned back to where it is supposed to be.
2943 */
2944 clear_thread_rwlock_boost();
2945 return (1);
2946 }
2947
2948 lck_mtx_lock_spin_always(&c_seg->c_lock);
2949 }
2950 }
2951 C_SEG_WAKEUP_DONE(c_seg);
2952
2953 /*
2954 * Drop the rwlock_count so that the thread priority
2955 * is returned back to where it is supposed to be.
2956 */
2957 clear_thread_rwlock_boost();
2958
2959 return (0);
2960 }
2961
2962
2963 static void
2964 c_segment_sv_hash_drop_ref(int hash_indx)
2965 {
2966 struct c_sv_hash_entry o_sv_he, n_sv_he;
2967
2968 while (1) {
2969
2970 o_sv_he.he_record = c_segment_sv_hash_table[hash_indx].he_record;
2971
2972 n_sv_he.he_ref = o_sv_he.he_ref - 1;
2973 n_sv_he.he_data = o_sv_he.he_data;
2974
2975 if (OSCompareAndSwap64((UInt64)o_sv_he.he_record, (UInt64)n_sv_he.he_record, (UInt64 *) &c_segment_sv_hash_table[hash_indx].he_record) == TRUE) {
2976 if (n_sv_he.he_ref == 0)
2977 OSAddAtomic(-1, &c_segment_svp_in_hash);
2978 break;
2979 }
2980 }
2981 }
2982
2983
2984 static int
2985 c_segment_sv_hash_insert(uint32_t data)
2986 {
2987 int hash_sindx;
2988 int misses;
2989 struct c_sv_hash_entry o_sv_he, n_sv_he;
2990 boolean_t got_ref = FALSE;
2991
2992 if (data == 0)
2993 OSAddAtomic(1, &c_segment_svp_zero_compressions);
2994 else
2995 OSAddAtomic(1, &c_segment_svp_nonzero_compressions);
2996
2997 hash_sindx = data & C_SV_HASH_MASK;
2998
2999 for (misses = 0; misses < C_SV_HASH_MAX_MISS; misses++)
3000 {
3001 o_sv_he.he_record = c_segment_sv_hash_table[hash_sindx].he_record;
3002
3003 while (o_sv_he.he_data == data || o_sv_he.he_ref == 0) {
3004 n_sv_he.he_ref = o_sv_he.he_ref + 1;
3005 n_sv_he.he_data = data;
3006
3007 if (OSCompareAndSwap64((UInt64)o_sv_he.he_record, (UInt64)n_sv_he.he_record, (UInt64 *) &c_segment_sv_hash_table[hash_sindx].he_record) == TRUE) {
3008 if (n_sv_he.he_ref == 1)
3009 OSAddAtomic(1, &c_segment_svp_in_hash);
3010 got_ref = TRUE;
3011 break;
3012 }
3013 o_sv_he.he_record = c_segment_sv_hash_table[hash_sindx].he_record;
3014 }
3015 if (got_ref == TRUE)
3016 break;
3017 hash_sindx++;
3018
3019 if (hash_sindx == C_SV_HASH_SIZE)
3020 hash_sindx = 0;
3021 }
3022 if (got_ref == FALSE)
3023 return(-1);
3024
3025 return (hash_sindx);
3026 }
3027
3028
3029 #if RECORD_THE_COMPRESSED_DATA
3030
3031 static void
3032 c_compressed_record_data(char *src, int c_size)
3033 {
3034 if ((c_compressed_record_cptr + c_size + 4) >= c_compressed_record_ebuf)
3035 panic("c_compressed_record_cptr >= c_compressed_record_ebuf");
3036
3037 *(int *)((void *)c_compressed_record_cptr) = c_size;
3038
3039 c_compressed_record_cptr += 4;
3040
3041 memcpy(c_compressed_record_cptr, src, c_size);
3042 c_compressed_record_cptr += c_size;
3043 }
3044 #endif
3045
3046
3047 static int
3048 c_compress_page(char *src, c_slot_mapping_t slot_ptr, c_segment_t *current_chead, char *scratch_buf)
3049 {
3050 int c_size;
3051 int c_rounded_size = 0;
3052 int max_csize;
3053 c_slot_t cs;
3054 c_segment_t c_seg;
3055
3056 KERNEL_DEBUG(0xe0400000 | DBG_FUNC_START, *current_chead, 0, 0, 0, 0);
3057 retry:
3058 if ((c_seg = c_seg_allocate(current_chead)) == NULL)
3059 return (1);
3060 /*
3061 * returns with c_seg lock held
3062 * and PAGE_REPLACEMENT_DISALLOWED(TRUE)...
3063 * c_nextslot has been allocated and
3064 * c_store.c_buffer populated
3065 */
3066 assert(c_seg->c_state == C_IS_FILLING);
3067
3068 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_seg->c_nextslot);
3069
3070 cs->c_packed_ptr = C_SLOT_PACK_PTR(slot_ptr);
3071 assert(slot_ptr == (c_slot_mapping_t)C_SLOT_UNPACK_PTR(cs));
3072
3073 cs->c_offset = c_seg->c_nextoffset;
3074
3075 max_csize = C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES((int32_t)cs->c_offset);
3076
3077 if (max_csize > PAGE_SIZE)
3078 max_csize = PAGE_SIZE;
3079
3080 #if CHECKSUM_THE_DATA
3081 cs->c_hash_data = hash_string(src, PAGE_SIZE);
3082 #endif
3083
3084 if (vm_compressor_algorithm() != VM_COMPRESSOR_DEFAULT_CODEC) {
3085 } else {
3086 c_size = WKdm_compress_new((const WK_word *)(uintptr_t)src, (WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
3087 (WK_word *)(uintptr_t)scratch_buf, max_csize - 4);
3088 }
3089 assert(c_size <= (max_csize - 4) && c_size >= -1);
3090
3091 if (c_size == -1) {
3092
3093 if (max_csize < PAGE_SIZE) {
3094 c_current_seg_filled(c_seg, current_chead);
3095 assert(*current_chead == NULL);
3096
3097 lck_mtx_unlock_always(&c_seg->c_lock);
3098
3099 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3100 goto retry;
3101 }
3102 c_size = PAGE_SIZE;
3103
3104 memcpy(&c_seg->c_store.c_buffer[cs->c_offset], src, c_size);
3105
3106 OSAddAtomic(1, &c_segment_noncompressible_pages);
3107
3108 } else if (c_size == 0) {
3109 int hash_index;
3110
3111 /*
3112 * special case - this is a page completely full of a single 32 bit value
3113 */
3114 hash_index = c_segment_sv_hash_insert(*(uint32_t *)(uintptr_t)src);
3115
3116 if (hash_index != -1) {
3117 slot_ptr->s_cindx = hash_index;
3118 slot_ptr->s_cseg = C_SV_CSEG_ID;
3119
3120 OSAddAtomic(1, &c_segment_svp_hash_succeeded);
3121 #if RECORD_THE_COMPRESSED_DATA
3122 c_compressed_record_data(src, 4);
3123 #endif
3124 goto sv_compression;
3125 }
3126 c_size = 4;
3127
3128 memcpy(&c_seg->c_store.c_buffer[cs->c_offset], src, c_size);
3129
3130 OSAddAtomic(1, &c_segment_svp_hash_failed);
3131 }
3132
3133 #if RECORD_THE_COMPRESSED_DATA
3134 c_compressed_record_data((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size);
3135 #endif
3136
3137 #if CHECKSUM_THE_COMPRESSED_DATA
3138 cs->c_hash_compressed_data = hash_string((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size);
3139 #endif
3140 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
3141
3142 PACK_C_SIZE(cs, c_size);
3143 c_seg->c_bytes_used += c_rounded_size;
3144 c_seg->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
3145
3146 slot_ptr->s_cindx = c_seg->c_nextslot++;
3147 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
3148 slot_ptr->s_cseg = c_seg->c_mysegno + 1;
3149
3150 sv_compression:
3151 if (c_seg->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg->c_nextslot >= C_SLOT_MAX_INDEX) {
3152 c_current_seg_filled(c_seg, current_chead);
3153 assert(*current_chead == NULL);
3154 }
3155 lck_mtx_unlock_always(&c_seg->c_lock);
3156
3157 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3158
3159 #if RECORD_THE_COMPRESSED_DATA
3160 if ((c_compressed_record_cptr - c_compressed_record_sbuf) >= C_SEG_ALLOCSIZE) {
3161 c_compressed_record_write(c_compressed_record_sbuf, (int)(c_compressed_record_cptr - c_compressed_record_sbuf));
3162 c_compressed_record_cptr = c_compressed_record_sbuf;
3163 }
3164 #endif
3165 if (c_size) {
3166 OSAddAtomic64(c_size, &c_segment_compressed_bytes);
3167 OSAddAtomic64(c_rounded_size, &compressor_bytes_used);
3168 }
3169 OSAddAtomic64(PAGE_SIZE, &c_segment_input_bytes);
3170
3171 OSAddAtomic(1, &c_segment_pages_compressed);
3172 OSAddAtomic(1, &sample_period_compression_count);
3173
3174 KERNEL_DEBUG(0xe0400000 | DBG_FUNC_END, *current_chead, c_size, c_segment_input_bytes, c_segment_compressed_bytes, 0);
3175
3176 return (0);
3177 }
3178
3179 static inline void sv_decompress(int32_t *ddst, int32_t pattern) {
3180 #if __x86_64__
3181 memset_word(ddst, pattern, PAGE_SIZE / sizeof(int32_t));
3182 #else
3183 size_t i;
3184
3185 /* Unroll the pattern fill loop 4x to encourage the
3186 * compiler to emit NEON stores, cf.
3187 * <rdar://problem/25839866> Loop autovectorization
3188 * anomalies.
3189 * We use separate loops for each PAGE_SIZE
3190 * to allow the autovectorizer to engage, as PAGE_SIZE
3191 * is currently not a constant.
3192 */
3193
3194 if (PAGE_SIZE == 4096) {
3195 for (i = 0; i < (4096U / sizeof(int32_t)); i += 4) {
3196 *ddst++ = pattern;
3197 *ddst++ = pattern;
3198 *ddst++ = pattern;
3199 *ddst++ = pattern;
3200 }
3201 } else {
3202 assert(PAGE_SIZE == 16384);
3203 for (i = 0; i < (int)(16384U / sizeof(int32_t)); i += 4) {
3204 *ddst++ = pattern;
3205 *ddst++ = pattern;
3206 *ddst++ = pattern;
3207 *ddst++ = pattern;
3208 }
3209 }
3210 #endif
3211 }
3212
3213 static int
3214 c_decompress_page(char *dst, volatile c_slot_mapping_t slot_ptr, int flags, int *zeroslot)
3215 {
3216 c_slot_t cs;
3217 c_segment_t c_seg;
3218 uint32_t c_segno;
3219 int c_indx;
3220 int c_rounded_size;
3221 uint32_t c_size;
3222 int retval = 0;
3223 boolean_t need_unlock = TRUE;
3224 boolean_t consider_defragmenting = FALSE;
3225 boolean_t kdp_mode = FALSE;
3226
3227 if (__improbable(flags & C_KDP)) {
3228 if (not_in_kdp) {
3229 panic("C_KDP passed to decompress page from outside of debugger context");
3230 }
3231
3232 assert((flags & C_KEEP) == C_KEEP);
3233 assert((flags & C_DONT_BLOCK) == C_DONT_BLOCK);
3234
3235 if ((flags & (C_DONT_BLOCK | C_KEEP)) != (C_DONT_BLOCK | C_KEEP)) {
3236 return (-2);
3237 }
3238
3239 kdp_mode = TRUE;
3240 *zeroslot = 0;
3241 }
3242
3243 ReTry:
3244 if (__probable(!kdp_mode)) {
3245 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3246 } else {
3247 if (kdp_lck_rw_lock_is_acquired_exclusive(&c_master_lock)) {
3248 return (-2);
3249 }
3250 }
3251
3252 #if HIBERNATION
3253 /*
3254 * if hibernation is enabled, it indicates (via a call
3255 * to 'vm_decompressor_lock' that no further
3256 * decompressions are allowed once it reaches
3257 * the point of flushing all of the currently dirty
3258 * anonymous memory through the compressor and out
3259 * to disk... in this state we allow freeing of compressed
3260 * pages and must honor the C_DONT_BLOCK case
3261 */
3262 if (dst && decompressions_blocked == TRUE) {
3263 if (flags & C_DONT_BLOCK) {
3264
3265 if (__probable(!kdp_mode)) {
3266 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3267 }
3268
3269 *zeroslot = 0;
3270 return (-2);
3271 }
3272 /*
3273 * it's safe to atomically assert and block behind the
3274 * lock held in shared mode because "decompressions_blocked" is
3275 * only set and cleared and the thread_wakeup done when the lock
3276 * is held exclusively
3277 */
3278 assert_wait((event_t)&decompressions_blocked, THREAD_UNINT);
3279
3280 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3281
3282 thread_block(THREAD_CONTINUE_NULL);
3283
3284 goto ReTry;
3285 }
3286 #endif
3287 /* s_cseg is actually "segno+1" */
3288 c_segno = slot_ptr->s_cseg - 1;
3289
3290 if (__improbable(c_segno >= c_segments_available))
3291 panic("c_decompress_page: c_segno %d >= c_segments_available %d, slot_ptr(%p), slot_data(%x)",
3292 c_segno, c_segments_available, slot_ptr, *(int *)((void *)slot_ptr));
3293
3294 if (__improbable(c_segments[c_segno].c_segno < c_segments_available))
3295 panic("c_decompress_page: c_segno %d is free, slot_ptr(%p), slot_data(%x)",
3296 c_segno, slot_ptr, *(int *)((void *)slot_ptr));
3297
3298 c_seg = c_segments[c_segno].c_seg;
3299
3300 if (__probable(!kdp_mode)) {
3301 lck_mtx_lock_spin_always(&c_seg->c_lock);
3302 } else {
3303 if (kdp_lck_mtx_lock_spin_is_acquired(&c_seg->c_lock)) {
3304 return (-2);
3305 }
3306 }
3307
3308 assert(c_seg->c_state != C_IS_EMPTY && c_seg->c_state != C_IS_FREE);
3309
3310 if (dst == NULL && c_seg->c_busy_swapping) {
3311 assert(c_seg->c_busy);
3312
3313 goto bypass_busy_check;
3314 }
3315 if (flags & C_DONT_BLOCK) {
3316 if (c_seg->c_busy || (C_SEG_IS_ONDISK(c_seg) && dst)) {
3317 *zeroslot = 0;
3318
3319 retval = -2;
3320 goto done;
3321 }
3322 }
3323 if (c_seg->c_busy) {
3324
3325 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3326
3327 c_seg_wait_on_busy(c_seg);
3328
3329 goto ReTry;
3330 }
3331 bypass_busy_check:
3332
3333 c_indx = slot_ptr->s_cindx;
3334
3335 if (__improbable(c_indx >= c_seg->c_nextslot))
3336 panic("c_decompress_page: c_indx %d >= c_nextslot %d, c_seg(%p), slot_ptr(%p), slot_data(%x)",
3337 c_indx, c_seg->c_nextslot, c_seg, slot_ptr, *(int *)((void *)slot_ptr));
3338
3339 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
3340
3341 c_size = UNPACK_C_SIZE(cs);
3342
3343 if (__improbable(c_size == 0))
3344 panic("c_decompress_page: c_size == 0, c_seg(%p), slot_ptr(%p), slot_data(%x)",
3345 c_seg, slot_ptr, *(int *)((void *)slot_ptr));
3346
3347 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
3348
3349 if (dst) {
3350 uint32_t age_of_cseg;
3351 clock_sec_t cur_ts_sec;
3352 clock_nsec_t cur_ts_nsec;
3353
3354 if (C_SEG_IS_ONDISK(c_seg)) {
3355 assert(kdp_mode == FALSE);
3356 retval = c_seg_swapin(c_seg, FALSE, TRUE);
3357 assert(retval == 0);
3358
3359 retval = 1;
3360 }
3361 if (c_seg->c_state == C_ON_BAD_Q) {
3362 assert(c_seg->c_store.c_buffer == NULL);
3363
3364 retval = -1;
3365 goto c_seg_invalid_data;
3366 }
3367 #if CHECKSUM_THE_COMPRESSED_DATA
3368 if (cs->c_hash_compressed_data != hash_string((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size))
3369 panic("compressed data doesn't match original hash: 0x%x, seg: %p, offset: %d, c_size: %d", cs->c_hash_compressed_data, c_seg, cs->c_offset, c_size);
3370 #endif
3371 if (c_rounded_size == PAGE_SIZE) {
3372 /*
3373 * page wasn't compressible... just copy it out
3374 */
3375 memcpy(dst, &c_seg->c_store.c_buffer[cs->c_offset], PAGE_SIZE);
3376 } else if (c_size == 4) {
3377 int32_t data;
3378 int32_t *dptr;
3379
3380 /*
3381 * page was populated with a single value
3382 * that didn't fit into our fast hash
3383 * so we packed it in as a single non-compressed value
3384 * that we need to populate the page with
3385 */
3386 dptr = (int32_t *)(uintptr_t)dst;
3387 data = *(int32_t *)(&c_seg->c_store.c_buffer[cs->c_offset]);
3388 sv_decompress(dptr, data);
3389 } else {
3390 uint32_t my_cpu_no;
3391 char *scratch_buf;
3392
3393 if (__probable(!kdp_mode)) {
3394 /*
3395 * we're behind the c_seg lock held in spin mode
3396 * which means pre-emption is disabled... therefore
3397 * the following sequence is atomic and safe
3398 */
3399 my_cpu_no = cpu_number();
3400
3401 assert(my_cpu_no < compressor_cpus);
3402
3403 scratch_buf = &compressor_scratch_bufs[my_cpu_no * vm_compressor_get_decode_scratch_size()];
3404 } else {
3405 scratch_buf = kdp_compressor_scratch_buf;
3406 }
3407
3408 if (vm_compressor_algorithm() != VM_COMPRESSOR_DEFAULT_CODEC) {
3409 } else {
3410 WKdm_decompress_new((WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
3411 (WK_word *)(uintptr_t)dst, (WK_word *)(uintptr_t)scratch_buf, c_size);
3412 }
3413 }
3414
3415 #if CHECKSUM_THE_DATA
3416 if (cs->c_hash_data != hash_string(dst, PAGE_SIZE))
3417 panic("decompressed data doesn't match original cs: %p, hash: %d, offset: %d, c_size: %d", cs, cs->c_hash_data, cs->c_offset, c_size);
3418
3419 #endif
3420 if (c_seg->c_swappedin_ts == 0 && !kdp_mode) {
3421
3422 clock_get_system_nanotime(&cur_ts_sec, &cur_ts_nsec);
3423
3424 age_of_cseg = (uint32_t)cur_ts_sec - c_seg->c_creation_ts;
3425 if (age_of_cseg < DECOMPRESSION_SAMPLE_MAX_AGE)
3426 OSAddAtomic(1, &age_of_decompressions_during_sample_period[age_of_cseg]);
3427 else
3428 OSAddAtomic(1, &overage_decompressions_during_sample_period);
3429
3430 OSAddAtomic(1, &sample_period_decompression_count);
3431 }
3432 }
3433 c_seg_invalid_data:
3434
3435 if (flags & C_KEEP) {
3436 *zeroslot = 0;
3437 goto done;
3438 }
3439 assert(kdp_mode == FALSE);
3440
3441 c_seg->c_bytes_unused += c_rounded_size;
3442 c_seg->c_bytes_used -= c_rounded_size;
3443 PACK_C_SIZE(cs, 0);
3444
3445 if (c_indx < c_seg->c_firstemptyslot)
3446 c_seg->c_firstemptyslot = c_indx;
3447
3448 OSAddAtomic(-1, &c_segment_pages_compressed);
3449
3450 if (c_seg->c_state != C_ON_BAD_Q && !(C_SEG_IS_ONDISK(c_seg))) {
3451 /*
3452 * C_SEG_IS_ONDISK == TRUE can occur when we're doing a
3453 * free of a compressed page (i.e. dst == NULL)
3454 */
3455 OSAddAtomic64(-c_rounded_size, &compressor_bytes_used);
3456 }
3457 if (c_seg->c_busy_swapping) {
3458 /*
3459 * bypass case for c_busy_swapping...
3460 * let the swapin/swapout paths deal with putting
3461 * the c_seg on the minor compaction queue if needed
3462 */
3463 assert(c_seg->c_busy);
3464 goto done;
3465 }
3466 assert(!c_seg->c_busy);
3467
3468 if (c_seg->c_state != C_IS_FILLING) {
3469 if (c_seg->c_bytes_used == 0) {
3470 if ( !(C_SEG_IS_ONDISK(c_seg))) {
3471 int pages_populated;
3472
3473 pages_populated = (round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE;
3474 c_seg->c_populated_offset = C_SEG_BYTES_TO_OFFSET(0);
3475
3476 if (pages_populated) {
3477
3478 assert(c_seg->c_state != C_ON_BAD_Q);
3479 assert(c_seg->c_store.c_buffer != NULL);
3480
3481 C_SEG_BUSY(c_seg);
3482 lck_mtx_unlock_always(&c_seg->c_lock);
3483
3484 kernel_memory_depopulate(compressor_map, (vm_offset_t) c_seg->c_store.c_buffer, pages_populated * PAGE_SIZE, KMA_COMPRESSOR);
3485
3486 lck_mtx_lock_spin_always(&c_seg->c_lock);
3487 C_SEG_WAKEUP_DONE(c_seg);
3488 }
3489 if (!c_seg->c_on_minorcompact_q && c_seg->c_state != C_ON_SWAPOUT_Q)
3490 c_seg_need_delayed_compaction(c_seg, FALSE);
3491 } else {
3492 if (c_seg->c_state != C_ON_SWAPPEDOUTSPARSE_Q) {
3493
3494 c_seg_move_to_sparse_list(c_seg);
3495 consider_defragmenting = TRUE;
3496 }
3497 }
3498 } else if (c_seg->c_on_minorcompact_q) {
3499
3500 assert(c_seg->c_state != C_ON_BAD_Q);
3501
3502 if (C_SEG_SHOULD_MINORCOMPACT(c_seg)) {
3503 c_seg_try_minor_compaction_and_unlock(c_seg);
3504 need_unlock = FALSE;
3505 }
3506 } else if ( !(C_SEG_IS_ONDISK(c_seg))) {
3507
3508 if (c_seg->c_state != C_ON_BAD_Q && c_seg->c_state != C_ON_SWAPOUT_Q && C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE) {
3509 c_seg_need_delayed_compaction(c_seg, FALSE);
3510 }
3511 } else if (c_seg->c_state != C_ON_SWAPPEDOUTSPARSE_Q && C_SEG_ONDISK_IS_SPARSE(c_seg)) {
3512
3513 c_seg_move_to_sparse_list(c_seg);
3514 consider_defragmenting = TRUE;
3515 }
3516 }
3517 done:
3518 if (__improbable(kdp_mode)) {
3519 return retval;
3520 }
3521
3522 if (need_unlock == TRUE)
3523 lck_mtx_unlock_always(&c_seg->c_lock);
3524
3525 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3526
3527 if (consider_defragmenting == TRUE)
3528 vm_swap_consider_defragmenting();
3529
3530
3531 return (retval);
3532 }
3533
3534
3535 int
3536 vm_compressor_get(ppnum_t pn, int *slot, int flags)
3537 {
3538 c_slot_mapping_t slot_ptr;
3539 char *dst;
3540 int zeroslot = 1;
3541 int retval;
3542
3543 #if __x86_64__
3544 dst = PHYSMAP_PTOV((uint64_t)pn << (uint64_t)PAGE_SHIFT);
3545 #else
3546 #error "unsupported architecture"
3547 #endif
3548 slot_ptr = (c_slot_mapping_t)slot;
3549
3550 if (slot_ptr->s_cseg == C_SV_CSEG_ID) {
3551 int32_t data;
3552 int32_t *dptr;
3553
3554 /*
3555 * page was populated with a single value
3556 * that found a home in our hash table
3557 * grab that value from the hash and populate the page
3558 * that we need to populate the page with
3559 */
3560 dptr = (int32_t *)(uintptr_t)dst;
3561 data = c_segment_sv_hash_table[slot_ptr->s_cindx].he_data;
3562 #if __x86_64__
3563 memset_word(dptr, data, PAGE_SIZE / sizeof(int32_t));
3564 #else
3565 {
3566 int i;
3567
3568 for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++)
3569 *dptr++ = data;
3570 }
3571 #endif
3572 if ( !(flags & C_KEEP)) {
3573 c_segment_sv_hash_drop_ref(slot_ptr->s_cindx);
3574
3575 OSAddAtomic(-1, &c_segment_pages_compressed);
3576 *slot = 0;
3577 }
3578 if (data)
3579 OSAddAtomic(1, &c_segment_svp_nonzero_decompressions);
3580 else
3581 OSAddAtomic(1, &c_segment_svp_zero_decompressions);
3582
3583 return (0);
3584 }
3585
3586 retval = c_decompress_page(dst, slot_ptr, flags, &zeroslot);
3587
3588 /*
3589 * zeroslot will be set to 0 by c_decompress_page if (flags & C_KEEP)
3590 * or (flags & C_DONT_BLOCK) and we found 'c_busy' or 'C_SEG_IS_ONDISK' to be TRUE
3591 */
3592 if (zeroslot) {
3593 *slot = 0;
3594 }
3595 /*
3596 * returns 0 if we successfully decompressed a page from a segment already in memory
3597 * returns 1 if we had to first swap in the segment, before successfully decompressing the page
3598 * returns -1 if we encountered an error swapping in the segment - decompression failed
3599 * returns -2 if (flags & C_DONT_BLOCK) and we found 'c_busy' or 'C_SEG_IS_ONDISK' to be true
3600 */
3601 return (retval);
3602 }
3603
3604
3605 int
3606 vm_compressor_free(int *slot, int flags)
3607 {
3608 c_slot_mapping_t slot_ptr;
3609 int zeroslot = 1;
3610 int retval;
3611
3612 assert(flags == 0 || flags == C_DONT_BLOCK);
3613
3614 slot_ptr = (c_slot_mapping_t)slot;
3615
3616 if (slot_ptr->s_cseg == C_SV_CSEG_ID) {
3617
3618 c_segment_sv_hash_drop_ref(slot_ptr->s_cindx);
3619 OSAddAtomic(-1, &c_segment_pages_compressed);
3620
3621 *slot = 0;
3622 return (0);
3623 }
3624 retval = c_decompress_page(NULL, slot_ptr, flags, &zeroslot);
3625 /*
3626 * returns 0 if we successfully freed the specified compressed page
3627 * returns -2 if (flags & C_DONT_BLOCK) and we found 'c_busy' set
3628 */
3629
3630 if (retval == 0)
3631 *slot = 0;
3632 else
3633 assert(retval == -2);
3634
3635 return (retval);
3636 }
3637
3638
3639 int
3640 vm_compressor_put(ppnum_t pn, int *slot, void **current_chead, char *scratch_buf)
3641 {
3642 char *src;
3643 int retval;
3644
3645 #if __x86_64__
3646 src = PHYSMAP_PTOV((uint64_t)pn << (uint64_t)PAGE_SHIFT);
3647 #else
3648 #error "unsupported architecture"
3649 #endif
3650 retval = c_compress_page(src, (c_slot_mapping_t)slot, (c_segment_t *)current_chead, scratch_buf);
3651
3652 return (retval);
3653 }
3654
3655 void
3656 vm_compressor_transfer(
3657 int *dst_slot_p,
3658 int *src_slot_p)
3659 {
3660 c_slot_mapping_t dst_slot, src_slot;
3661 c_segment_t c_seg;
3662 int c_indx;
3663 c_slot_t cs;
3664
3665 src_slot = (c_slot_mapping_t) src_slot_p;
3666
3667 if (src_slot->s_cseg == C_SV_CSEG_ID) {
3668 *dst_slot_p = *src_slot_p;
3669 *src_slot_p = 0;
3670 return;
3671 }
3672 dst_slot = (c_slot_mapping_t) dst_slot_p;
3673 Retry:
3674 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3675 /* get segment for src_slot */
3676 c_seg = c_segments[src_slot->s_cseg -1].c_seg;
3677 /* lock segment */
3678 lck_mtx_lock_spin_always(&c_seg->c_lock);
3679 /* wait if it's busy */
3680 if (c_seg->c_busy && !c_seg->c_busy_swapping) {
3681 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3682 c_seg_wait_on_busy(c_seg);
3683 goto Retry;
3684 }
3685 /* find the c_slot */
3686 c_indx = src_slot->s_cindx;
3687 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
3688 /* point the c_slot back to dst_slot instead of src_slot */
3689 cs->c_packed_ptr = C_SLOT_PACK_PTR(dst_slot);
3690 /* transfer */
3691 *dst_slot_p = *src_slot_p;
3692 *src_slot_p = 0;
3693 lck_mtx_unlock_always(&c_seg->c_lock);
3694 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3695 }
3696
3697 #if CONFIG_FREEZE
3698
3699 int freezer_finished_filling = 0;
3700
3701 void
3702 vm_compressor_finished_filling(
3703 void **current_chead)
3704 {
3705 c_segment_t c_seg;
3706
3707 if ((c_seg = *(c_segment_t *)current_chead) == NULL)
3708 return;
3709
3710 assert(c_seg->c_state == C_IS_FILLING);
3711
3712 lck_mtx_lock_spin_always(&c_seg->c_lock);
3713
3714 c_current_seg_filled(c_seg, (c_segment_t *)current_chead);
3715
3716 lck_mtx_unlock_always(&c_seg->c_lock);
3717
3718 freezer_finished_filling++;
3719 }
3720
3721
3722 /*
3723 * This routine is used to transfer the compressed chunks from
3724 * the c_seg/cindx pointed to by slot_p into a new c_seg headed
3725 * by the current_chead and a new cindx within that c_seg.
3726 *
3727 * Currently, this routine is only used by the "freezer backed by
3728 * compressor with swap" mode to create a series of c_segs that
3729 * only contain compressed data belonging to one task. So, we
3730 * move a task's previously compressed data into a set of new
3731 * c_segs which will also hold the task's yet to be compressed data.
3732 */
3733
3734 kern_return_t
3735 vm_compressor_relocate(
3736 void **current_chead,
3737 int *slot_p)
3738 {
3739 c_slot_mapping_t slot_ptr;
3740 c_slot_mapping_t src_slot;
3741 uint32_t c_rounded_size;
3742 uint32_t c_size;
3743 uint16_t dst_slot;
3744 c_slot_t c_dst;
3745 c_slot_t c_src;
3746 int c_indx;
3747 c_segment_t c_seg_dst = NULL;
3748 c_segment_t c_seg_src = NULL;
3749 kern_return_t kr = KERN_SUCCESS;
3750
3751
3752 src_slot = (c_slot_mapping_t) slot_p;
3753
3754 if (src_slot->s_cseg == C_SV_CSEG_ID) {
3755 /*
3756 * no need to relocate... this is a page full of a single
3757 * value which is hashed to a single entry not contained
3758 * in a c_segment_t
3759 */
3760 return (kr);
3761 }
3762
3763 Relookup_dst:
3764 c_seg_dst = c_seg_allocate((c_segment_t *)current_chead);
3765 /*
3766 * returns with c_seg lock held
3767 * and PAGE_REPLACEMENT_DISALLOWED(TRUE)...
3768 * c_nextslot has been allocated and
3769 * c_store.c_buffer populated
3770 */
3771 if (c_seg_dst == NULL) {
3772 /*
3773 * Out of compression segments?
3774 */
3775 kr = KERN_RESOURCE_SHORTAGE;
3776 goto out;
3777 }
3778
3779 assert(c_seg_dst->c_busy == 0);
3780
3781 C_SEG_BUSY(c_seg_dst);
3782
3783 dst_slot = c_seg_dst->c_nextslot;
3784
3785 lck_mtx_unlock_always(&c_seg_dst->c_lock);
3786
3787 Relookup_src:
3788 c_seg_src = c_segments[src_slot->s_cseg - 1].c_seg;
3789
3790 assert(c_seg_dst != c_seg_src);
3791
3792 lck_mtx_lock_spin_always(&c_seg_src->c_lock);
3793
3794 if (C_SEG_IS_ONDISK(c_seg_src)) {
3795
3796 /*
3797 * A "thaw" can mark a process as eligible for
3798 * another freeze cycle without bringing any of
3799 * its swapped out c_segs back from disk (because
3800 * that is done on-demand).
3801 *
3802 * If the src c_seg we find for our pre-compressed
3803 * data is already on-disk, then we are dealing
3804 * with an app's data that is already packed and
3805 * swapped out. Don't do anything.
3806 */
3807
3808 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3809
3810 lck_mtx_unlock_always(&c_seg_src->c_lock);
3811
3812 c_seg_src = NULL;
3813
3814 goto out;
3815 }
3816
3817 if (c_seg_src->c_busy) {
3818
3819 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3820 c_seg_wait_on_busy(c_seg_src);
3821
3822 c_seg_src = NULL;
3823
3824 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3825
3826 goto Relookup_src;
3827 }
3828
3829 C_SEG_BUSY(c_seg_src);
3830
3831 lck_mtx_unlock_always(&c_seg_src->c_lock);
3832
3833 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3834
3835 /* find the c_slot */
3836 c_indx = src_slot->s_cindx;
3837
3838 c_src = C_SEG_SLOT_FROM_INDEX(c_seg_src, c_indx);
3839
3840 c_size = UNPACK_C_SIZE(c_src);
3841
3842 assert(c_size);
3843
3844 if (c_size > (uint32_t)(C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES((int32_t)c_seg_dst->c_nextoffset))) {
3845 /*
3846 * This segment is full. We need a new one.
3847 */
3848
3849 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3850
3851 lck_mtx_lock_spin_always(&c_seg_src->c_lock);
3852 C_SEG_WAKEUP_DONE(c_seg_src);
3853 lck_mtx_unlock_always(&c_seg_src->c_lock);
3854
3855 c_seg_src = NULL;
3856
3857 lck_mtx_lock_spin_always(&c_seg_dst->c_lock);
3858
3859 assert(c_seg_dst->c_busy);
3860 assert(c_seg_dst->c_state == C_IS_FILLING);
3861 assert(!c_seg_dst->c_on_minorcompact_q);
3862
3863 c_current_seg_filled(c_seg_dst, (c_segment_t *)current_chead);
3864 assert(*current_chead == NULL);
3865
3866 C_SEG_WAKEUP_DONE(c_seg_dst);
3867
3868 lck_mtx_unlock_always(&c_seg_dst->c_lock);
3869
3870 c_seg_dst = NULL;
3871
3872 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3873
3874 goto Relookup_dst;
3875 }
3876
3877 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, c_seg_dst->c_nextslot);
3878
3879 memcpy(&c_seg_dst->c_store.c_buffer[c_seg_dst->c_nextoffset], &c_seg_src->c_store.c_buffer[c_src->c_offset], c_size);
3880
3881 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
3882
3883 cslot_copy(c_dst, c_src);
3884 c_dst->c_offset = c_seg_dst->c_nextoffset;
3885
3886 if (c_seg_dst->c_firstemptyslot == c_seg_dst->c_nextslot)
3887 c_seg_dst->c_firstemptyslot++;
3888
3889 c_seg_dst->c_nextslot++;
3890 c_seg_dst->c_bytes_used += c_rounded_size;
3891 c_seg_dst->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
3892
3893
3894 PACK_C_SIZE(c_src, 0);
3895
3896 c_seg_src->c_bytes_used -= c_rounded_size;
3897 c_seg_src->c_bytes_unused += c_rounded_size;
3898
3899 if (c_indx < c_seg_src->c_firstemptyslot) {
3900 c_seg_src->c_firstemptyslot = c_indx;
3901 }
3902
3903 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, dst_slot);
3904
3905 PAGE_REPLACEMENT_ALLOWED(TRUE);
3906 slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
3907 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
3908 slot_ptr->s_cseg = c_seg_dst->c_mysegno + 1;
3909 slot_ptr->s_cindx = dst_slot;
3910
3911 PAGE_REPLACEMENT_ALLOWED(FALSE);
3912
3913 out:
3914 if (c_seg_src) {
3915
3916 lck_mtx_lock_spin_always(&c_seg_src->c_lock);
3917
3918 C_SEG_WAKEUP_DONE(c_seg_src);
3919
3920 if (c_seg_src->c_bytes_used == 0 && c_seg_src->c_state != C_IS_FILLING) {
3921 if (!c_seg_src->c_on_minorcompact_q)
3922 c_seg_need_delayed_compaction(c_seg_src, FALSE);
3923 }
3924
3925 lck_mtx_unlock_always(&c_seg_src->c_lock);
3926 }
3927
3928 if (c_seg_dst) {
3929
3930 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3931
3932 lck_mtx_lock_spin_always(&c_seg_dst->c_lock);
3933
3934 if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX_INDEX) {
3935 /*
3936 * Nearing or exceeded maximum slot and offset capacity.
3937 */
3938 assert(c_seg_dst->c_busy);
3939 assert(c_seg_dst->c_state == C_IS_FILLING);
3940 assert(!c_seg_dst->c_on_minorcompact_q);
3941
3942 c_current_seg_filled(c_seg_dst, (c_segment_t *)current_chead);
3943 assert(*current_chead == NULL);
3944 }
3945
3946 C_SEG_WAKEUP_DONE(c_seg_dst);
3947
3948 lck_mtx_unlock_always(&c_seg_dst->c_lock);
3949
3950 c_seg_dst = NULL;
3951
3952 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3953 }
3954
3955 return kr;
3956 }
3957 #endif /* CONFIG_FREEZE */
mirror of XNU