]> git.saurik.com Git - apple/xnu.git/blob - osfmk/vm/vm_compressor.c
6c5a42214b4e350824cd78201d2ee2f384b12980
[apple/xnu.git] / osfmk / vm / vm_compressor.c
1 /*
2 * Copyright (c) 2000-2013 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <vm/vm_compressor.h>
30
31 #if CONFIG_PHANTOM_CACHE
32 #include <vm/vm_phantom_cache.h>
33 #endif
34
35 #include <vm/vm_map.h>
36 #include <vm/vm_pageout.h>
37 #include <vm/memory_object.h>
38 #include <vm/vm_compressor_algorithms.h>
39 #include <vm/vm_fault.h>
40 #include <vm/vm_protos.h>
41 #include <mach/mach_host.h> /* for host_info() */
42 #include <kern/ledger.h>
43 #include <kern/policy_internal.h>
44 #include <kern/thread_group.h>
45 #include <san/kasan.h>
46
47 #if defined(__x86_64__)
48 #include <i386/misc_protos.h>
49 #endif
50 #if defined(__arm64__)
51 #include <arm/machine_routines.h>
52 #endif
53
54 #include <IOKit/IOHibernatePrivate.h>
55
56 extern boolean_t vm_darkwake_mode;
57
58 #if POPCOUNT_THE_COMPRESSED_DATA
59 boolean_t popcount_c_segs = TRUE;
60
61 static inline uint32_t
62 vmc_pop(uintptr_t ins, int sz)
63 {
64 uint32_t rv = 0;
65
66 if (__probable(popcount_c_segs == FALSE)) {
67 return 0xDEAD707C;
68 }
69
70 while (sz >= 16) {
71 uint32_t rv1, rv2;
72 uint64_t *ins64 = (uint64_t *) ins;
73 uint64_t *ins642 = (uint64_t *) (ins + 8);
74 rv1 = __builtin_popcountll(*ins64);
75 rv2 = __builtin_popcountll(*ins642);
76 rv += rv1 + rv2;
77 sz -= 16;
78 ins += 16;
79 }
80
81 while (sz >= 4) {
82 uint32_t *ins32 = (uint32_t *) ins;
83 rv += __builtin_popcount(*ins32);
84 sz -= 4;
85 ins += 4;
86 }
87
88 while (sz > 0) {
89 char *ins8 = (char *)ins;
90 rv += __builtin_popcount(*ins8);
91 sz--;
92 ins++;
93 }
94 return rv;
95 }
96 #endif
97
98 #if VALIDATE_C_SEGMENTS
99 boolean_t validate_c_segs = TRUE;
100 #endif
101 /*
102 * vm_compressor_mode has a heirarchy of control to set its value.
103 * boot-args are checked first, then device-tree, and finally
104 * the default value that is defined below. See vm_fault_init() for
105 * the boot-arg & device-tree code.
106 */
107
108 #if CONFIG_EMBEDDED
109
110 #if CONFIG_FREEZE
111 int vm_compressor_mode = VM_PAGER_FREEZER_DEFAULT;
112
113 void *freezer_chead; /* The chead used to track c_segs allocated for the exclusive use of holding just one task's compressed memory.*/
114 char *freezer_compressor_scratch_buf = NULL;
115
116 extern int c_freezer_swapout_page_count; /* This count keeps track of the # of compressed pages holding just one task's compressed memory on the swapout queue. This count is used during each freeze i.e. on a per-task basis.*/
117
118 #else /* CONFIG_FREEZE */
119 int vm_compressor_mode = VM_PAGER_NOT_CONFIGURED;
120 #endif /* CONFIG_FREEZE */
121
122 int vm_scale = 1;
123
124 #else /* CONFIG_EMBEDDED */
125 int vm_compressor_mode = VM_PAGER_COMPRESSOR_WITH_SWAP;
126 int vm_scale = 16;
127
128 #endif /* CONFIG_EMBEDDED */
129
130 int vm_compressor_is_active = 0;
131 int vm_compression_limit = 0;
132 int vm_compressor_available = 0;
133
134 extern void vm_pageout_io_throttle(void);
135
136 #if CHECKSUM_THE_DATA || CHECKSUM_THE_SWAP || CHECKSUM_THE_COMPRESSED_DATA
137 extern unsigned int hash_string(char *cp, int len);
138 static unsigned int vmc_hash(char *, int);
139 boolean_t checksum_c_segs = TRUE;
140
141 unsigned int
142 vmc_hash(char *cp, int len)
143 {
144 if (__probable(checksum_c_segs == FALSE)) {
145 return 0xDEAD7A37;
146 }
147 return hash_string(cp, len);
148 }
149 #endif
150
151 #define UNPACK_C_SIZE(cs) ((cs->c_size == (PAGE_SIZE-1)) ? PAGE_SIZE : cs->c_size)
152 #define PACK_C_SIZE(cs, size) (cs->c_size = ((size == PAGE_SIZE) ? PAGE_SIZE - 1 : size))
153
154
155 struct c_sv_hash_entry {
156 union {
157 struct {
158 uint32_t c_sv_he_ref;
159 uint32_t c_sv_he_data;
160 } c_sv_he;
161 uint64_t c_sv_he_record;
162 } c_sv_he_un;
163 };
164
165 #define he_ref c_sv_he_un.c_sv_he.c_sv_he_ref
166 #define he_data c_sv_he_un.c_sv_he.c_sv_he_data
167 #define he_record c_sv_he_un.c_sv_he_record
168
169 #define C_SV_HASH_MAX_MISS 32
170 #define C_SV_HASH_SIZE ((1 << 10))
171 #define C_SV_HASH_MASK ((1 << 10) - 1)
172 #define C_SV_CSEG_ID ((1 << 22) - 1)
173
174
175 union c_segu {
176 c_segment_t c_seg;
177 uintptr_t c_segno;
178 };
179
180
181
182 #define C_SLOT_PACK_PTR(ptr) (((uintptr_t)ptr - (uintptr_t) KERNEL_PMAP_HEAP_RANGE_START) >> 2)
183 #define C_SLOT_UNPACK_PTR(cslot) ((uintptr_t)(cslot->c_packed_ptr << 2) + (uintptr_t) KERNEL_PMAP_HEAP_RANGE_START)
184
185
186 uint32_t c_segment_count = 0;
187 uint32_t c_segment_count_max = 0;
188
189 uint64_t c_generation_id = 0;
190 uint64_t c_generation_id_flush_barrier;
191
192
193 #define HIBERNATE_FLUSHING_SECS_TO_COMPLETE 120
194
195 boolean_t hibernate_no_swapspace = FALSE;
196 clock_sec_t hibernate_flushing_deadline = 0;
197
198
199 #if RECORD_THE_COMPRESSED_DATA
200 char *c_compressed_record_sbuf;
201 char *c_compressed_record_ebuf;
202 char *c_compressed_record_cptr;
203 #endif
204
205
206 queue_head_t c_age_list_head;
207 queue_head_t c_swappedin_list_head;
208 queue_head_t c_swapout_list_head;
209 queue_head_t c_swapio_list_head;
210 queue_head_t c_swappedout_list_head;
211 queue_head_t c_swappedout_sparse_list_head;
212 queue_head_t c_major_list_head;
213 queue_head_t c_filling_list_head;
214 queue_head_t c_bad_list_head;
215
216 uint32_t c_age_count = 0;
217 uint32_t c_swappedin_count = 0;
218 uint32_t c_swapout_count = 0;
219 uint32_t c_swapio_count = 0;
220 uint32_t c_swappedout_count = 0;
221 uint32_t c_swappedout_sparse_count = 0;
222 uint32_t c_major_count = 0;
223 uint32_t c_filling_count = 0;
224 uint32_t c_empty_count = 0;
225 uint32_t c_bad_count = 0;
226
227
228 queue_head_t c_minor_list_head;
229 uint32_t c_minor_count = 0;
230
231 int c_overage_swapped_count = 0;
232 int c_overage_swapped_limit = 0;
233
234 int c_seg_fixed_array_len;
235 union c_segu *c_segments;
236 vm_offset_t c_buffers;
237 vm_size_t c_buffers_size;
238 caddr_t c_segments_next_page;
239 boolean_t c_segments_busy;
240 uint32_t c_segments_available;
241 uint32_t c_segments_limit;
242 uint32_t c_segments_nearing_limit;
243
244 uint32_t c_segment_svp_in_hash;
245 uint32_t c_segment_svp_hash_succeeded;
246 uint32_t c_segment_svp_hash_failed;
247 uint32_t c_segment_svp_zero_compressions;
248 uint32_t c_segment_svp_nonzero_compressions;
249 uint32_t c_segment_svp_zero_decompressions;
250 uint32_t c_segment_svp_nonzero_decompressions;
251
252 uint32_t c_segment_noncompressible_pages;
253
254 uint32_t c_segment_pages_compressed;
255 uint32_t c_segment_pages_compressed_limit;
256 uint32_t c_segment_pages_compressed_nearing_limit;
257 uint32_t c_free_segno_head = (uint32_t)-1;
258
259 uint32_t vm_compressor_minorcompact_threshold_divisor = 10;
260 uint32_t vm_compressor_majorcompact_threshold_divisor = 10;
261 uint32_t vm_compressor_unthrottle_threshold_divisor = 10;
262 uint32_t vm_compressor_catchup_threshold_divisor = 10;
263
264 uint32_t vm_compressor_minorcompact_threshold_divisor_overridden = 0;
265 uint32_t vm_compressor_majorcompact_threshold_divisor_overridden = 0;
266 uint32_t vm_compressor_unthrottle_threshold_divisor_overridden = 0;
267 uint32_t vm_compressor_catchup_threshold_divisor_overridden = 0;
268
269 #define C_SEGMENTS_PER_PAGE (PAGE_SIZE / sizeof(union c_segu))
270
271
272 lck_grp_attr_t vm_compressor_lck_grp_attr;
273 lck_attr_t vm_compressor_lck_attr;
274 lck_grp_t vm_compressor_lck_grp;
275 lck_mtx_t *c_list_lock;
276 lck_rw_t c_master_lock;
277 boolean_t decompressions_blocked = FALSE;
278
279 zone_t compressor_segment_zone;
280 int c_compressor_swap_trigger = 0;
281
282 uint32_t compressor_cpus;
283 char *compressor_scratch_bufs;
284 char *kdp_compressor_scratch_buf;
285 char *kdp_compressor_decompressed_page;
286 addr64_t kdp_compressor_decompressed_page_paddr;
287 ppnum_t kdp_compressor_decompressed_page_ppnum;
288
289 clock_sec_t start_of_sample_period_sec = 0;
290 clock_nsec_t start_of_sample_period_nsec = 0;
291 clock_sec_t start_of_eval_period_sec = 0;
292 clock_nsec_t start_of_eval_period_nsec = 0;
293 uint32_t sample_period_decompression_count = 0;
294 uint32_t sample_period_compression_count = 0;
295 uint32_t last_eval_decompression_count = 0;
296 uint32_t last_eval_compression_count = 0;
297
298 #define DECOMPRESSION_SAMPLE_MAX_AGE (60 * 30)
299
300 boolean_t vm_swapout_ripe_segments = FALSE;
301 uint32_t vm_ripe_target_age = (60 * 60 * 48);
302
303 uint32_t swapout_target_age = 0;
304 uint32_t age_of_decompressions_during_sample_period[DECOMPRESSION_SAMPLE_MAX_AGE];
305 uint32_t overage_decompressions_during_sample_period = 0;
306
307
308 void do_fastwake_warmup(queue_head_t *, boolean_t);
309 boolean_t fastwake_warmup = FALSE;
310 boolean_t fastwake_recording_in_progress = FALSE;
311 clock_sec_t dont_trim_until_ts = 0;
312
313 uint64_t c_segment_warmup_count;
314 uint64_t first_c_segment_to_warm_generation_id = 0;
315 uint64_t last_c_segment_to_warm_generation_id = 0;
316 boolean_t hibernate_flushing = FALSE;
317
318 int64_t c_segment_input_bytes __attribute__((aligned(8))) = 0;
319 int64_t c_segment_compressed_bytes __attribute__((aligned(8))) = 0;
320 int64_t compressor_bytes_used __attribute__((aligned(8))) = 0;
321
322
323 struct c_sv_hash_entry c_segment_sv_hash_table[C_SV_HASH_SIZE] __attribute__ ((aligned(8)));
324
325 static boolean_t compressor_needs_to_swap(void);
326 static void vm_compressor_swap_trigger_thread(void);
327 static void vm_compressor_do_delayed_compactions(boolean_t);
328 static void vm_compressor_compact_and_swap(boolean_t);
329 static void vm_compressor_age_swapped_in_segments(boolean_t);
330
331 #if !CONFIG_EMBEDDED
332 static void vm_compressor_take_paging_space_action(void);
333 #endif
334
335 void compute_swapout_target_age(void);
336
337 boolean_t c_seg_major_compact(c_segment_t, c_segment_t);
338 boolean_t c_seg_major_compact_ok(c_segment_t, c_segment_t);
339
340 int c_seg_minor_compaction_and_unlock(c_segment_t, boolean_t);
341 int c_seg_do_minor_compaction_and_unlock(c_segment_t, boolean_t, boolean_t, boolean_t);
342 void c_seg_try_minor_compaction_and_unlock(c_segment_t c_seg);
343
344 void c_seg_move_to_sparse_list(c_segment_t);
345 void c_seg_insert_into_q(queue_head_t *, c_segment_t);
346
347 uint64_t vm_available_memory(void);
348 uint64_t vm_compressor_pages_compressed(void);
349
350 /*
351 * indicate the need to do a major compaction if
352 * the overall set of in-use compression segments
353 * becomes sparse... on systems that support pressure
354 * driven swapping, this will also cause swapouts to
355 * be initiated.
356 */
357 static inline boolean_t
358 vm_compressor_needs_to_major_compact()
359 {
360 uint32_t incore_seg_count;
361
362 incore_seg_count = c_segment_count - c_swappedout_count - c_swappedout_sparse_count;
363
364 if ((c_segment_count >= (c_segments_nearing_limit / 8)) &&
365 ((incore_seg_count * C_SEG_MAX_PAGES) - VM_PAGE_COMPRESSOR_COUNT) >
366 ((incore_seg_count / 8) * C_SEG_MAX_PAGES)) {
367 return 1;
368 }
369 return 0;
370 }
371
372
373 uint64_t
374 vm_available_memory(void)
375 {
376 return ((uint64_t)AVAILABLE_NON_COMPRESSED_MEMORY) * PAGE_SIZE_64;
377 }
378
379
380 uint64_t
381 vm_compressor_pages_compressed(void)
382 {
383 return c_segment_pages_compressed * PAGE_SIZE_64;
384 }
385
386
387 boolean_t
388 vm_compressor_low_on_space(void)
389 {
390 if ((c_segment_pages_compressed > c_segment_pages_compressed_nearing_limit) ||
391 (c_segment_count > c_segments_nearing_limit)) {
392 return TRUE;
393 }
394
395 return FALSE;
396 }
397
398
399 boolean_t
400 vm_compressor_out_of_space(void)
401 {
402 if ((c_segment_pages_compressed >= c_segment_pages_compressed_limit) ||
403 (c_segment_count >= c_segments_limit)) {
404 return TRUE;
405 }
406
407 return FALSE;
408 }
409
410
411 int
412 vm_wants_task_throttled(task_t task)
413 {
414 if (task == kernel_task) {
415 return 0;
416 }
417
418 if (VM_CONFIG_SWAP_IS_ACTIVE) {
419 if ((vm_compressor_low_on_space() || HARD_THROTTLE_LIMIT_REACHED()) &&
420 (unsigned int)pmap_compressed(task->map->pmap) > (c_segment_pages_compressed / 4)) {
421 return 1;
422 }
423 }
424 return 0;
425 }
426
427
428 #if DEVELOPMENT || DEBUG
429 boolean_t kill_on_no_paging_space = FALSE; /* On compressor/swap exhaustion, kill the largest process regardless of
430 * its chosen process policy. Controlled by a boot-arg of the same name. */
431 #endif /* DEVELOPMENT || DEBUG */
432
433 #if !CONFIG_EMBEDDED
434
435 static uint32_t no_paging_space_action_in_progress = 0;
436 extern void memorystatus_send_low_swap_note(void);
437
438 static void
439 vm_compressor_take_paging_space_action(void)
440 {
441 if (no_paging_space_action_in_progress == 0) {
442 if (OSCompareAndSwap(0, 1, (UInt32 *)&no_paging_space_action_in_progress)) {
443 if (no_paging_space_action()) {
444 #if DEVELOPMENT || DEBUG
445 if (kill_on_no_paging_space == TRUE) {
446 /*
447 * Since we are choosing to always kill a process, we don't need the
448 * "out of application memory" dialog box in this mode. And, hence we won't
449 * send the knote.
450 */
451 no_paging_space_action_in_progress = 0;
452 return;
453 }
454 #endif /* DEVELOPMENT || DEBUG */
455 memorystatus_send_low_swap_note();
456 }
457
458 no_paging_space_action_in_progress = 0;
459 }
460 }
461 }
462 #endif /* !CONFIG_EMBEDDED */
463
464
465 void
466 vm_compressor_init_locks(void)
467 {
468 lck_grp_attr_setdefault(&vm_compressor_lck_grp_attr);
469 lck_grp_init(&vm_compressor_lck_grp, "vm_compressor", &vm_compressor_lck_grp_attr);
470 lck_attr_setdefault(&vm_compressor_lck_attr);
471
472 lck_rw_init(&c_master_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
473 }
474
475
476 void
477 vm_decompressor_lock(void)
478 {
479 PAGE_REPLACEMENT_ALLOWED(TRUE);
480
481 decompressions_blocked = TRUE;
482
483 PAGE_REPLACEMENT_ALLOWED(FALSE);
484 }
485
486 void
487 vm_decompressor_unlock(void)
488 {
489 PAGE_REPLACEMENT_ALLOWED(TRUE);
490
491 decompressions_blocked = FALSE;
492
493 PAGE_REPLACEMENT_ALLOWED(FALSE);
494
495 thread_wakeup((event_t)&decompressions_blocked);
496 }
497
498 static inline void
499 cslot_copy(c_slot_t cdst, c_slot_t csrc)
500 {
501 #if CHECKSUM_THE_DATA
502 cdst->c_hash_data = csrc->c_hash_data;
503 #endif
504 #if CHECKSUM_THE_COMPRESSED_DATA
505 cdst->c_hash_compressed_data = csrc->c_hash_compressed_data;
506 #endif
507 #if POPCOUNT_THE_COMPRESSED_DATA
508 cdst->c_pop_cdata = csrc->c_pop_cdata;
509 #endif
510 cdst->c_size = csrc->c_size;
511 cdst->c_packed_ptr = csrc->c_packed_ptr;
512 #if defined(__arm__) || defined(__arm64__)
513 cdst->c_codec = csrc->c_codec;
514 #endif
515 }
516
517 vm_map_t compressor_map;
518 uint64_t compressor_pool_max_size;
519 uint64_t compressor_pool_size;
520 uint32_t compressor_pool_multiplier;
521
522 #if DEVELOPMENT || DEBUG
523 /*
524 * Compressor segments are write-protected in development/debug
525 * kernels to help debug memory corruption.
526 * In cases where performance is a concern, this can be disabled
527 * via the boot-arg "-disable_cseg_write_protection".
528 */
529 boolean_t write_protect_c_segs = TRUE;
530 int vm_compressor_test_seg_wp;
531 uint32_t vm_ktrace_enabled;
532 #endif /* DEVELOPMENT || DEBUG */
533
534 void
535 vm_compressor_init(void)
536 {
537 thread_t thread;
538 struct c_slot cs_dummy;
539 c_slot_t cs = &cs_dummy;
540 int c_segment_min_size;
541 int c_segment_padded_size;
542 int attempts = 1;
543 kern_return_t retval = KERN_SUCCESS;
544 vm_offset_t start_addr = 0;
545 vm_size_t c_segments_arr_size = 0, compressor_submap_size = 0;
546 vm_map_kernel_flags_t vmk_flags;
547 #if RECORD_THE_COMPRESSED_DATA
548 vm_size_t c_compressed_record_sbuf_size = 0;
549 #endif /* RECORD_THE_COMPRESSED_DATA */
550
551 #if DEVELOPMENT || DEBUG
552 char bootarg_name[32];
553 if (PE_parse_boot_argn("-kill_on_no_paging_space", bootarg_name, sizeof(bootarg_name))) {
554 kill_on_no_paging_space = TRUE;
555 }
556 if (PE_parse_boot_argn("-disable_cseg_write_protection", bootarg_name, sizeof(bootarg_name))) {
557 write_protect_c_segs = FALSE;
558 }
559 int vmcval = 1;
560 PE_parse_boot_argn("vm_compressor_validation", &vmcval, sizeof(vmcval));
561
562 if (kern_feature_override(KF_COMPRSV_OVRD)) {
563 vmcval = 0;
564 }
565 if (vmcval == 0) {
566 #if POPCOUNT_THE_COMPRESSED_DATA
567 popcount_c_segs = FALSE;
568 #endif
569 #if CHECKSUM_THE_DATA || CHECKSUM_THE_COMPRESSED_DATA
570 checksum_c_segs = FALSE;
571 #endif
572 #if VALIDATE_C_SEGMENTS
573 validate_c_segs = FALSE;
574 #endif
575 write_protect_c_segs = FALSE;
576 }
577 #endif /* DEVELOPMENT || DEBUG */
578
579 /*
580 * ensure that any pointer that gets created from
581 * the vm_page zone can be packed properly
582 */
583 cs->c_packed_ptr = C_SLOT_PACK_PTR(zone_map_min_address);
584
585 if (C_SLOT_UNPACK_PTR(cs) != (uintptr_t)zone_map_min_address) {
586 panic("C_SLOT_UNPACK_PTR failed on zone_map_min_address - %p", (void *)zone_map_min_address);
587 }
588
589 cs->c_packed_ptr = C_SLOT_PACK_PTR(zone_map_max_address);
590
591 if (C_SLOT_UNPACK_PTR(cs) != (uintptr_t)zone_map_max_address) {
592 panic("C_SLOT_UNPACK_PTR failed on zone_map_max_address - %p", (void *)zone_map_max_address);
593 }
594
595
596 assert((C_SEGMENTS_PER_PAGE * sizeof(union c_segu)) == PAGE_SIZE);
597
598 PE_parse_boot_argn("vm_compression_limit", &vm_compression_limit, sizeof(vm_compression_limit));
599
600 #ifdef CONFIG_EMBEDDED
601 #if XNU_TARGET_OS_WATCH
602 // rdar://problem/51012698
603 vm_compressor_minorcompact_threshold_divisor = 40;
604 #else
605 vm_compressor_minorcompact_threshold_divisor = 20;
606 #endif
607 vm_compressor_majorcompact_threshold_divisor = 30;
608 vm_compressor_unthrottle_threshold_divisor = 40;
609 vm_compressor_catchup_threshold_divisor = 60;
610 #else
611 if (max_mem <= (3ULL * 1024ULL * 1024ULL * 1024ULL)) {
612 vm_compressor_minorcompact_threshold_divisor = 11;
613 vm_compressor_majorcompact_threshold_divisor = 13;
614 vm_compressor_unthrottle_threshold_divisor = 20;
615 vm_compressor_catchup_threshold_divisor = 35;
616 } else {
617 vm_compressor_minorcompact_threshold_divisor = 20;
618 vm_compressor_majorcompact_threshold_divisor = 25;
619 vm_compressor_unthrottle_threshold_divisor = 35;
620 vm_compressor_catchup_threshold_divisor = 50;
621 }
622 #endif
623 /*
624 * vm_page_init_lck_grp is now responsible for calling vm_compressor_init_locks
625 * c_master_lock needs to be available early so that "vm_page_find_contiguous" can
626 * use PAGE_REPLACEMENT_ALLOWED to coordinate with the compressor.
627 */
628
629 c_list_lock = lck_mtx_alloc_init(&vm_compressor_lck_grp, &vm_compressor_lck_attr);
630
631 queue_init(&c_bad_list_head);
632 queue_init(&c_age_list_head);
633 queue_init(&c_minor_list_head);
634 queue_init(&c_major_list_head);
635 queue_init(&c_filling_list_head);
636 queue_init(&c_swapout_list_head);
637 queue_init(&c_swapio_list_head);
638 queue_init(&c_swappedin_list_head);
639 queue_init(&c_swappedout_list_head);
640 queue_init(&c_swappedout_sparse_list_head);
641
642 c_free_segno_head = -1;
643 c_segments_available = 0;
644
645 if (vm_compression_limit) {
646 compressor_pool_size = (uint64_t)vm_compression_limit * PAGE_SIZE_64;
647 }
648
649 compressor_pool_max_size = C_SEG_MAX_LIMIT;
650 compressor_pool_max_size *= C_SEG_BUFSIZE;
651
652 #if !CONFIG_EMBEDDED
653
654 if (vm_compression_limit == 0) {
655 if (max_mem <= (4ULL * 1024ULL * 1024ULL * 1024ULL)) {
656 compressor_pool_size = 16ULL * max_mem;
657 } else if (max_mem <= (8ULL * 1024ULL * 1024ULL * 1024ULL)) {
658 compressor_pool_size = 8ULL * max_mem;
659 } else if (max_mem <= (32ULL * 1024ULL * 1024ULL * 1024ULL)) {
660 compressor_pool_size = 4ULL * max_mem;
661 } else {
662 compressor_pool_size = 2ULL * max_mem;
663 }
664 }
665 if (max_mem <= (8ULL * 1024ULL * 1024ULL * 1024ULL)) {
666 compressor_pool_multiplier = 1;
667 } else if (max_mem <= (32ULL * 1024ULL * 1024ULL * 1024ULL)) {
668 compressor_pool_multiplier = 2;
669 } else {
670 compressor_pool_multiplier = 4;
671 }
672
673 #elif defined(__arm__)
674
675 #define VM_RESERVE_SIZE (1024 * 1024 * 256)
676 #define MAX_COMPRESSOR_POOL_SIZE (1024 * 1024 * 450)
677
678 if (compressor_pool_max_size > MAX_COMPRESSOR_POOL_SIZE) {
679 compressor_pool_max_size = MAX_COMPRESSOR_POOL_SIZE;
680 }
681
682 if (vm_compression_limit == 0) {
683 compressor_pool_size = ((kernel_map->max_offset - kernel_map->min_offset) - kernel_map->size) - VM_RESERVE_SIZE;
684 }
685 compressor_pool_multiplier = 1;
686 #else
687 if (compressor_pool_max_size > max_mem) {
688 compressor_pool_max_size = max_mem;
689 }
690
691 if (vm_compression_limit == 0) {
692 compressor_pool_size = max_mem;
693 }
694 compressor_pool_multiplier = 1;
695 #endif
696 if (compressor_pool_size > compressor_pool_max_size) {
697 compressor_pool_size = compressor_pool_max_size;
698 }
699
700 try_again:
701 c_segments_limit = (uint32_t)(compressor_pool_size / (vm_size_t)(C_SEG_ALLOCSIZE));
702 c_segments_nearing_limit = (uint32_t)(((uint64_t)c_segments_limit * 98ULL) / 100ULL);
703
704 c_segment_pages_compressed_limit = (c_segments_limit * (C_SEG_BUFSIZE / PAGE_SIZE) * compressor_pool_multiplier);
705
706 if (c_segment_pages_compressed_limit < (uint32_t)(max_mem / PAGE_SIZE)) {
707 c_segment_pages_compressed_limit = (uint32_t)(max_mem / PAGE_SIZE);
708 }
709
710 c_segment_pages_compressed_nearing_limit = (uint32_t)(((uint64_t)c_segment_pages_compressed_limit * 98ULL) / 100ULL);
711
712 /*
713 * Submap needs space for:
714 * - c_segments
715 * - c_buffers
716 * - swap reclaimations -- C_SEG_BUFSIZE
717 */
718 c_segments_arr_size = vm_map_round_page((sizeof(union c_segu) * c_segments_limit), VM_MAP_PAGE_MASK(kernel_map));
719 c_buffers_size = vm_map_round_page(((vm_size_t)C_SEG_ALLOCSIZE * (vm_size_t)c_segments_limit), VM_MAP_PAGE_MASK(kernel_map));
720
721 compressor_submap_size = c_segments_arr_size + c_buffers_size + C_SEG_BUFSIZE;
722
723 #if RECORD_THE_COMPRESSED_DATA
724 c_compressed_record_sbuf_size = (vm_size_t)C_SEG_ALLOCSIZE + (PAGE_SIZE * 2);
725 compressor_submap_size += c_compressed_record_sbuf_size;
726 #endif /* RECORD_THE_COMPRESSED_DATA */
727
728 vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
729 vmk_flags.vmkf_permanent = TRUE;
730 retval = kmem_suballoc(kernel_map, &start_addr, compressor_submap_size,
731 FALSE, VM_FLAGS_ANYWHERE, vmk_flags, VM_KERN_MEMORY_COMPRESSOR,
732 &compressor_map);
733
734 if (retval != KERN_SUCCESS) {
735 if (++attempts > 3) {
736 panic("vm_compressor_init: kmem_suballoc failed - 0x%llx", (uint64_t)compressor_submap_size);
737 }
738
739 compressor_pool_size = compressor_pool_size / 2;
740
741 kprintf("retrying creation of the compressor submap at 0x%llx bytes\n", compressor_pool_size);
742 goto try_again;
743 }
744 if (kernel_memory_allocate(compressor_map, (vm_offset_t *)(&c_segments), (sizeof(union c_segu) * c_segments_limit), 0, KMA_KOBJECT | KMA_VAONLY | KMA_PERMANENT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS) {
745 panic("vm_compressor_init: kernel_memory_allocate failed - c_segments\n");
746 }
747 if (kernel_memory_allocate(compressor_map, &c_buffers, c_buffers_size, 0, KMA_COMPRESSOR | KMA_VAONLY | KMA_PERMANENT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS) {
748 panic("vm_compressor_init: kernel_memory_allocate failed - c_buffers\n");
749 }
750
751
752 c_segment_min_size = sizeof(struct c_segment) + (C_SEG_SLOT_VAR_ARRAY_MIN_LEN * sizeof(struct c_slot));
753
754 for (c_segment_padded_size = 128; c_segment_padded_size < c_segment_min_size; c_segment_padded_size = c_segment_padded_size << 1) {
755 ;
756 }
757
758 compressor_segment_zone = zinit(c_segment_padded_size, c_segments_limit * c_segment_padded_size, PAGE_SIZE, "compressor_segment");
759 zone_change(compressor_segment_zone, Z_CALLERACCT, FALSE);
760 zone_change(compressor_segment_zone, Z_NOENCRYPT, TRUE);
761
762 c_seg_fixed_array_len = (c_segment_padded_size - sizeof(struct c_segment)) / sizeof(struct c_slot);
763
764 c_segments_busy = FALSE;
765
766 c_segments_next_page = (caddr_t)c_segments;
767 vm_compressor_algorithm_init();
768
769 {
770 host_basic_info_data_t hinfo;
771 mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
772
773 #define BSD_HOST 1
774 host_info((host_t)BSD_HOST, HOST_BASIC_INFO, (host_info_t)&hinfo, &count);
775
776 compressor_cpus = hinfo.max_cpus;
777 compressor_scratch_bufs = kalloc_tag(compressor_cpus * vm_compressor_get_decode_scratch_size(), VM_KERN_MEMORY_COMPRESSOR);
778
779 kdp_compressor_scratch_buf = kalloc_tag(vm_compressor_get_decode_scratch_size(), VM_KERN_MEMORY_COMPRESSOR);
780
781 /*
782 * kdp_compressor_decompressed_page must be page aligned because we access
783 * it through the physical apperture by page number. kalloc() does not
784 * guarantee alignment.
785 */
786 vm_offset_t addr;
787 if (kernel_memory_allocate(kernel_map, &addr, PAGE_SIZE, 0, KMA_KOBJECT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS) {
788 panic("vm_compressor_init: kernel_memory_allocate failed - kdp_compressor_decompressed_page\n");
789 }
790 assert((addr & PAGE_MASK) == 0);
791 kdp_compressor_decompressed_page = (void *)addr;
792 kdp_compressor_decompressed_page_paddr = kvtophys((vm_offset_t)kdp_compressor_decompressed_page);
793 kdp_compressor_decompressed_page_ppnum = (ppnum_t) atop(kdp_compressor_decompressed_page_paddr);
794 }
795 #if CONFIG_FREEZE
796 freezer_compressor_scratch_buf = kalloc_tag(vm_compressor_get_encode_scratch_size(), VM_KERN_MEMORY_COMPRESSOR);
797 #endif
798
799 #if RECORD_THE_COMPRESSED_DATA
800 if (kernel_memory_allocate(compressor_map, (vm_offset_t *)&c_compressed_record_sbuf, c_compressed_record_sbuf_size, 0, KMA_KOBJECT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS) {
801 panic("vm_compressor_init: kernel_memory_allocate failed - c_compressed_record_sbuf\n");
802 }
803
804 c_compressed_record_cptr = c_compressed_record_sbuf;
805 c_compressed_record_ebuf = c_compressed_record_sbuf + c_compressed_record_sbuf_size;
806 #endif
807
808 if (kernel_thread_start_priority((thread_continue_t)vm_compressor_swap_trigger_thread, NULL,
809 BASEPRI_VM, &thread) != KERN_SUCCESS) {
810 panic("vm_compressor_swap_trigger_thread: create failed");
811 }
812 thread_deallocate(thread);
813
814 if (vm_pageout_internal_start() != KERN_SUCCESS) {
815 panic("vm_compressor_init: Failed to start the internal pageout thread.\n");
816 }
817 if (VM_CONFIG_SWAP_IS_PRESENT) {
818 vm_compressor_swap_init();
819 }
820
821 if (VM_CONFIG_COMPRESSOR_IS_ACTIVE) {
822 vm_compressor_is_active = 1;
823 }
824
825 #if CONFIG_FREEZE
826 memorystatus_freeze_enabled = TRUE;
827 #endif /* CONFIG_FREEZE */
828
829 vm_compressor_available = 1;
830
831 vm_page_reactivate_all_throttled();
832 }
833
834
835 #if VALIDATE_C_SEGMENTS
836
837 static void
838 c_seg_validate(c_segment_t c_seg, boolean_t must_be_compact)
839 {
840 int c_indx;
841 int32_t bytes_used;
842 uint32_t c_rounded_size;
843 uint32_t c_size;
844 c_slot_t cs;
845
846 if (__probable(validate_c_segs == FALSE)) {
847 return;
848 }
849 if (c_seg->c_firstemptyslot < c_seg->c_nextslot) {
850 c_indx = c_seg->c_firstemptyslot;
851 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
852
853 if (cs == NULL) {
854 panic("c_seg_validate: no slot backing c_firstemptyslot");
855 }
856
857 if (cs->c_size) {
858 panic("c_seg_validate: c_firstemptyslot has non-zero size (%d)\n", cs->c_size);
859 }
860 }
861 bytes_used = 0;
862
863 for (c_indx = 0; c_indx < c_seg->c_nextslot; c_indx++) {
864 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
865
866 c_size = UNPACK_C_SIZE(cs);
867
868 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
869
870 bytes_used += c_rounded_size;
871
872 #if CHECKSUM_THE_COMPRESSED_DATA
873 unsigned csvhash;
874 if (c_size && cs->c_hash_compressed_data != (csvhash = vmc_hash((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size))) {
875 addr64_t csvphys = kvtophys((vm_offset_t)&c_seg->c_store.c_buffer[cs->c_offset]);
876 panic("Compressed data doesn't match original %p phys: 0x%llx %d %p %d %d 0x%x 0x%x", c_seg, csvphys, cs->c_offset, cs, c_indx, c_size, cs->c_hash_compressed_data, csvhash);
877 }
878 #endif
879 #if POPCOUNT_THE_COMPRESSED_DATA
880 unsigned csvpop;
881 if (c_size) {
882 uintptr_t csvaddr = (uintptr_t) &c_seg->c_store.c_buffer[cs->c_offset];
883 if (cs->c_pop_cdata != (csvpop = vmc_pop(csvaddr, c_size))) {
884 panic("Compressed data popcount doesn't match original, bit distance: %d %p (phys: %p) %p %p 0x%llx 0x%x 0x%x 0x%x", (csvpop - cs->c_pop_cdata), (void *)csvaddr, (void *) kvtophys(csvaddr), c_seg, cs, (uint64_t)cs->c_offset, c_size, csvpop, cs->c_pop_cdata);
885 }
886 }
887 #endif
888 }
889
890 if (bytes_used != c_seg->c_bytes_used) {
891 panic("c_seg_validate: bytes_used mismatch - found %d, segment has %d\n", bytes_used, c_seg->c_bytes_used);
892 }
893
894 if (c_seg->c_bytes_used > C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset)) {
895 panic("c_seg_validate: c_bytes_used > c_nextoffset - c_nextoffset = %d, c_bytes_used = %d\n",
896 (int32_t)C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset), c_seg->c_bytes_used);
897 }
898
899 if (must_be_compact) {
900 if (c_seg->c_bytes_used != C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset)) {
901 panic("c_seg_validate: c_bytes_used doesn't match c_nextoffset - c_nextoffset = %d, c_bytes_used = %d\n",
902 (int32_t)C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset), c_seg->c_bytes_used);
903 }
904 }
905 }
906
907 #endif
908
909
910 void
911 c_seg_need_delayed_compaction(c_segment_t c_seg, boolean_t c_list_lock_held)
912 {
913 boolean_t clear_busy = FALSE;
914
915 if (c_list_lock_held == FALSE) {
916 if (!lck_mtx_try_lock_spin_always(c_list_lock)) {
917 C_SEG_BUSY(c_seg);
918
919 lck_mtx_unlock_always(&c_seg->c_lock);
920 lck_mtx_lock_spin_always(c_list_lock);
921 lck_mtx_lock_spin_always(&c_seg->c_lock);
922
923 clear_busy = TRUE;
924 }
925 }
926 assert(c_seg->c_state != C_IS_FILLING);
927
928 if (!c_seg->c_on_minorcompact_q && !(C_SEG_IS_ON_DISK_OR_SOQ(c_seg))) {
929 queue_enter(&c_minor_list_head, c_seg, c_segment_t, c_list);
930 c_seg->c_on_minorcompact_q = 1;
931 c_minor_count++;
932 }
933 if (c_list_lock_held == FALSE) {
934 lck_mtx_unlock_always(c_list_lock);
935 }
936
937 if (clear_busy == TRUE) {
938 C_SEG_WAKEUP_DONE(c_seg);
939 }
940 }
941
942
943 unsigned int c_seg_moved_to_sparse_list = 0;
944
945 void
946 c_seg_move_to_sparse_list(c_segment_t c_seg)
947 {
948 boolean_t clear_busy = FALSE;
949
950 if (!lck_mtx_try_lock_spin_always(c_list_lock)) {
951 C_SEG_BUSY(c_seg);
952
953 lck_mtx_unlock_always(&c_seg->c_lock);
954 lck_mtx_lock_spin_always(c_list_lock);
955 lck_mtx_lock_spin_always(&c_seg->c_lock);
956
957 clear_busy = TRUE;
958 }
959 c_seg_switch_state(c_seg, C_ON_SWAPPEDOUTSPARSE_Q, FALSE);
960
961 c_seg_moved_to_sparse_list++;
962
963 lck_mtx_unlock_always(c_list_lock);
964
965 if (clear_busy == TRUE) {
966 C_SEG_WAKEUP_DONE(c_seg);
967 }
968 }
969
970
971 void
972 c_seg_insert_into_q(queue_head_t *qhead, c_segment_t c_seg)
973 {
974 c_segment_t c_seg_next;
975
976 if (queue_empty(qhead)) {
977 queue_enter(qhead, c_seg, c_segment_t, c_age_list);
978 } else {
979 c_seg_next = (c_segment_t)queue_first(qhead);
980
981 while (TRUE) {
982 if (c_seg->c_generation_id < c_seg_next->c_generation_id) {
983 queue_insert_before(qhead, c_seg, c_seg_next, c_segment_t, c_age_list);
984 break;
985 }
986 c_seg_next = (c_segment_t) queue_next(&c_seg_next->c_age_list);
987
988 if (queue_end(qhead, (queue_entry_t) c_seg_next)) {
989 queue_enter(qhead, c_seg, c_segment_t, c_age_list);
990 break;
991 }
992 }
993 }
994 }
995
996
997 int try_minor_compaction_failed = 0;
998 int try_minor_compaction_succeeded = 0;
999
1000 void
1001 c_seg_try_minor_compaction_and_unlock(c_segment_t c_seg)
1002 {
1003 assert(c_seg->c_on_minorcompact_q);
1004 /*
1005 * c_seg is currently on the delayed minor compaction
1006 * queue and we have c_seg locked... if we can get the
1007 * c_list_lock w/o blocking (if we blocked we could deadlock
1008 * because the lock order is c_list_lock then c_seg's lock)
1009 * we'll pull it from the delayed list and free it directly
1010 */
1011 if (!lck_mtx_try_lock_spin_always(c_list_lock)) {
1012 /*
1013 * c_list_lock is held, we need to bail
1014 */
1015 try_minor_compaction_failed++;
1016
1017 lck_mtx_unlock_always(&c_seg->c_lock);
1018 } else {
1019 try_minor_compaction_succeeded++;
1020
1021 C_SEG_BUSY(c_seg);
1022 c_seg_do_minor_compaction_and_unlock(c_seg, TRUE, FALSE, FALSE);
1023 }
1024 }
1025
1026
1027 int
1028 c_seg_do_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy, boolean_t need_list_lock, boolean_t disallow_page_replacement)
1029 {
1030 int c_seg_freed;
1031
1032 assert(c_seg->c_busy);
1033 assert(!C_SEG_IS_ON_DISK_OR_SOQ(c_seg));
1034
1035 /*
1036 * check for the case that can occur when we are not swapping
1037 * and this segment has been major compacted in the past
1038 * and moved to the majorcompact q to remove it from further
1039 * consideration... if the occupancy falls too low we need
1040 * to put it back on the age_q so that it will be considered
1041 * in the next major compaction sweep... if we don't do this
1042 * we will eventually run into the c_segments_limit
1043 */
1044 if (c_seg->c_state == C_ON_MAJORCOMPACT_Q && C_SEG_SHOULD_MAJORCOMPACT_NOW(c_seg)) {
1045 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
1046 }
1047 if (!c_seg->c_on_minorcompact_q) {
1048 if (clear_busy == TRUE) {
1049 C_SEG_WAKEUP_DONE(c_seg);
1050 }
1051
1052 lck_mtx_unlock_always(&c_seg->c_lock);
1053
1054 return 0;
1055 }
1056 queue_remove(&c_minor_list_head, c_seg, c_segment_t, c_list);
1057 c_seg->c_on_minorcompact_q = 0;
1058 c_minor_count--;
1059
1060 lck_mtx_unlock_always(c_list_lock);
1061
1062 if (disallow_page_replacement == TRUE) {
1063 lck_mtx_unlock_always(&c_seg->c_lock);
1064
1065 PAGE_REPLACEMENT_DISALLOWED(TRUE);
1066
1067 lck_mtx_lock_spin_always(&c_seg->c_lock);
1068 }
1069 c_seg_freed = c_seg_minor_compaction_and_unlock(c_seg, clear_busy);
1070
1071 if (disallow_page_replacement == TRUE) {
1072 PAGE_REPLACEMENT_DISALLOWED(FALSE);
1073 }
1074
1075 if (need_list_lock == TRUE) {
1076 lck_mtx_lock_spin_always(c_list_lock);
1077 }
1078
1079 return c_seg_freed;
1080 }
1081
1082
1083 void
1084 c_seg_wait_on_busy(c_segment_t c_seg)
1085 {
1086 c_seg->c_wanted = 1;
1087 assert_wait((event_t) (c_seg), THREAD_UNINT);
1088
1089 lck_mtx_unlock_always(&c_seg->c_lock);
1090 thread_block(THREAD_CONTINUE_NULL);
1091 }
1092
1093
1094 void
1095 c_seg_switch_state(c_segment_t c_seg, int new_state, boolean_t insert_head)
1096 {
1097 int old_state = c_seg->c_state;
1098
1099 #if !CONFIG_EMBEDDED
1100 #if DEVELOPMENT || DEBUG
1101 if (new_state != C_IS_FILLING) {
1102 LCK_MTX_ASSERT(&c_seg->c_lock, LCK_MTX_ASSERT_OWNED);
1103 }
1104 LCK_MTX_ASSERT(c_list_lock, LCK_MTX_ASSERT_OWNED);
1105 #endif
1106 #endif /* !CONFIG_EMBEDDED */
1107 switch (old_state) {
1108 case C_IS_EMPTY:
1109 assert(new_state == C_IS_FILLING || new_state == C_IS_FREE);
1110
1111 c_empty_count--;
1112 break;
1113
1114 case C_IS_FILLING:
1115 assert(new_state == C_ON_AGE_Q || new_state == C_ON_SWAPOUT_Q);
1116
1117 queue_remove(&c_filling_list_head, c_seg, c_segment_t, c_age_list);
1118 c_filling_count--;
1119 break;
1120
1121 case C_ON_AGE_Q:
1122 assert(new_state == C_ON_SWAPOUT_Q || new_state == C_ON_MAJORCOMPACT_Q ||
1123 new_state == C_IS_FREE);
1124
1125 queue_remove(&c_age_list_head, c_seg, c_segment_t, c_age_list);
1126 c_age_count--;
1127 break;
1128
1129 case C_ON_SWAPPEDIN_Q:
1130 assert(new_state == C_ON_AGE_Q || new_state == C_IS_FREE);
1131
1132 queue_remove(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
1133 c_swappedin_count--;
1134 break;
1135
1136 case C_ON_SWAPOUT_Q:
1137 assert(new_state == C_ON_AGE_Q || new_state == C_IS_FREE || new_state == C_IS_EMPTY || new_state == C_ON_SWAPIO_Q);
1138
1139 queue_remove(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
1140 thread_wakeup((event_t)&compaction_swapper_running);
1141 c_swapout_count--;
1142 break;
1143
1144 case C_ON_SWAPIO_Q:
1145 assert(new_state == C_ON_SWAPPEDOUT_Q || new_state == C_ON_SWAPPEDOUTSPARSE_Q || new_state == C_ON_AGE_Q);
1146
1147 queue_remove(&c_swapio_list_head, c_seg, c_segment_t, c_age_list);
1148 c_swapio_count--;
1149 break;
1150
1151 case C_ON_SWAPPEDOUT_Q:
1152 assert(new_state == C_ON_SWAPPEDIN_Q || new_state == C_ON_AGE_Q ||
1153 new_state == C_ON_SWAPPEDOUTSPARSE_Q ||
1154 new_state == C_ON_BAD_Q || new_state == C_IS_EMPTY || new_state == C_IS_FREE);
1155
1156 queue_remove(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
1157 c_swappedout_count--;
1158 break;
1159
1160 case C_ON_SWAPPEDOUTSPARSE_Q:
1161 assert(new_state == C_ON_SWAPPEDIN_Q || new_state == C_ON_AGE_Q ||
1162 new_state == C_ON_BAD_Q || new_state == C_IS_EMPTY || new_state == C_IS_FREE);
1163
1164 queue_remove(&c_swappedout_sparse_list_head, c_seg, c_segment_t, c_age_list);
1165 c_swappedout_sparse_count--;
1166 break;
1167
1168 case C_ON_MAJORCOMPACT_Q:
1169 assert(new_state == C_ON_AGE_Q || new_state == C_IS_FREE);
1170
1171 queue_remove(&c_major_list_head, c_seg, c_segment_t, c_age_list);
1172 c_major_count--;
1173 break;
1174
1175 case C_ON_BAD_Q:
1176 assert(new_state == C_IS_FREE);
1177
1178 queue_remove(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
1179 c_bad_count--;
1180 break;
1181
1182 default:
1183 panic("c_seg %p has bad c_state = %d\n", c_seg, old_state);
1184 }
1185
1186 switch (new_state) {
1187 case C_IS_FREE:
1188 assert(old_state != C_IS_FILLING);
1189
1190 break;
1191
1192 case C_IS_EMPTY:
1193 assert(old_state == C_ON_SWAPOUT_Q || old_state == C_ON_SWAPPEDOUT_Q || old_state == C_ON_SWAPPEDOUTSPARSE_Q);
1194
1195 c_empty_count++;
1196 break;
1197
1198 case C_IS_FILLING:
1199 assert(old_state == C_IS_EMPTY);
1200
1201 queue_enter(&c_filling_list_head, c_seg, c_segment_t, c_age_list);
1202 c_filling_count++;
1203 break;
1204
1205 case C_ON_AGE_Q:
1206 assert(old_state == C_IS_FILLING || old_state == C_ON_SWAPPEDIN_Q ||
1207 old_state == C_ON_SWAPOUT_Q || old_state == C_ON_SWAPIO_Q ||
1208 old_state == C_ON_MAJORCOMPACT_Q || old_state == C_ON_SWAPPEDOUT_Q || old_state == C_ON_SWAPPEDOUTSPARSE_Q);
1209
1210 if (old_state == C_IS_FILLING) {
1211 queue_enter(&c_age_list_head, c_seg, c_segment_t, c_age_list);
1212 } else {
1213 if (!queue_empty(&c_age_list_head)) {
1214 c_segment_t c_first;
1215
1216 c_first = (c_segment_t)queue_first(&c_age_list_head);
1217 c_seg->c_creation_ts = c_first->c_creation_ts;
1218 }
1219 queue_enter_first(&c_age_list_head, c_seg, c_segment_t, c_age_list);
1220 }
1221 c_age_count++;
1222 break;
1223
1224 case C_ON_SWAPPEDIN_Q:
1225 assert(old_state == C_ON_SWAPPEDOUT_Q || old_state == C_ON_SWAPPEDOUTSPARSE_Q);
1226
1227 if (insert_head == TRUE) {
1228 queue_enter_first(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
1229 } else {
1230 queue_enter(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
1231 }
1232 c_swappedin_count++;
1233 break;
1234
1235 case C_ON_SWAPOUT_Q:
1236 assert(old_state == C_ON_AGE_Q || old_state == C_IS_FILLING);
1237
1238 if (insert_head == TRUE) {
1239 queue_enter_first(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
1240 } else {
1241 queue_enter(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
1242 }
1243 c_swapout_count++;
1244 break;
1245
1246 case C_ON_SWAPIO_Q:
1247 assert(old_state == C_ON_SWAPOUT_Q);
1248
1249 if (insert_head == TRUE) {
1250 queue_enter_first(&c_swapio_list_head, c_seg, c_segment_t, c_age_list);
1251 } else {
1252 queue_enter(&c_swapio_list_head, c_seg, c_segment_t, c_age_list);
1253 }
1254 c_swapio_count++;
1255 break;
1256
1257 case C_ON_SWAPPEDOUT_Q:
1258 assert(old_state == C_ON_SWAPIO_Q);
1259
1260 if (insert_head == TRUE) {
1261 queue_enter_first(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
1262 } else {
1263 queue_enter(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
1264 }
1265 c_swappedout_count++;
1266 break;
1267
1268 case C_ON_SWAPPEDOUTSPARSE_Q:
1269 assert(old_state == C_ON_SWAPIO_Q || old_state == C_ON_SWAPPEDOUT_Q);
1270
1271 if (insert_head == TRUE) {
1272 queue_enter_first(&c_swappedout_sparse_list_head, c_seg, c_segment_t, c_age_list);
1273 } else {
1274 queue_enter(&c_swappedout_sparse_list_head, c_seg, c_segment_t, c_age_list);
1275 }
1276
1277 c_swappedout_sparse_count++;
1278 break;
1279
1280 case C_ON_MAJORCOMPACT_Q:
1281 assert(old_state == C_ON_AGE_Q);
1282
1283 if (insert_head == TRUE) {
1284 queue_enter_first(&c_major_list_head, c_seg, c_segment_t, c_age_list);
1285 } else {
1286 queue_enter(&c_major_list_head, c_seg, c_segment_t, c_age_list);
1287 }
1288 c_major_count++;
1289 break;
1290
1291 case C_ON_BAD_Q:
1292 assert(old_state == C_ON_SWAPPEDOUT_Q || old_state == C_ON_SWAPPEDOUTSPARSE_Q);
1293
1294 if (insert_head == TRUE) {
1295 queue_enter_first(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
1296 } else {
1297 queue_enter(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
1298 }
1299 c_bad_count++;
1300 break;
1301
1302 default:
1303 panic("c_seg %p requesting bad c_state = %d\n", c_seg, new_state);
1304 }
1305 c_seg->c_state = new_state;
1306 }
1307
1308
1309
1310 void
1311 c_seg_free(c_segment_t c_seg)
1312 {
1313 assert(c_seg->c_busy);
1314
1315 lck_mtx_unlock_always(&c_seg->c_lock);
1316 lck_mtx_lock_spin_always(c_list_lock);
1317 lck_mtx_lock_spin_always(&c_seg->c_lock);
1318
1319 c_seg_free_locked(c_seg);
1320 }
1321
1322
1323 void
1324 c_seg_free_locked(c_segment_t c_seg)
1325 {
1326 int segno;
1327 int pages_populated = 0;
1328 int32_t *c_buffer = NULL;
1329 uint64_t c_swap_handle = 0;
1330
1331 assert(c_seg->c_busy);
1332 assert(c_seg->c_slots_used == 0);
1333 assert(!c_seg->c_on_minorcompact_q);
1334 assert(!c_seg->c_busy_swapping);
1335
1336 if (c_seg->c_overage_swap == TRUE) {
1337 c_overage_swapped_count--;
1338 c_seg->c_overage_swap = FALSE;
1339 }
1340 if (!(C_SEG_IS_ONDISK(c_seg))) {
1341 c_buffer = c_seg->c_store.c_buffer;
1342 } else {
1343 c_swap_handle = c_seg->c_store.c_swap_handle;
1344 }
1345
1346 c_seg_switch_state(c_seg, C_IS_FREE, FALSE);
1347
1348 lck_mtx_unlock_always(c_list_lock);
1349
1350 if (c_buffer) {
1351 pages_populated = (round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE;
1352 c_seg->c_store.c_buffer = NULL;
1353 } else {
1354 c_seg->c_store.c_swap_handle = (uint64_t)-1;
1355 }
1356
1357 lck_mtx_unlock_always(&c_seg->c_lock);
1358
1359 if (c_buffer) {
1360 if (pages_populated) {
1361 kernel_memory_depopulate(compressor_map, (vm_offset_t) c_buffer, pages_populated * PAGE_SIZE, KMA_COMPRESSOR);
1362 }
1363 } else if (c_swap_handle) {
1364 /*
1365 * Free swap space on disk.
1366 */
1367 vm_swap_free(c_swap_handle);
1368 }
1369 lck_mtx_lock_spin_always(&c_seg->c_lock);
1370 /*
1371 * c_seg must remain busy until
1372 * after the call to vm_swap_free
1373 */
1374 C_SEG_WAKEUP_DONE(c_seg);
1375 lck_mtx_unlock_always(&c_seg->c_lock);
1376
1377 segno = c_seg->c_mysegno;
1378
1379 lck_mtx_lock_spin_always(c_list_lock);
1380 /*
1381 * because the c_buffer is now associated with the segno,
1382 * we can't put the segno back on the free list until
1383 * after we have depopulated the c_buffer range, or
1384 * we run the risk of depopulating a range that is
1385 * now being used in one of the compressor heads
1386 */
1387 c_segments[segno].c_segno = c_free_segno_head;
1388 c_free_segno_head = segno;
1389 c_segment_count--;
1390
1391 lck_mtx_unlock_always(c_list_lock);
1392
1393 lck_mtx_destroy(&c_seg->c_lock, &vm_compressor_lck_grp);
1394
1395 if (c_seg->c_slot_var_array_len) {
1396 kfree(c_seg->c_slot_var_array, sizeof(struct c_slot) * c_seg->c_slot_var_array_len);
1397 }
1398
1399 zfree(compressor_segment_zone, c_seg);
1400 }
1401
1402 #if DEVELOPMENT || DEBUG
1403 int c_seg_trim_page_count = 0;
1404 #endif
1405
1406 void
1407 c_seg_trim_tail(c_segment_t c_seg)
1408 {
1409 c_slot_t cs;
1410 uint32_t c_size;
1411 uint32_t c_offset;
1412 uint32_t c_rounded_size;
1413 uint16_t current_nextslot;
1414 uint32_t current_populated_offset;
1415
1416 if (c_seg->c_bytes_used == 0) {
1417 return;
1418 }
1419 current_nextslot = c_seg->c_nextslot;
1420 current_populated_offset = c_seg->c_populated_offset;
1421
1422 while (c_seg->c_nextslot) {
1423 cs = C_SEG_SLOT_FROM_INDEX(c_seg, (c_seg->c_nextslot - 1));
1424
1425 c_size = UNPACK_C_SIZE(cs);
1426
1427 if (c_size) {
1428 if (current_nextslot != c_seg->c_nextslot) {
1429 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
1430 c_offset = cs->c_offset + C_SEG_BYTES_TO_OFFSET(c_rounded_size);
1431
1432 c_seg->c_nextoffset = c_offset;
1433 c_seg->c_populated_offset = (c_offset + (C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1)) &
1434 ~(C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1);
1435
1436 if (c_seg->c_firstemptyslot > c_seg->c_nextslot) {
1437 c_seg->c_firstemptyslot = c_seg->c_nextslot;
1438 }
1439 #if DEVELOPMENT || DEBUG
1440 c_seg_trim_page_count += ((round_page_32(C_SEG_OFFSET_TO_BYTES(current_populated_offset)) -
1441 round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) /
1442 PAGE_SIZE);
1443 #endif
1444 }
1445 break;
1446 }
1447 c_seg->c_nextslot--;
1448 }
1449 assert(c_seg->c_nextslot);
1450 }
1451
1452
1453 int
1454 c_seg_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy)
1455 {
1456 c_slot_mapping_t slot_ptr;
1457 uint32_t c_offset = 0;
1458 uint32_t old_populated_offset;
1459 uint32_t c_rounded_size;
1460 uint32_t c_size;
1461 int c_indx = 0;
1462 int i;
1463 c_slot_t c_dst;
1464 c_slot_t c_src;
1465
1466 assert(c_seg->c_busy);
1467
1468 #if VALIDATE_C_SEGMENTS
1469 c_seg_validate(c_seg, FALSE);
1470 #endif
1471 if (c_seg->c_bytes_used == 0) {
1472 c_seg_free(c_seg);
1473 return 1;
1474 }
1475 lck_mtx_unlock_always(&c_seg->c_lock);
1476
1477 if (c_seg->c_firstemptyslot >= c_seg->c_nextslot || C_SEG_UNUSED_BYTES(c_seg) < PAGE_SIZE) {
1478 goto done;
1479 }
1480
1481 /* TODO: assert first emptyslot's c_size is actually 0 */
1482
1483 #if DEVELOPMENT || DEBUG
1484 C_SEG_MAKE_WRITEABLE(c_seg);
1485 #endif
1486
1487 #if VALIDATE_C_SEGMENTS
1488 c_seg->c_was_minor_compacted++;
1489 #endif
1490 c_indx = c_seg->c_firstemptyslot;
1491 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
1492
1493 old_populated_offset = c_seg->c_populated_offset;
1494 c_offset = c_dst->c_offset;
1495
1496 for (i = c_indx + 1; i < c_seg->c_nextslot && c_offset < c_seg->c_nextoffset; i++) {
1497 c_src = C_SEG_SLOT_FROM_INDEX(c_seg, i);
1498
1499 c_size = UNPACK_C_SIZE(c_src);
1500
1501 if (c_size == 0) {
1502 continue;
1503 }
1504
1505 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
1506 /* N.B.: This memcpy may be an overlapping copy */
1507 memcpy(&c_seg->c_store.c_buffer[c_offset], &c_seg->c_store.c_buffer[c_src->c_offset], c_rounded_size);
1508
1509 cslot_copy(c_dst, c_src);
1510 c_dst->c_offset = c_offset;
1511
1512 slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
1513 slot_ptr->s_cindx = c_indx;
1514
1515 c_offset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
1516 PACK_C_SIZE(c_src, 0);
1517 c_indx++;
1518
1519 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
1520 }
1521 c_seg->c_firstemptyslot = c_indx;
1522 c_seg->c_nextslot = c_indx;
1523 c_seg->c_nextoffset = c_offset;
1524 c_seg->c_populated_offset = (c_offset + (C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1)) & ~(C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1);
1525 c_seg->c_bytes_unused = 0;
1526
1527 #if VALIDATE_C_SEGMENTS
1528 c_seg_validate(c_seg, TRUE);
1529 #endif
1530 if (old_populated_offset > c_seg->c_populated_offset) {
1531 uint32_t gc_size;
1532 int32_t *gc_ptr;
1533
1534 gc_size = C_SEG_OFFSET_TO_BYTES(old_populated_offset - c_seg->c_populated_offset);
1535 gc_ptr = &c_seg->c_store.c_buffer[c_seg->c_populated_offset];
1536
1537 kernel_memory_depopulate(compressor_map, (vm_offset_t)gc_ptr, gc_size, KMA_COMPRESSOR);
1538 }
1539
1540 #if DEVELOPMENT || DEBUG
1541 C_SEG_WRITE_PROTECT(c_seg);
1542 #endif
1543
1544 done:
1545 if (clear_busy == TRUE) {
1546 lck_mtx_lock_spin_always(&c_seg->c_lock);
1547 C_SEG_WAKEUP_DONE(c_seg);
1548 lck_mtx_unlock_always(&c_seg->c_lock);
1549 }
1550 return 0;
1551 }
1552
1553
1554 static void
1555 c_seg_alloc_nextslot(c_segment_t c_seg)
1556 {
1557 struct c_slot *old_slot_array = NULL;
1558 struct c_slot *new_slot_array = NULL;
1559 int newlen;
1560 int oldlen;
1561
1562 if (c_seg->c_nextslot < c_seg_fixed_array_len) {
1563 return;
1564 }
1565
1566 if ((c_seg->c_nextslot - c_seg_fixed_array_len) >= c_seg->c_slot_var_array_len) {
1567 oldlen = c_seg->c_slot_var_array_len;
1568 old_slot_array = c_seg->c_slot_var_array;
1569
1570 if (oldlen == 0) {
1571 newlen = C_SEG_SLOT_VAR_ARRAY_MIN_LEN;
1572 } else {
1573 newlen = oldlen * 2;
1574 }
1575
1576 new_slot_array = (struct c_slot *)kalloc(sizeof(struct c_slot) * newlen);
1577
1578 lck_mtx_lock_spin_always(&c_seg->c_lock);
1579
1580 if (old_slot_array) {
1581 memcpy((char *)new_slot_array, (char *)old_slot_array, sizeof(struct c_slot) * oldlen);
1582 }
1583
1584 c_seg->c_slot_var_array_len = newlen;
1585 c_seg->c_slot_var_array = new_slot_array;
1586
1587 lck_mtx_unlock_always(&c_seg->c_lock);
1588
1589 if (old_slot_array) {
1590 kfree(old_slot_array, sizeof(struct c_slot) * oldlen);
1591 }
1592 }
1593 }
1594
1595
1596
1597 struct {
1598 uint64_t asked_permission;
1599 uint64_t compactions;
1600 uint64_t moved_slots;
1601 uint64_t moved_bytes;
1602 uint64_t wasted_space_in_swapouts;
1603 uint64_t count_of_swapouts;
1604 uint64_t count_of_freed_segs;
1605 } c_seg_major_compact_stats;
1606
1607
1608 #define C_MAJOR_COMPACTION_SIZE_APPROPRIATE ((C_SEG_BUFSIZE * 90) / 100)
1609
1610
1611 boolean_t
1612 c_seg_major_compact_ok(
1613 c_segment_t c_seg_dst,
1614 c_segment_t c_seg_src)
1615 {
1616 c_seg_major_compact_stats.asked_permission++;
1617
1618 if (c_seg_src->c_bytes_used >= C_MAJOR_COMPACTION_SIZE_APPROPRIATE &&
1619 c_seg_dst->c_bytes_used >= C_MAJOR_COMPACTION_SIZE_APPROPRIATE) {
1620 return FALSE;
1621 }
1622
1623 if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX_INDEX) {
1624 /*
1625 * destination segment is full... can't compact
1626 */
1627 return FALSE;
1628 }
1629
1630 return TRUE;
1631 }
1632
1633
1634 boolean_t
1635 c_seg_major_compact(
1636 c_segment_t c_seg_dst,
1637 c_segment_t c_seg_src)
1638 {
1639 c_slot_mapping_t slot_ptr;
1640 uint32_t c_rounded_size;
1641 uint32_t c_size;
1642 uint16_t dst_slot;
1643 int i;
1644 c_slot_t c_dst;
1645 c_slot_t c_src;
1646 boolean_t keep_compacting = TRUE;
1647
1648 /*
1649 * segments are not locked but they are both marked c_busy
1650 * which keeps c_decompress from working on them...
1651 * we can safely allocate new pages, move compressed data
1652 * from c_seg_src to c_seg_dst and update both c_segment's
1653 * state w/o holding the master lock
1654 */
1655 #if DEVELOPMENT || DEBUG
1656 C_SEG_MAKE_WRITEABLE(c_seg_dst);
1657 #endif
1658
1659 #if VALIDATE_C_SEGMENTS
1660 c_seg_dst->c_was_major_compacted++;
1661 c_seg_src->c_was_major_donor++;
1662 #endif
1663 c_seg_major_compact_stats.compactions++;
1664
1665 dst_slot = c_seg_dst->c_nextslot;
1666
1667 for (i = 0; i < c_seg_src->c_nextslot; i++) {
1668 c_src = C_SEG_SLOT_FROM_INDEX(c_seg_src, i);
1669
1670 c_size = UNPACK_C_SIZE(c_src);
1671
1672 if (c_size == 0) {
1673 /* BATCH: move what we have so far; */
1674 continue;
1675 }
1676
1677 if (C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset - c_seg_dst->c_nextoffset) < (unsigned) c_size) {
1678 int size_to_populate;
1679
1680 /* doesn't fit */
1681 size_to_populate = C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset);
1682
1683 if (size_to_populate == 0) {
1684 /* can't fit */
1685 keep_compacting = FALSE;
1686 break;
1687 }
1688 if (size_to_populate > C_SEG_MAX_POPULATE_SIZE) {
1689 size_to_populate = C_SEG_MAX_POPULATE_SIZE;
1690 }
1691
1692 kernel_memory_populate(compressor_map,
1693 (vm_offset_t) &c_seg_dst->c_store.c_buffer[c_seg_dst->c_populated_offset],
1694 size_to_populate,
1695 KMA_COMPRESSOR,
1696 VM_KERN_MEMORY_COMPRESSOR);
1697
1698 c_seg_dst->c_populated_offset += C_SEG_BYTES_TO_OFFSET(size_to_populate);
1699 assert(C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset) <= C_SEG_BUFSIZE);
1700 }
1701 c_seg_alloc_nextslot(c_seg_dst);
1702
1703 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, c_seg_dst->c_nextslot);
1704
1705 memcpy(&c_seg_dst->c_store.c_buffer[c_seg_dst->c_nextoffset], &c_seg_src->c_store.c_buffer[c_src->c_offset], c_size);
1706
1707 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
1708
1709 c_seg_major_compact_stats.moved_slots++;
1710 c_seg_major_compact_stats.moved_bytes += c_size;
1711
1712 cslot_copy(c_dst, c_src);
1713 c_dst->c_offset = c_seg_dst->c_nextoffset;
1714
1715 if (c_seg_dst->c_firstemptyslot == c_seg_dst->c_nextslot) {
1716 c_seg_dst->c_firstemptyslot++;
1717 }
1718 c_seg_dst->c_slots_used++;
1719 c_seg_dst->c_nextslot++;
1720 c_seg_dst->c_bytes_used += c_rounded_size;
1721 c_seg_dst->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
1722
1723 PACK_C_SIZE(c_src, 0);
1724
1725 c_seg_src->c_bytes_used -= c_rounded_size;
1726 c_seg_src->c_bytes_unused += c_rounded_size;
1727 c_seg_src->c_firstemptyslot = 0;
1728
1729 assert(c_seg_src->c_slots_used);
1730 c_seg_src->c_slots_used--;
1731
1732 if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX_INDEX) {
1733 /* dest segment is now full */
1734 keep_compacting = FALSE;
1735 break;
1736 }
1737 }
1738 #if DEVELOPMENT || DEBUG
1739 C_SEG_WRITE_PROTECT(c_seg_dst);
1740 #endif
1741 if (dst_slot < c_seg_dst->c_nextslot) {
1742 PAGE_REPLACEMENT_ALLOWED(TRUE);
1743 /*
1744 * we've now locked out c_decompress from
1745 * converting the slot passed into it into
1746 * a c_segment_t which allows us to use
1747 * the backptr to change which c_segment and
1748 * index the slot points to
1749 */
1750 while (dst_slot < c_seg_dst->c_nextslot) {
1751 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, dst_slot);
1752
1753 slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
1754 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
1755 slot_ptr->s_cseg = c_seg_dst->c_mysegno + 1;
1756 slot_ptr->s_cindx = dst_slot++;
1757 }
1758 PAGE_REPLACEMENT_ALLOWED(FALSE);
1759 }
1760 return keep_compacting;
1761 }
1762
1763
1764 uint64_t
1765 vm_compressor_compute_elapsed_msecs(clock_sec_t end_sec, clock_nsec_t end_nsec, clock_sec_t start_sec, clock_nsec_t start_nsec)
1766 {
1767 uint64_t end_msecs;
1768 uint64_t start_msecs;
1769
1770 end_msecs = (end_sec * 1000) + end_nsec / 1000000;
1771 start_msecs = (start_sec * 1000) + start_nsec / 1000000;
1772
1773 return end_msecs - start_msecs;
1774 }
1775
1776
1777
1778 uint32_t compressor_eval_period_in_msecs = 250;
1779 uint32_t compressor_sample_min_in_msecs = 500;
1780 uint32_t compressor_sample_max_in_msecs = 10000;
1781 uint32_t compressor_thrashing_threshold_per_10msecs = 50;
1782 uint32_t compressor_thrashing_min_per_10msecs = 20;
1783
1784 /* When true, reset sample data next chance we get. */
1785 static boolean_t compressor_need_sample_reset = FALSE;
1786
1787
1788 void
1789 compute_swapout_target_age(void)
1790 {
1791 clock_sec_t cur_ts_sec;
1792 clock_nsec_t cur_ts_nsec;
1793 uint32_t min_operations_needed_in_this_sample;
1794 uint64_t elapsed_msecs_in_eval;
1795 uint64_t elapsed_msecs_in_sample;
1796 boolean_t need_eval_reset = FALSE;
1797
1798 clock_get_system_nanotime(&cur_ts_sec, &cur_ts_nsec);
1799
1800 elapsed_msecs_in_sample = vm_compressor_compute_elapsed_msecs(cur_ts_sec, cur_ts_nsec, start_of_sample_period_sec, start_of_sample_period_nsec);
1801
1802 if (compressor_need_sample_reset ||
1803 elapsed_msecs_in_sample >= compressor_sample_max_in_msecs) {
1804 compressor_need_sample_reset = TRUE;
1805 need_eval_reset = TRUE;
1806 goto done;
1807 }
1808 elapsed_msecs_in_eval = vm_compressor_compute_elapsed_msecs(cur_ts_sec, cur_ts_nsec, start_of_eval_period_sec, start_of_eval_period_nsec);
1809
1810 if (elapsed_msecs_in_eval < compressor_eval_period_in_msecs) {
1811 goto done;
1812 }
1813 need_eval_reset = TRUE;
1814
1815 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_START, elapsed_msecs_in_eval, sample_period_compression_count, sample_period_decompression_count, 0, 0);
1816
1817 min_operations_needed_in_this_sample = (compressor_thrashing_min_per_10msecs * (uint32_t)elapsed_msecs_in_eval) / 10;
1818
1819 if ((sample_period_compression_count - last_eval_compression_count) < min_operations_needed_in_this_sample ||
1820 (sample_period_decompression_count - last_eval_decompression_count) < min_operations_needed_in_this_sample) {
1821 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, sample_period_compression_count - last_eval_compression_count,
1822 sample_period_decompression_count - last_eval_decompression_count, 0, 1, 0);
1823
1824 swapout_target_age = 0;
1825
1826 compressor_need_sample_reset = TRUE;
1827 need_eval_reset = TRUE;
1828 goto done;
1829 }
1830 last_eval_compression_count = sample_period_compression_count;
1831 last_eval_decompression_count = sample_period_decompression_count;
1832
1833 if (elapsed_msecs_in_sample < compressor_sample_min_in_msecs) {
1834 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, swapout_target_age, 0, 0, 5, 0);
1835 goto done;
1836 }
1837 if (sample_period_decompression_count > ((compressor_thrashing_threshold_per_10msecs * elapsed_msecs_in_sample) / 10)) {
1838 uint64_t running_total;
1839 uint64_t working_target;
1840 uint64_t aging_target;
1841 uint32_t oldest_age_of_csegs_sampled = 0;
1842 uint64_t working_set_approximation = 0;
1843
1844 swapout_target_age = 0;
1845
1846 working_target = (sample_period_decompression_count / 100) * 95; /* 95 percent */
1847 aging_target = (sample_period_decompression_count / 100) * 1; /* 1 percent */
1848 running_total = 0;
1849
1850 for (oldest_age_of_csegs_sampled = 0; oldest_age_of_csegs_sampled < DECOMPRESSION_SAMPLE_MAX_AGE; oldest_age_of_csegs_sampled++) {
1851 running_total += age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
1852
1853 working_set_approximation += oldest_age_of_csegs_sampled * age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
1854
1855 if (running_total >= working_target) {
1856 break;
1857 }
1858 }
1859 if (oldest_age_of_csegs_sampled < DECOMPRESSION_SAMPLE_MAX_AGE) {
1860 working_set_approximation = (working_set_approximation * 1000) / elapsed_msecs_in_sample;
1861
1862 if (working_set_approximation < VM_PAGE_COMPRESSOR_COUNT) {
1863 running_total = overage_decompressions_during_sample_period;
1864
1865 for (oldest_age_of_csegs_sampled = DECOMPRESSION_SAMPLE_MAX_AGE - 1; oldest_age_of_csegs_sampled; oldest_age_of_csegs_sampled--) {
1866 running_total += age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
1867
1868 if (running_total >= aging_target) {
1869 break;
1870 }
1871 }
1872 swapout_target_age = (uint32_t)cur_ts_sec - oldest_age_of_csegs_sampled;
1873
1874 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, swapout_target_age, working_set_approximation, VM_PAGE_COMPRESSOR_COUNT, 2, 0);
1875 } else {
1876 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, working_set_approximation, VM_PAGE_COMPRESSOR_COUNT, 0, 3, 0);
1877 }
1878 } else {
1879 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, working_target, running_total, 0, 4, 0);
1880 }
1881
1882 compressor_need_sample_reset = TRUE;
1883 need_eval_reset = TRUE;
1884 } else {
1885 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, sample_period_decompression_count, (compressor_thrashing_threshold_per_10msecs * elapsed_msecs_in_sample) / 10, 0, 6, 0);
1886 }
1887 done:
1888 if (compressor_need_sample_reset == TRUE) {
1889 bzero(age_of_decompressions_during_sample_period, sizeof(age_of_decompressions_during_sample_period));
1890 overage_decompressions_during_sample_period = 0;
1891
1892 start_of_sample_period_sec = cur_ts_sec;
1893 start_of_sample_period_nsec = cur_ts_nsec;
1894 sample_period_decompression_count = 0;
1895 sample_period_compression_count = 0;
1896 last_eval_decompression_count = 0;
1897 last_eval_compression_count = 0;
1898 compressor_need_sample_reset = FALSE;
1899 }
1900 if (need_eval_reset == TRUE) {
1901 start_of_eval_period_sec = cur_ts_sec;
1902 start_of_eval_period_nsec = cur_ts_nsec;
1903 }
1904 }
1905
1906
1907 int compaction_swapper_init_now = 0;
1908 int compaction_swapper_running = 0;
1909 int compaction_swapper_awakened = 0;
1910 int compaction_swapper_abort = 0;
1911
1912
1913 #if CONFIG_JETSAM
1914 boolean_t memorystatus_kill_on_VM_compressor_thrashing(boolean_t);
1915 boolean_t memorystatus_kill_on_VM_compressor_space_shortage(boolean_t);
1916 boolean_t memorystatus_kill_on_FC_thrashing(boolean_t);
1917 int compressor_thrashing_induced_jetsam = 0;
1918 int filecache_thrashing_induced_jetsam = 0;
1919 static boolean_t vm_compressor_thrashing_detected = FALSE;
1920 #endif /* CONFIG_JETSAM */
1921
1922 static boolean_t
1923 compressor_needs_to_swap(void)
1924 {
1925 boolean_t should_swap = FALSE;
1926
1927 if (vm_swapout_ripe_segments == TRUE && c_overage_swapped_count < c_overage_swapped_limit) {
1928 c_segment_t c_seg;
1929 clock_sec_t now;
1930 clock_sec_t age;
1931 clock_nsec_t nsec;
1932
1933 clock_get_system_nanotime(&now, &nsec);
1934 age = 0;
1935
1936 lck_mtx_lock_spin_always(c_list_lock);
1937
1938 if (!queue_empty(&c_age_list_head)) {
1939 c_seg = (c_segment_t) queue_first(&c_age_list_head);
1940
1941 age = now - c_seg->c_creation_ts;
1942 }
1943 lck_mtx_unlock_always(c_list_lock);
1944
1945 if (age >= vm_ripe_target_age) {
1946 return TRUE;
1947 }
1948 }
1949 if (VM_CONFIG_SWAP_IS_ACTIVE) {
1950 if (COMPRESSOR_NEEDS_TO_SWAP()) {
1951 return TRUE;
1952 }
1953 if (VM_PAGE_Q_THROTTLED(&vm_pageout_queue_external) && vm_page_anonymous_count < (vm_page_inactive_count / 20)) {
1954 return TRUE;
1955 }
1956 if (vm_page_free_count < (vm_page_free_reserved - (COMPRESSOR_FREE_RESERVED_LIMIT * 2))) {
1957 return TRUE;
1958 }
1959 }
1960 compute_swapout_target_age();
1961
1962 if (swapout_target_age) {
1963 c_segment_t c_seg;
1964
1965 lck_mtx_lock_spin_always(c_list_lock);
1966
1967 if (!queue_empty(&c_age_list_head)) {
1968 c_seg = (c_segment_t) queue_first(&c_age_list_head);
1969
1970 if (c_seg->c_creation_ts > swapout_target_age) {
1971 swapout_target_age = 0;
1972 }
1973 }
1974 lck_mtx_unlock_always(c_list_lock);
1975 }
1976 #if CONFIG_PHANTOM_CACHE
1977 if (vm_phantom_cache_check_pressure()) {
1978 should_swap = TRUE;
1979 }
1980 #endif
1981 if (swapout_target_age) {
1982 should_swap = TRUE;
1983 }
1984
1985 #if CONFIG_JETSAM
1986 if (should_swap || vm_compressor_low_on_space() == TRUE) {
1987 if (vm_compressor_thrashing_detected == FALSE) {
1988 vm_compressor_thrashing_detected = TRUE;
1989
1990 if (swapout_target_age || vm_compressor_low_on_space() == TRUE) {
1991 if (swapout_target_age) {
1992 /* The compressor is thrashing. */
1993 memorystatus_kill_on_VM_compressor_thrashing(TRUE /* async */);
1994 } else {
1995 /* The compressor is running low on space. */
1996 memorystatus_kill_on_VM_compressor_space_shortage(TRUE /* async */);
1997 }
1998 compressor_thrashing_induced_jetsam++;
1999 } else {
2000 memorystatus_kill_on_FC_thrashing(TRUE /* async */);
2001 filecache_thrashing_induced_jetsam++;
2002 }
2003 }
2004 /*
2005 * let the jetsam take precedence over
2006 * any major compactions we might have
2007 * been able to do... otherwise we run
2008 * the risk of doing major compactions
2009 * on segments we're about to free up
2010 * due to the jetsam activity.
2011 */
2012 should_swap = FALSE;
2013 }
2014
2015 #endif /* CONFIG_JETSAM */
2016
2017 if (should_swap == FALSE) {
2018 /*
2019 * vm_compressor_needs_to_major_compact returns true only if we're
2020 * about to run out of available compressor segments... in this
2021 * case, we absolutely need to run a major compaction even if
2022 * we've just kicked off a jetsam or we don't otherwise need to
2023 * swap... terminating objects releases
2024 * pages back to the uncompressed cache, but does not guarantee
2025 * that we will free up even a single compression segment
2026 */
2027 should_swap = vm_compressor_needs_to_major_compact();
2028 }
2029
2030 /*
2031 * returning TRUE when swap_supported == FALSE
2032 * will cause the major compaction engine to
2033 * run, but will not trigger any swapping...
2034 * segments that have been major compacted
2035 * will be moved to the majorcompact queue
2036 */
2037 return should_swap;
2038 }
2039
2040 #if CONFIG_JETSAM
2041 /*
2042 * This function is called from the jetsam thread after killing something to
2043 * mitigate thrashing.
2044 *
2045 * We need to restart our thrashing detection heuristics since memory pressure
2046 * has potentially changed significantly, and we don't want to detect on old
2047 * data from before the jetsam.
2048 */
2049 void
2050 vm_thrashing_jetsam_done(void)
2051 {
2052 vm_compressor_thrashing_detected = FALSE;
2053
2054 /* Were we compressor-thrashing or filecache-thrashing? */
2055 if (swapout_target_age) {
2056 swapout_target_age = 0;
2057 compressor_need_sample_reset = TRUE;
2058 }
2059 #if CONFIG_PHANTOM_CACHE
2060 else {
2061 vm_phantom_cache_restart_sample();
2062 }
2063 #endif
2064 }
2065 #endif /* CONFIG_JETSAM */
2066
2067 uint32_t vm_wake_compactor_swapper_calls = 0;
2068 uint32_t vm_run_compactor_already_running = 0;
2069 uint32_t vm_run_compactor_empty_minor_q = 0;
2070 uint32_t vm_run_compactor_did_compact = 0;
2071 uint32_t vm_run_compactor_waited = 0;
2072
2073 void
2074 vm_run_compactor(void)
2075 {
2076 if (c_segment_count == 0) {
2077 return;
2078 }
2079
2080 lck_mtx_lock_spin_always(c_list_lock);
2081
2082 if (c_minor_count == 0) {
2083 vm_run_compactor_empty_minor_q++;
2084
2085 lck_mtx_unlock_always(c_list_lock);
2086 return;
2087 }
2088 if (compaction_swapper_running) {
2089 if (vm_pageout_state.vm_restricted_to_single_processor == FALSE) {
2090 vm_run_compactor_already_running++;
2091
2092 lck_mtx_unlock_always(c_list_lock);
2093 return;
2094 }
2095 vm_run_compactor_waited++;
2096
2097 assert_wait((event_t)&compaction_swapper_running, THREAD_UNINT);
2098
2099 lck_mtx_unlock_always(c_list_lock);
2100
2101 thread_block(THREAD_CONTINUE_NULL);
2102
2103 return;
2104 }
2105 vm_run_compactor_did_compact++;
2106
2107 fastwake_warmup = FALSE;
2108 compaction_swapper_running = 1;
2109
2110 vm_compressor_do_delayed_compactions(FALSE);
2111
2112 compaction_swapper_running = 0;
2113
2114 lck_mtx_unlock_always(c_list_lock);
2115
2116 thread_wakeup((event_t)&compaction_swapper_running);
2117 }
2118
2119
2120 void
2121 vm_wake_compactor_swapper(void)
2122 {
2123 if (compaction_swapper_running || compaction_swapper_awakened || c_segment_count == 0) {
2124 return;
2125 }
2126
2127 if (c_minor_count || vm_compressor_needs_to_major_compact()) {
2128 lck_mtx_lock_spin_always(c_list_lock);
2129
2130 fastwake_warmup = FALSE;
2131
2132 if (compaction_swapper_running == 0 && compaction_swapper_awakened == 0) {
2133 vm_wake_compactor_swapper_calls++;
2134
2135 compaction_swapper_awakened = 1;
2136 thread_wakeup((event_t)&c_compressor_swap_trigger);
2137 }
2138 lck_mtx_unlock_always(c_list_lock);
2139 }
2140 }
2141
2142
2143 void
2144 vm_consider_swapping()
2145 {
2146 c_segment_t c_seg, c_seg_next;
2147 clock_sec_t now;
2148 clock_nsec_t nsec;
2149
2150 assert(VM_CONFIG_SWAP_IS_PRESENT);
2151
2152 lck_mtx_lock_spin_always(c_list_lock);
2153
2154 compaction_swapper_abort = 1;
2155
2156 while (compaction_swapper_running) {
2157 assert_wait((event_t)&compaction_swapper_running, THREAD_UNINT);
2158
2159 lck_mtx_unlock_always(c_list_lock);
2160
2161 thread_block(THREAD_CONTINUE_NULL);
2162
2163 lck_mtx_lock_spin_always(c_list_lock);
2164 }
2165 compaction_swapper_abort = 0;
2166 compaction_swapper_running = 1;
2167
2168 vm_swapout_ripe_segments = TRUE;
2169
2170 if (!queue_empty(&c_major_list_head)) {
2171 clock_get_system_nanotime(&now, &nsec);
2172
2173 c_seg = (c_segment_t)queue_first(&c_major_list_head);
2174
2175 while (!queue_end(&c_major_list_head, (queue_entry_t)c_seg)) {
2176 if (c_overage_swapped_count >= c_overage_swapped_limit) {
2177 break;
2178 }
2179
2180 c_seg_next = (c_segment_t) queue_next(&c_seg->c_age_list);
2181
2182 if ((now - c_seg->c_creation_ts) >= vm_ripe_target_age) {
2183 lck_mtx_lock_spin_always(&c_seg->c_lock);
2184
2185 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
2186
2187 lck_mtx_unlock_always(&c_seg->c_lock);
2188 }
2189 c_seg = c_seg_next;
2190 }
2191 }
2192 vm_compressor_compact_and_swap(FALSE);
2193
2194 compaction_swapper_running = 0;
2195
2196 vm_swapout_ripe_segments = FALSE;
2197
2198 lck_mtx_unlock_always(c_list_lock);
2199
2200 thread_wakeup((event_t)&compaction_swapper_running);
2201 }
2202
2203
2204 void
2205 vm_consider_waking_compactor_swapper(void)
2206 {
2207 boolean_t need_wakeup = FALSE;
2208
2209 if (c_segment_count == 0) {
2210 return;
2211 }
2212
2213 if (compaction_swapper_running || compaction_swapper_awakened) {
2214 return;
2215 }
2216
2217 if (!compaction_swapper_inited && !compaction_swapper_init_now) {
2218 compaction_swapper_init_now = 1;
2219 need_wakeup = TRUE;
2220 }
2221
2222 if (c_minor_count && (COMPRESSOR_NEEDS_TO_MINOR_COMPACT())) {
2223 need_wakeup = TRUE;
2224 } else if (compressor_needs_to_swap()) {
2225 need_wakeup = TRUE;
2226 } else if (c_minor_count) {
2227 uint64_t total_bytes;
2228
2229 total_bytes = compressor_object->resident_page_count * PAGE_SIZE_64;
2230
2231 if ((total_bytes - compressor_bytes_used) > total_bytes / 10) {
2232 need_wakeup = TRUE;
2233 }
2234 }
2235 if (need_wakeup == TRUE) {
2236 lck_mtx_lock_spin_always(c_list_lock);
2237
2238 fastwake_warmup = FALSE;
2239
2240 if (compaction_swapper_running == 0 && compaction_swapper_awakened == 0) {
2241 memoryshot(VM_WAKEUP_COMPACTOR_SWAPPER, DBG_FUNC_NONE);
2242
2243 compaction_swapper_awakened = 1;
2244 thread_wakeup((event_t)&c_compressor_swap_trigger);
2245 }
2246 lck_mtx_unlock_always(c_list_lock);
2247 }
2248 }
2249
2250
2251 #define C_SWAPOUT_LIMIT 4
2252 #define DELAYED_COMPACTIONS_PER_PASS 30
2253
2254 void
2255 vm_compressor_do_delayed_compactions(boolean_t flush_all)
2256 {
2257 c_segment_t c_seg;
2258 int number_compacted = 0;
2259 boolean_t needs_to_swap = FALSE;
2260
2261
2262 #if !CONFIG_EMBEDDED
2263 LCK_MTX_ASSERT(c_list_lock, LCK_MTX_ASSERT_OWNED);
2264 #endif /* !CONFIG_EMBEDDED */
2265
2266 while (!queue_empty(&c_minor_list_head) && needs_to_swap == FALSE) {
2267 c_seg = (c_segment_t)queue_first(&c_minor_list_head);
2268
2269 lck_mtx_lock_spin_always(&c_seg->c_lock);
2270
2271 if (c_seg->c_busy) {
2272 lck_mtx_unlock_always(c_list_lock);
2273 c_seg_wait_on_busy(c_seg);
2274 lck_mtx_lock_spin_always(c_list_lock);
2275
2276 continue;
2277 }
2278 C_SEG_BUSY(c_seg);
2279
2280 c_seg_do_minor_compaction_and_unlock(c_seg, TRUE, FALSE, TRUE);
2281
2282 if (VM_CONFIG_SWAP_IS_ACTIVE && (number_compacted++ > DELAYED_COMPACTIONS_PER_PASS)) {
2283 if ((flush_all == TRUE || compressor_needs_to_swap() == TRUE) && c_swapout_count < C_SWAPOUT_LIMIT) {
2284 needs_to_swap = TRUE;
2285 }
2286
2287 number_compacted = 0;
2288 }
2289 lck_mtx_lock_spin_always(c_list_lock);
2290 }
2291 }
2292
2293
2294 #define C_SEGMENT_SWAPPEDIN_AGE_LIMIT 10
2295
2296 static void
2297 vm_compressor_age_swapped_in_segments(boolean_t flush_all)
2298 {
2299 c_segment_t c_seg;
2300 clock_sec_t now;
2301 clock_nsec_t nsec;
2302
2303 clock_get_system_nanotime(&now, &nsec);
2304
2305 while (!queue_empty(&c_swappedin_list_head)) {
2306 c_seg = (c_segment_t)queue_first(&c_swappedin_list_head);
2307
2308 if (flush_all == FALSE && (now - c_seg->c_swappedin_ts) < C_SEGMENT_SWAPPEDIN_AGE_LIMIT) {
2309 break;
2310 }
2311
2312 lck_mtx_lock_spin_always(&c_seg->c_lock);
2313
2314 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
2315
2316 lck_mtx_unlock_always(&c_seg->c_lock);
2317 }
2318 }
2319
2320
2321 extern int vm_num_swap_files;
2322 extern int vm_num_pinned_swap_files;
2323 extern int vm_swappin_enabled;
2324
2325 extern unsigned int vm_swapfile_total_segs_used;
2326 extern unsigned int vm_swapfile_total_segs_alloced;
2327
2328
2329 void
2330 vm_compressor_flush(void)
2331 {
2332 uint64_t vm_swap_put_failures_at_start;
2333 wait_result_t wait_result = 0;
2334 AbsoluteTime startTime, endTime;
2335 clock_sec_t now_sec;
2336 clock_nsec_t now_nsec;
2337 uint64_t nsec;
2338
2339 HIBLOG("vm_compressor_flush - starting\n");
2340
2341 clock_get_uptime(&startTime);
2342
2343 lck_mtx_lock_spin_always(c_list_lock);
2344
2345 fastwake_warmup = FALSE;
2346 compaction_swapper_abort = 1;
2347
2348 while (compaction_swapper_running) {
2349 assert_wait((event_t)&compaction_swapper_running, THREAD_UNINT);
2350
2351 lck_mtx_unlock_always(c_list_lock);
2352
2353 thread_block(THREAD_CONTINUE_NULL);
2354
2355 lck_mtx_lock_spin_always(c_list_lock);
2356 }
2357 compaction_swapper_abort = 0;
2358 compaction_swapper_running = 1;
2359
2360 hibernate_flushing = TRUE;
2361 hibernate_no_swapspace = FALSE;
2362 c_generation_id_flush_barrier = c_generation_id + 1000;
2363
2364 clock_get_system_nanotime(&now_sec, &now_nsec);
2365 hibernate_flushing_deadline = now_sec + HIBERNATE_FLUSHING_SECS_TO_COMPLETE;
2366
2367 vm_swap_put_failures_at_start = vm_swap_put_failures;
2368
2369 vm_compressor_compact_and_swap(TRUE);
2370
2371 while (!queue_empty(&c_swapout_list_head)) {
2372 assert_wait_timeout((event_t) &compaction_swapper_running, THREAD_INTERRUPTIBLE, 5000, 1000 * NSEC_PER_USEC);
2373
2374 lck_mtx_unlock_always(c_list_lock);
2375
2376 wait_result = thread_block(THREAD_CONTINUE_NULL);
2377
2378 lck_mtx_lock_spin_always(c_list_lock);
2379
2380 if (wait_result == THREAD_TIMED_OUT) {
2381 break;
2382 }
2383 }
2384 hibernate_flushing = FALSE;
2385 compaction_swapper_running = 0;
2386
2387 if (vm_swap_put_failures > vm_swap_put_failures_at_start) {
2388 HIBLOG("vm_compressor_flush failed to clean %llu segments - vm_page_compressor_count(%d)\n",
2389 vm_swap_put_failures - vm_swap_put_failures_at_start, VM_PAGE_COMPRESSOR_COUNT);
2390 }
2391
2392 lck_mtx_unlock_always(c_list_lock);
2393
2394 thread_wakeup((event_t)&compaction_swapper_running);
2395
2396 clock_get_uptime(&endTime);
2397 SUB_ABSOLUTETIME(&endTime, &startTime);
2398 absolutetime_to_nanoseconds(endTime, &nsec);
2399
2400 HIBLOG("vm_compressor_flush completed - took %qd msecs - vm_num_swap_files = %d, vm_num_pinned_swap_files = %d, vm_swappin_enabled = %d\n",
2401 nsec / 1000000ULL, vm_num_swap_files, vm_num_pinned_swap_files, vm_swappin_enabled);
2402 }
2403
2404
2405 int compaction_swap_trigger_thread_awakened = 0;
2406
2407 static void
2408 vm_compressor_swap_trigger_thread(void)
2409 {
2410 current_thread()->options |= TH_OPT_VMPRIV;
2411
2412 /*
2413 * compaction_swapper_init_now is set when the first call to
2414 * vm_consider_waking_compactor_swapper is made from
2415 * vm_pageout_scan... since this function is called upon
2416 * thread creation, we want to make sure to delay adjusting
2417 * the tuneables until we are awakened via vm_pageout_scan
2418 * so that we are at a point where the vm_swapfile_open will
2419 * be operating on the correct directory (in case the default
2420 * of /var/vm/ is overridden by the dymanic_pager
2421 */
2422 if (compaction_swapper_init_now) {
2423 vm_compaction_swapper_do_init();
2424
2425 if (vm_pageout_state.vm_restricted_to_single_processor == TRUE) {
2426 thread_vm_bind_group_add();
2427 }
2428 thread_set_thread_name(current_thread(), "VM_cswap_trigger");
2429 compaction_swapper_init_now = 0;
2430 }
2431 lck_mtx_lock_spin_always(c_list_lock);
2432
2433 compaction_swap_trigger_thread_awakened++;
2434 compaction_swapper_awakened = 0;
2435
2436 if (compaction_swapper_running == 0) {
2437 compaction_swapper_running = 1;
2438
2439 vm_compressor_compact_and_swap(FALSE);
2440
2441 compaction_swapper_running = 0;
2442 }
2443 assert_wait((event_t)&c_compressor_swap_trigger, THREAD_UNINT);
2444
2445 if (compaction_swapper_running == 0) {
2446 thread_wakeup((event_t)&compaction_swapper_running);
2447 }
2448
2449 lck_mtx_unlock_always(c_list_lock);
2450
2451 thread_block((thread_continue_t)vm_compressor_swap_trigger_thread);
2452
2453 /* NOTREACHED */
2454 }
2455
2456
2457 void
2458 vm_compressor_record_warmup_start(void)
2459 {
2460 c_segment_t c_seg;
2461
2462 lck_mtx_lock_spin_always(c_list_lock);
2463
2464 if (first_c_segment_to_warm_generation_id == 0) {
2465 if (!queue_empty(&c_age_list_head)) {
2466 c_seg = (c_segment_t)queue_last(&c_age_list_head);
2467
2468 first_c_segment_to_warm_generation_id = c_seg->c_generation_id;
2469 } else {
2470 first_c_segment_to_warm_generation_id = 0;
2471 }
2472
2473 fastwake_recording_in_progress = TRUE;
2474 }
2475 lck_mtx_unlock_always(c_list_lock);
2476 }
2477
2478
2479 void
2480 vm_compressor_record_warmup_end(void)
2481 {
2482 c_segment_t c_seg;
2483
2484 lck_mtx_lock_spin_always(c_list_lock);
2485
2486 if (fastwake_recording_in_progress == TRUE) {
2487 if (!queue_empty(&c_age_list_head)) {
2488 c_seg = (c_segment_t)queue_last(&c_age_list_head);
2489
2490 last_c_segment_to_warm_generation_id = c_seg->c_generation_id;
2491 } else {
2492 last_c_segment_to_warm_generation_id = first_c_segment_to_warm_generation_id;
2493 }
2494
2495 fastwake_recording_in_progress = FALSE;
2496
2497 HIBLOG("vm_compressor_record_warmup (%qd - %qd)\n", first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id);
2498 }
2499 lck_mtx_unlock_always(c_list_lock);
2500 }
2501
2502
2503 #define DELAY_TRIM_ON_WAKE_SECS 25
2504
2505 void
2506 vm_compressor_delay_trim(void)
2507 {
2508 clock_sec_t sec;
2509 clock_nsec_t nsec;
2510
2511 clock_get_system_nanotime(&sec, &nsec);
2512 dont_trim_until_ts = sec + DELAY_TRIM_ON_WAKE_SECS;
2513 }
2514
2515
2516 void
2517 vm_compressor_do_warmup(void)
2518 {
2519 lck_mtx_lock_spin_always(c_list_lock);
2520
2521 if (first_c_segment_to_warm_generation_id == last_c_segment_to_warm_generation_id) {
2522 first_c_segment_to_warm_generation_id = last_c_segment_to_warm_generation_id = 0;
2523
2524 lck_mtx_unlock_always(c_list_lock);
2525 return;
2526 }
2527
2528 if (compaction_swapper_running == 0 && compaction_swapper_awakened == 0) {
2529 fastwake_warmup = TRUE;
2530
2531 compaction_swapper_awakened = 1;
2532 thread_wakeup((event_t)&c_compressor_swap_trigger);
2533 }
2534 lck_mtx_unlock_always(c_list_lock);
2535 }
2536
2537 void
2538 do_fastwake_warmup_all(void)
2539 {
2540 lck_mtx_lock_spin_always(c_list_lock);
2541
2542 if (queue_empty(&c_swappedout_list_head) && queue_empty(&c_swappedout_sparse_list_head)) {
2543 lck_mtx_unlock_always(c_list_lock);
2544 return;
2545 }
2546
2547 fastwake_warmup = TRUE;
2548
2549 do_fastwake_warmup(&c_swappedout_list_head, TRUE);
2550
2551 do_fastwake_warmup(&c_swappedout_sparse_list_head, TRUE);
2552
2553 fastwake_warmup = FALSE;
2554
2555 lck_mtx_unlock_always(c_list_lock);
2556 }
2557
2558 void
2559 do_fastwake_warmup(queue_head_t *c_queue, boolean_t consider_all_cseg)
2560 {
2561 c_segment_t c_seg = NULL;
2562 AbsoluteTime startTime, endTime;
2563 uint64_t nsec;
2564
2565
2566 HIBLOG("vm_compressor_fastwake_warmup (%qd - %qd) - starting\n", first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id);
2567
2568 clock_get_uptime(&startTime);
2569
2570 lck_mtx_unlock_always(c_list_lock);
2571
2572 proc_set_thread_policy(current_thread(),
2573 TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER2);
2574
2575 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2576
2577 lck_mtx_lock_spin_always(c_list_lock);
2578
2579 while (!queue_empty(c_queue) && fastwake_warmup == TRUE) {
2580 c_seg = (c_segment_t) queue_first(c_queue);
2581
2582 if (consider_all_cseg == FALSE) {
2583 if (c_seg->c_generation_id < first_c_segment_to_warm_generation_id ||
2584 c_seg->c_generation_id > last_c_segment_to_warm_generation_id) {
2585 break;
2586 }
2587
2588 if (vm_page_free_count < (AVAILABLE_MEMORY / 4)) {
2589 break;
2590 }
2591 }
2592
2593 lck_mtx_lock_spin_always(&c_seg->c_lock);
2594 lck_mtx_unlock_always(c_list_lock);
2595
2596 if (c_seg->c_busy) {
2597 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2598 c_seg_wait_on_busy(c_seg);
2599 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2600 } else {
2601 if (c_seg_swapin(c_seg, TRUE, FALSE) == 0) {
2602 lck_mtx_unlock_always(&c_seg->c_lock);
2603 }
2604 c_segment_warmup_count++;
2605
2606 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2607 vm_pageout_io_throttle();
2608 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2609 }
2610 lck_mtx_lock_spin_always(c_list_lock);
2611 }
2612 lck_mtx_unlock_always(c_list_lock);
2613
2614 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2615
2616 proc_set_thread_policy(current_thread(),
2617 TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER0);
2618
2619 clock_get_uptime(&endTime);
2620 SUB_ABSOLUTETIME(&endTime, &startTime);
2621 absolutetime_to_nanoseconds(endTime, &nsec);
2622
2623 HIBLOG("vm_compressor_fastwake_warmup completed - took %qd msecs\n", nsec / 1000000ULL);
2624
2625 lck_mtx_lock_spin_always(c_list_lock);
2626
2627 if (consider_all_cseg == FALSE) {
2628 first_c_segment_to_warm_generation_id = last_c_segment_to_warm_generation_id = 0;
2629 }
2630 }
2631
2632
2633 void
2634 vm_compressor_compact_and_swap(boolean_t flush_all)
2635 {
2636 c_segment_t c_seg, c_seg_next;
2637 boolean_t keep_compacting;
2638 clock_sec_t now;
2639 clock_nsec_t nsec;
2640
2641
2642 if (fastwake_warmup == TRUE) {
2643 uint64_t starting_warmup_count;
2644
2645 starting_warmup_count = c_segment_warmup_count;
2646
2647 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 11) | DBG_FUNC_START, c_segment_warmup_count,
2648 first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id, 0, 0);
2649 do_fastwake_warmup(&c_swappedout_list_head, FALSE);
2650 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 11) | DBG_FUNC_END, c_segment_warmup_count, c_segment_warmup_count - starting_warmup_count, 0, 0, 0);
2651
2652 fastwake_warmup = FALSE;
2653 }
2654
2655 /*
2656 * it's possible for the c_age_list_head to be empty if we
2657 * hit our limits for growing the compressor pool and we subsequently
2658 * hibernated... on the next hibernation we could see the queue as
2659 * empty and not proceeed even though we have a bunch of segments on
2660 * the swapped in queue that need to be dealt with.
2661 */
2662 vm_compressor_do_delayed_compactions(flush_all);
2663
2664 vm_compressor_age_swapped_in_segments(flush_all);
2665
2666 /*
2667 * we only need to grab the timestamp once per
2668 * invocation of this function since the
2669 * timescale we're interested in is measured
2670 * in days
2671 */
2672 clock_get_system_nanotime(&now, &nsec);
2673
2674 while (!queue_empty(&c_age_list_head) && compaction_swapper_abort == 0) {
2675 if (hibernate_flushing == TRUE) {
2676 clock_sec_t sec;
2677
2678 if (hibernate_should_abort()) {
2679 HIBLOG("vm_compressor_flush - hibernate_should_abort returned TRUE\n");
2680 break;
2681 }
2682 if (hibernate_no_swapspace == TRUE) {
2683 HIBLOG("vm_compressor_flush - out of swap space\n");
2684 break;
2685 }
2686 if (vm_swap_files_pinned() == FALSE) {
2687 HIBLOG("vm_compressor_flush - unpinned swap files\n");
2688 break;
2689 }
2690 if (hibernate_in_progress_with_pinned_swap == TRUE &&
2691 (vm_swapfile_total_segs_alloced == vm_swapfile_total_segs_used)) {
2692 HIBLOG("vm_compressor_flush - out of pinned swap space\n");
2693 break;
2694 }
2695 clock_get_system_nanotime(&sec, &nsec);
2696
2697 if (sec > hibernate_flushing_deadline) {
2698 HIBLOG("vm_compressor_flush - failed to finish before deadline\n");
2699 break;
2700 }
2701 }
2702 if (c_swapout_count >= C_SWAPOUT_LIMIT) {
2703 assert_wait_timeout((event_t) &compaction_swapper_running, THREAD_INTERRUPTIBLE, 100, 1000 * NSEC_PER_USEC);
2704
2705 lck_mtx_unlock_always(c_list_lock);
2706
2707 thread_block(THREAD_CONTINUE_NULL);
2708
2709 lck_mtx_lock_spin_always(c_list_lock);
2710 }
2711 /*
2712 * Minor compactions
2713 */
2714 vm_compressor_do_delayed_compactions(flush_all);
2715
2716 vm_compressor_age_swapped_in_segments(flush_all);
2717
2718 if (c_swapout_count >= C_SWAPOUT_LIMIT) {
2719 /*
2720 * we timed out on the above thread_block
2721 * let's loop around and try again
2722 * the timeout allows us to continue
2723 * to do minor compactions to make
2724 * more memory available
2725 */
2726 continue;
2727 }
2728
2729 /*
2730 * Swap out segments?
2731 */
2732 if (flush_all == FALSE) {
2733 boolean_t needs_to_swap;
2734
2735 lck_mtx_unlock_always(c_list_lock);
2736
2737 needs_to_swap = compressor_needs_to_swap();
2738
2739 #if !CONFIG_EMBEDDED
2740 if (needs_to_swap == TRUE && vm_swap_low_on_space()) {
2741 vm_compressor_take_paging_space_action();
2742 }
2743 #endif /* !CONFIG_EMBEDDED */
2744
2745 lck_mtx_lock_spin_always(c_list_lock);
2746
2747 if (needs_to_swap == FALSE) {
2748 break;
2749 }
2750 }
2751 if (queue_empty(&c_age_list_head)) {
2752 break;
2753 }
2754 c_seg = (c_segment_t) queue_first(&c_age_list_head);
2755
2756 assert(c_seg->c_state == C_ON_AGE_Q);
2757
2758 if (flush_all == TRUE && c_seg->c_generation_id > c_generation_id_flush_barrier) {
2759 break;
2760 }
2761
2762 lck_mtx_lock_spin_always(&c_seg->c_lock);
2763
2764 if (c_seg->c_busy) {
2765 lck_mtx_unlock_always(c_list_lock);
2766 c_seg_wait_on_busy(c_seg);
2767 lck_mtx_lock_spin_always(c_list_lock);
2768
2769 continue;
2770 }
2771 C_SEG_BUSY(c_seg);
2772
2773 if (c_seg_do_minor_compaction_and_unlock(c_seg, FALSE, TRUE, TRUE)) {
2774 /*
2775 * found an empty c_segment and freed it
2776 * so go grab the next guy in the queue
2777 */
2778 c_seg_major_compact_stats.count_of_freed_segs++;
2779 continue;
2780 }
2781 /*
2782 * Major compaction
2783 */
2784 keep_compacting = TRUE;
2785
2786 while (keep_compacting == TRUE) {
2787 assert(c_seg->c_busy);
2788
2789 /* look for another segment to consolidate */
2790
2791 c_seg_next = (c_segment_t) queue_next(&c_seg->c_age_list);
2792
2793 if (queue_end(&c_age_list_head, (queue_entry_t)c_seg_next)) {
2794 break;
2795 }
2796
2797 assert(c_seg_next->c_state == C_ON_AGE_Q);
2798
2799 if (c_seg_major_compact_ok(c_seg, c_seg_next) == FALSE) {
2800 break;
2801 }
2802
2803 lck_mtx_lock_spin_always(&c_seg_next->c_lock);
2804
2805 if (c_seg_next->c_busy) {
2806 lck_mtx_unlock_always(c_list_lock);
2807 c_seg_wait_on_busy(c_seg_next);
2808 lck_mtx_lock_spin_always(c_list_lock);
2809
2810 continue;
2811 }
2812 /* grab that segment */
2813 C_SEG_BUSY(c_seg_next);
2814
2815 if (c_seg_do_minor_compaction_and_unlock(c_seg_next, FALSE, TRUE, TRUE)) {
2816 /*
2817 * found an empty c_segment and freed it
2818 * so we can't continue to use c_seg_next
2819 */
2820 c_seg_major_compact_stats.count_of_freed_segs++;
2821 continue;
2822 }
2823
2824 /* unlock the list ... */
2825 lck_mtx_unlock_always(c_list_lock);
2826
2827 /* do the major compaction */
2828
2829 keep_compacting = c_seg_major_compact(c_seg, c_seg_next);
2830
2831 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2832
2833 lck_mtx_lock_spin_always(&c_seg_next->c_lock);
2834 /*
2835 * run a minor compaction on the donor segment
2836 * since we pulled at least some of it's
2837 * data into our target... if we've emptied
2838 * it, now is a good time to free it which
2839 * c_seg_minor_compaction_and_unlock also takes care of
2840 *
2841 * by passing TRUE, we ask for c_busy to be cleared
2842 * and c_wanted to be taken care of
2843 */
2844 if (c_seg_minor_compaction_and_unlock(c_seg_next, TRUE)) {
2845 c_seg_major_compact_stats.count_of_freed_segs++;
2846 }
2847
2848 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2849
2850 /* relock the list */
2851 lck_mtx_lock_spin_always(c_list_lock);
2852 } /* major compaction */
2853
2854 lck_mtx_lock_spin_always(&c_seg->c_lock);
2855
2856 assert(c_seg->c_busy);
2857 assert(!c_seg->c_on_minorcompact_q);
2858
2859 if (VM_CONFIG_SWAP_IS_ACTIVE) {
2860 /*
2861 * This mode of putting a generic c_seg on the swapout list is
2862 * only supported when we have general swapping enabled
2863 */
2864 c_seg_switch_state(c_seg, C_ON_SWAPOUT_Q, FALSE);
2865 } else {
2866 if ((vm_swapout_ripe_segments == TRUE && c_overage_swapped_count < c_overage_swapped_limit)) {
2867 assert(VM_CONFIG_SWAP_IS_PRESENT);
2868 /*
2869 * we are running compressor sweeps with swap-behind
2870 * make sure the c_seg has aged enough before swapping it
2871 * out...
2872 */
2873 if ((now - c_seg->c_creation_ts) >= vm_ripe_target_age) {
2874 c_seg->c_overage_swap = TRUE;
2875 c_overage_swapped_count++;
2876 c_seg_switch_state(c_seg, C_ON_SWAPOUT_Q, FALSE);
2877 }
2878 }
2879 }
2880 if (c_seg->c_state == C_ON_AGE_Q) {
2881 /*
2882 * this c_seg didn't get moved to the swapout queue
2883 * so we need to move it out of the way...
2884 * we just did a major compaction on it so put it
2885 * on that queue
2886 */
2887 c_seg_switch_state(c_seg, C_ON_MAJORCOMPACT_Q, FALSE);
2888 } else {
2889 c_seg_major_compact_stats.wasted_space_in_swapouts += C_SEG_BUFSIZE - c_seg->c_bytes_used;
2890 c_seg_major_compact_stats.count_of_swapouts++;
2891 }
2892 C_SEG_WAKEUP_DONE(c_seg);
2893
2894 lck_mtx_unlock_always(&c_seg->c_lock);
2895
2896 if (c_swapout_count) {
2897 lck_mtx_unlock_always(c_list_lock);
2898
2899 thread_wakeup((event_t)&c_swapout_list_head);
2900
2901 lck_mtx_lock_spin_always(c_list_lock);
2902 }
2903 }
2904 }
2905
2906
2907 static c_segment_t
2908 c_seg_allocate(c_segment_t *current_chead)
2909 {
2910 c_segment_t c_seg;
2911 int min_needed;
2912 int size_to_populate;
2913
2914 #if !CONFIG_EMBEDDED
2915 if (vm_compressor_low_on_space()) {
2916 vm_compressor_take_paging_space_action();
2917 }
2918 #endif /* !CONFIG_EMBEDDED */
2919
2920 if ((c_seg = *current_chead) == NULL) {
2921 uint32_t c_segno;
2922
2923 lck_mtx_lock_spin_always(c_list_lock);
2924
2925 while (c_segments_busy == TRUE) {
2926 assert_wait((event_t) (&c_segments_busy), THREAD_UNINT);
2927
2928 lck_mtx_unlock_always(c_list_lock);
2929
2930 thread_block(THREAD_CONTINUE_NULL);
2931
2932 lck_mtx_lock_spin_always(c_list_lock);
2933 }
2934 if (c_free_segno_head == (uint32_t)-1) {
2935 uint32_t c_segments_available_new;
2936
2937 if (c_segments_available >= c_segments_limit || c_segment_pages_compressed >= c_segment_pages_compressed_limit) {
2938 lck_mtx_unlock_always(c_list_lock);
2939
2940 return NULL;
2941 }
2942 c_segments_busy = TRUE;
2943 lck_mtx_unlock_always(c_list_lock);
2944
2945 kernel_memory_populate(compressor_map, (vm_offset_t)c_segments_next_page,
2946 PAGE_SIZE, KMA_KOBJECT, VM_KERN_MEMORY_COMPRESSOR);
2947 c_segments_next_page += PAGE_SIZE;
2948
2949 c_segments_available_new = c_segments_available + C_SEGMENTS_PER_PAGE;
2950
2951 if (c_segments_available_new > c_segments_limit) {
2952 c_segments_available_new = c_segments_limit;
2953 }
2954
2955 for (c_segno = c_segments_available + 1; c_segno < c_segments_available_new; c_segno++) {
2956 c_segments[c_segno - 1].c_segno = c_segno;
2957 }
2958
2959 lck_mtx_lock_spin_always(c_list_lock);
2960
2961 c_segments[c_segno - 1].c_segno = c_free_segno_head;
2962 c_free_segno_head = c_segments_available;
2963 c_segments_available = c_segments_available_new;
2964
2965 c_segments_busy = FALSE;
2966 thread_wakeup((event_t) (&c_segments_busy));
2967 }
2968 c_segno = c_free_segno_head;
2969 assert(c_segno >= 0 && c_segno < c_segments_limit);
2970
2971 c_free_segno_head = (uint32_t)c_segments[c_segno].c_segno;
2972
2973 /*
2974 * do the rest of the bookkeeping now while we're still behind
2975 * the list lock and grab our generation id now into a local
2976 * so that we can install it once we have the c_seg allocated
2977 */
2978 c_segment_count++;
2979 if (c_segment_count > c_segment_count_max) {
2980 c_segment_count_max = c_segment_count;
2981 }
2982
2983 lck_mtx_unlock_always(c_list_lock);
2984
2985 c_seg = (c_segment_t)zalloc(compressor_segment_zone);
2986 bzero((char *)c_seg, sizeof(struct c_segment));
2987
2988 c_seg->c_store.c_buffer = (int32_t *)C_SEG_BUFFER_ADDRESS(c_segno);
2989
2990 lck_mtx_init(&c_seg->c_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
2991
2992 c_seg->c_state = C_IS_EMPTY;
2993 c_seg->c_firstemptyslot = C_SLOT_MAX_INDEX;
2994 c_seg->c_mysegno = c_segno;
2995
2996 lck_mtx_lock_spin_always(c_list_lock);
2997 c_empty_count++;
2998 c_seg_switch_state(c_seg, C_IS_FILLING, FALSE);
2999 c_segments[c_segno].c_seg = c_seg;
3000 assert(c_segments[c_segno].c_segno > c_segments_available);
3001 lck_mtx_unlock_always(c_list_lock);
3002
3003 *current_chead = c_seg;
3004
3005 #if DEVELOPMENT || DEBUG
3006 C_SEG_MAKE_WRITEABLE(c_seg);
3007 #endif
3008 }
3009 c_seg_alloc_nextslot(c_seg);
3010
3011 size_to_populate = C_SEG_ALLOCSIZE - C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset);
3012
3013 if (size_to_populate) {
3014 min_needed = PAGE_SIZE + (C_SEG_ALLOCSIZE - C_SEG_BUFSIZE);
3015
3016 if (C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset - c_seg->c_nextoffset) < (unsigned) min_needed) {
3017 if (size_to_populate > C_SEG_MAX_POPULATE_SIZE) {
3018 size_to_populate = C_SEG_MAX_POPULATE_SIZE;
3019 }
3020
3021 OSAddAtomic64(size_to_populate / PAGE_SIZE, &vm_pageout_vminfo.vm_compressor_pages_grabbed);
3022
3023 kernel_memory_populate(compressor_map,
3024 (vm_offset_t) &c_seg->c_store.c_buffer[c_seg->c_populated_offset],
3025 size_to_populate,
3026 KMA_COMPRESSOR,
3027 VM_KERN_MEMORY_COMPRESSOR);
3028 } else {
3029 size_to_populate = 0;
3030 }
3031 }
3032 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3033
3034 lck_mtx_lock_spin_always(&c_seg->c_lock);
3035
3036 if (size_to_populate) {
3037 c_seg->c_populated_offset += C_SEG_BYTES_TO_OFFSET(size_to_populate);
3038 }
3039
3040 return c_seg;
3041 }
3042
3043 #if DEVELOPMENT || DEBUG
3044 #if CONFIG_FREEZE
3045 extern boolean_t memorystatus_freeze_to_memory;
3046 #endif /* CONFIG_FREEZE */
3047 #endif /* DEVELOPMENT || DEBUG */
3048
3049 static void
3050 c_current_seg_filled(c_segment_t c_seg, c_segment_t *current_chead)
3051 {
3052 uint32_t unused_bytes;
3053 uint32_t offset_to_depopulate;
3054 int new_state = C_ON_AGE_Q;
3055 clock_sec_t sec;
3056 clock_nsec_t nsec;
3057 boolean_t head_insert = FALSE;
3058
3059 unused_bytes = trunc_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset - c_seg->c_nextoffset));
3060
3061 if (unused_bytes) {
3062 offset_to_depopulate = C_SEG_BYTES_TO_OFFSET(round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_nextoffset)));
3063
3064 /*
3065 * release the extra physical page(s) at the end of the segment
3066 */
3067 lck_mtx_unlock_always(&c_seg->c_lock);
3068
3069 kernel_memory_depopulate(
3070 compressor_map,
3071 (vm_offset_t) &c_seg->c_store.c_buffer[offset_to_depopulate],
3072 unused_bytes,
3073 KMA_COMPRESSOR);
3074
3075 lck_mtx_lock_spin_always(&c_seg->c_lock);
3076
3077 c_seg->c_populated_offset = offset_to_depopulate;
3078 }
3079 assert(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset) <= C_SEG_BUFSIZE);
3080
3081 #if DEVELOPMENT || DEBUG
3082 {
3083 boolean_t c_seg_was_busy = FALSE;
3084
3085 if (!c_seg->c_busy) {
3086 C_SEG_BUSY(c_seg);
3087 } else {
3088 c_seg_was_busy = TRUE;
3089 }
3090
3091 lck_mtx_unlock_always(&c_seg->c_lock);
3092
3093 C_SEG_WRITE_PROTECT(c_seg);
3094
3095 lck_mtx_lock_spin_always(&c_seg->c_lock);
3096
3097 if (c_seg_was_busy == FALSE) {
3098 C_SEG_WAKEUP_DONE(c_seg);
3099 }
3100 }
3101 #endif
3102
3103 #if CONFIG_FREEZE
3104 if (current_chead == (c_segment_t*)&freezer_chead &&
3105 VM_CONFIG_SWAP_IS_PRESENT &&
3106 VM_CONFIG_FREEZER_SWAP_IS_ACTIVE
3107 #if DEVELOPMENT || DEBUG
3108 && !memorystatus_freeze_to_memory
3109 #endif /* DEVELOPMENT || DEBUG */
3110 ) {
3111 new_state = C_ON_SWAPOUT_Q;
3112 }
3113 #endif /* CONFIG_FREEZE */
3114
3115 if (vm_darkwake_mode == TRUE) {
3116 new_state = C_ON_SWAPOUT_Q;
3117 head_insert = TRUE;
3118 }
3119
3120 clock_get_system_nanotime(&sec, &nsec);
3121 c_seg->c_creation_ts = (uint32_t)sec;
3122
3123 lck_mtx_lock_spin_always(c_list_lock);
3124
3125 c_seg->c_generation_id = c_generation_id++;
3126 c_seg_switch_state(c_seg, new_state, head_insert);
3127
3128 #if CONFIG_FREEZE
3129 if (c_seg->c_state == C_ON_SWAPOUT_Q) {
3130 /*
3131 * darkwake and freezer can't co-exist together
3132 * We'll need to fix this accounting as a start.
3133 */
3134 assert(vm_darkwake_mode == FALSE);
3135 c_freezer_swapout_page_count += (C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset)) / PAGE_SIZE_64;
3136 }
3137 #endif /* CONFIG_FREEZE */
3138
3139 if (c_seg->c_state == C_ON_AGE_Q && C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE) {
3140 c_seg_need_delayed_compaction(c_seg, TRUE);
3141 }
3142
3143 lck_mtx_unlock_always(c_list_lock);
3144
3145 if (c_seg->c_state == C_ON_SWAPOUT_Q) {
3146 thread_wakeup((event_t)&c_swapout_list_head);
3147 }
3148
3149 *current_chead = NULL;
3150 }
3151
3152
3153 /*
3154 * returns with c_seg locked
3155 */
3156 void
3157 c_seg_swapin_requeue(c_segment_t c_seg, boolean_t has_data, boolean_t minor_compact_ok, boolean_t age_on_swapin_q)
3158 {
3159 clock_sec_t sec;
3160 clock_nsec_t nsec;
3161
3162 clock_get_system_nanotime(&sec, &nsec);
3163
3164 lck_mtx_lock_spin_always(c_list_lock);
3165 lck_mtx_lock_spin_always(&c_seg->c_lock);
3166
3167 assert(c_seg->c_busy_swapping);
3168 assert(c_seg->c_busy);
3169
3170 c_seg->c_busy_swapping = 0;
3171
3172 if (c_seg->c_overage_swap == TRUE) {
3173 c_overage_swapped_count--;
3174 c_seg->c_overage_swap = FALSE;
3175 }
3176 if (has_data == TRUE) {
3177 if (age_on_swapin_q == TRUE) {
3178 c_seg_switch_state(c_seg, C_ON_SWAPPEDIN_Q, FALSE);
3179 } else {
3180 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
3181 }
3182
3183 if (minor_compact_ok == TRUE && !c_seg->c_on_minorcompact_q && C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE) {
3184 c_seg_need_delayed_compaction(c_seg, TRUE);
3185 }
3186 } else {
3187 c_seg->c_store.c_buffer = (int32_t*) NULL;
3188 c_seg->c_populated_offset = C_SEG_BYTES_TO_OFFSET(0);
3189
3190 c_seg_switch_state(c_seg, C_ON_BAD_Q, FALSE);
3191 }
3192 c_seg->c_swappedin_ts = (uint32_t)sec;
3193
3194 lck_mtx_unlock_always(c_list_lock);
3195 }
3196
3197
3198
3199 /*
3200 * c_seg has to be locked and is returned locked if the c_seg isn't freed
3201 * PAGE_REPLACMENT_DISALLOWED has to be TRUE on entry and is returned TRUE
3202 * c_seg_swapin returns 1 if the c_seg was freed, 0 otherwise
3203 */
3204
3205 int
3206 c_seg_swapin(c_segment_t c_seg, boolean_t force_minor_compaction, boolean_t age_on_swapin_q)
3207 {
3208 vm_offset_t addr = 0;
3209 uint32_t io_size = 0;
3210 uint64_t f_offset;
3211
3212 assert(C_SEG_IS_ONDISK(c_seg));
3213
3214 #if !CHECKSUM_THE_SWAP
3215 c_seg_trim_tail(c_seg);
3216 #endif
3217 io_size = round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset));
3218 f_offset = c_seg->c_store.c_swap_handle;
3219
3220 C_SEG_BUSY(c_seg);
3221 c_seg->c_busy_swapping = 1;
3222
3223 /*
3224 * This thread is likely going to block for I/O.
3225 * Make sure it is ready to run when the I/O completes because
3226 * it needs to clear the busy bit on the c_seg so that other
3227 * waiting threads can make progress too. To do that, boost
3228 * the rwlock_count so that the priority is boosted.
3229 */
3230 set_thread_rwlock_boost();
3231 lck_mtx_unlock_always(&c_seg->c_lock);
3232
3233 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3234
3235 addr = (vm_offset_t)C_SEG_BUFFER_ADDRESS(c_seg->c_mysegno);
3236 c_seg->c_store.c_buffer = (int32_t*) addr;
3237
3238 kernel_memory_populate(compressor_map, addr, io_size, KMA_COMPRESSOR, VM_KERN_MEMORY_COMPRESSOR);
3239
3240 if (vm_swap_get(c_seg, f_offset, io_size) != KERN_SUCCESS) {
3241 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3242
3243 kernel_memory_depopulate(compressor_map, addr, io_size, KMA_COMPRESSOR);
3244
3245 c_seg_swapin_requeue(c_seg, FALSE, TRUE, age_on_swapin_q);
3246 } else {
3247 #if ENCRYPTED_SWAP
3248 vm_swap_decrypt(c_seg);
3249 #endif /* ENCRYPTED_SWAP */
3250
3251 #if CHECKSUM_THE_SWAP
3252 if (c_seg->cseg_swap_size != io_size) {
3253 panic("swapin size doesn't match swapout size");
3254 }
3255
3256 if (c_seg->cseg_hash != vmc_hash((char*) c_seg->c_store.c_buffer, (int)io_size)) {
3257 panic("c_seg_swapin - Swap hash mismatch\n");
3258 }
3259 #endif /* CHECKSUM_THE_SWAP */
3260
3261 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3262
3263 c_seg_swapin_requeue(c_seg, TRUE, force_minor_compaction == TRUE ? FALSE : TRUE, age_on_swapin_q);
3264
3265 OSAddAtomic64(c_seg->c_bytes_used, &compressor_bytes_used);
3266
3267 if (force_minor_compaction == TRUE) {
3268 if (c_seg_minor_compaction_and_unlock(c_seg, FALSE)) {
3269 /*
3270 * c_seg was completely empty so it was freed,
3271 * so be careful not to reference it again
3272 *
3273 * Drop the rwlock_count so that the thread priority
3274 * is returned back to where it is supposed to be.
3275 */
3276 clear_thread_rwlock_boost();
3277 return 1;
3278 }
3279
3280 lck_mtx_lock_spin_always(&c_seg->c_lock);
3281 }
3282 }
3283 C_SEG_WAKEUP_DONE(c_seg);
3284
3285 /*
3286 * Drop the rwlock_count so that the thread priority
3287 * is returned back to where it is supposed to be.
3288 */
3289 clear_thread_rwlock_boost();
3290
3291 return 0;
3292 }
3293
3294
3295 static void
3296 c_segment_sv_hash_drop_ref(int hash_indx)
3297 {
3298 struct c_sv_hash_entry o_sv_he, n_sv_he;
3299
3300 while (1) {
3301 o_sv_he.he_record = c_segment_sv_hash_table[hash_indx].he_record;
3302
3303 n_sv_he.he_ref = o_sv_he.he_ref - 1;
3304 n_sv_he.he_data = o_sv_he.he_data;
3305
3306 if (OSCompareAndSwap64((UInt64)o_sv_he.he_record, (UInt64)n_sv_he.he_record, (UInt64 *) &c_segment_sv_hash_table[hash_indx].he_record) == TRUE) {
3307 if (n_sv_he.he_ref == 0) {
3308 OSAddAtomic(-1, &c_segment_svp_in_hash);
3309 }
3310 break;
3311 }
3312 }
3313 }
3314
3315
3316 static int
3317 c_segment_sv_hash_insert(uint32_t data)
3318 {
3319 int hash_sindx;
3320 int misses;
3321 struct c_sv_hash_entry o_sv_he, n_sv_he;
3322 boolean_t got_ref = FALSE;
3323
3324 if (data == 0) {
3325 OSAddAtomic(1, &c_segment_svp_zero_compressions);
3326 } else {
3327 OSAddAtomic(1, &c_segment_svp_nonzero_compressions);
3328 }
3329
3330 hash_sindx = data & C_SV_HASH_MASK;
3331
3332 for (misses = 0; misses < C_SV_HASH_MAX_MISS; misses++) {
3333 o_sv_he.he_record = c_segment_sv_hash_table[hash_sindx].he_record;
3334
3335 while (o_sv_he.he_data == data || o_sv_he.he_ref == 0) {
3336 n_sv_he.he_ref = o_sv_he.he_ref + 1;
3337 n_sv_he.he_data = data;
3338
3339 if (OSCompareAndSwap64((UInt64)o_sv_he.he_record, (UInt64)n_sv_he.he_record, (UInt64 *) &c_segment_sv_hash_table[hash_sindx].he_record) == TRUE) {
3340 if (n_sv_he.he_ref == 1) {
3341 OSAddAtomic(1, &c_segment_svp_in_hash);
3342 }
3343 got_ref = TRUE;
3344 break;
3345 }
3346 o_sv_he.he_record = c_segment_sv_hash_table[hash_sindx].he_record;
3347 }
3348 if (got_ref == TRUE) {
3349 break;
3350 }
3351 hash_sindx++;
3352
3353 if (hash_sindx == C_SV_HASH_SIZE) {
3354 hash_sindx = 0;
3355 }
3356 }
3357 if (got_ref == FALSE) {
3358 return -1;
3359 }
3360
3361 return hash_sindx;
3362 }
3363
3364
3365 #if RECORD_THE_COMPRESSED_DATA
3366
3367 static void
3368 c_compressed_record_data(char *src, int c_size)
3369 {
3370 if ((c_compressed_record_cptr + c_size + 4) >= c_compressed_record_ebuf) {
3371 panic("c_compressed_record_cptr >= c_compressed_record_ebuf");
3372 }
3373
3374 *(int *)((void *)c_compressed_record_cptr) = c_size;
3375
3376 c_compressed_record_cptr += 4;
3377
3378 memcpy(c_compressed_record_cptr, src, c_size);
3379 c_compressed_record_cptr += c_size;
3380 }
3381 #endif
3382
3383
3384 static int
3385 c_compress_page(char *src, c_slot_mapping_t slot_ptr, c_segment_t *current_chead, char *scratch_buf)
3386 {
3387 int c_size;
3388 int c_rounded_size = 0;
3389 int max_csize;
3390 c_slot_t cs;
3391 c_segment_t c_seg;
3392
3393 KERNEL_DEBUG(0xe0400000 | DBG_FUNC_START, *current_chead, 0, 0, 0, 0);
3394 retry:
3395 if ((c_seg = c_seg_allocate(current_chead)) == NULL) {
3396 return 1;
3397 }
3398 /*
3399 * returns with c_seg lock held
3400 * and PAGE_REPLACEMENT_DISALLOWED(TRUE)...
3401 * c_nextslot has been allocated and
3402 * c_store.c_buffer populated
3403 */
3404 assert(c_seg->c_state == C_IS_FILLING);
3405
3406 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_seg->c_nextslot);
3407
3408 cs->c_packed_ptr = C_SLOT_PACK_PTR(slot_ptr);
3409 assert(slot_ptr == (c_slot_mapping_t)C_SLOT_UNPACK_PTR(cs));
3410
3411 cs->c_offset = c_seg->c_nextoffset;
3412
3413 max_csize = C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES((int32_t)cs->c_offset);
3414
3415 if (max_csize > PAGE_SIZE) {
3416 max_csize = PAGE_SIZE;
3417 }
3418
3419 #if CHECKSUM_THE_DATA
3420 cs->c_hash_data = vmc_hash(src, PAGE_SIZE);
3421 #endif
3422 boolean_t incomp_copy = FALSE;
3423 int max_csize_adj = (max_csize - 4);
3424
3425 if (vm_compressor_algorithm() != VM_COMPRESSOR_DEFAULT_CODEC) {
3426 #if defined(__arm__) || defined(__arm64__)
3427 uint16_t ccodec = CINVALID;
3428
3429 if (max_csize >= C_SEG_OFFSET_ALIGNMENT_BOUNDARY) {
3430 c_size = metacompressor((const uint8_t *) src,
3431 (uint8_t *) &c_seg->c_store.c_buffer[cs->c_offset],
3432 max_csize_adj, &ccodec,
3433 scratch_buf, &incomp_copy);
3434 #if C_SEG_OFFSET_ALIGNMENT_BOUNDARY > 4
3435 if (c_size > max_csize_adj) {
3436 c_size = -1;
3437 }
3438 #endif
3439 } else {
3440 c_size = -1;
3441 }
3442 assert(ccodec == CCWK || ccodec == CCLZ4);
3443 cs->c_codec = ccodec;
3444 #endif
3445 } else {
3446 #if defined(__arm__) || defined(__arm64__)
3447 cs->c_codec = CCWK;
3448 #endif
3449 #if defined(__arm64__)
3450 __unreachable_ok_push
3451 if (PAGE_SIZE == 4096) {
3452 c_size = WKdm_compress_4k((WK_word *)(uintptr_t)src, (WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
3453 (WK_word *)(uintptr_t)scratch_buf, max_csize_adj);
3454 } else {
3455 c_size = WKdm_compress_16k((WK_word *)(uintptr_t)src, (WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
3456 (WK_word *)(uintptr_t)scratch_buf, max_csize_adj);
3457 }
3458 __unreachable_ok_pop
3459 #else
3460 c_size = WKdm_compress_new((const WK_word *)(uintptr_t)src, (WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
3461 (WK_word *)(uintptr_t)scratch_buf, max_csize_adj);
3462 #endif
3463 }
3464 assertf(((c_size <= max_csize_adj) && (c_size >= -1)),
3465 "c_size invalid (%d, %d), cur compressions: %d", c_size, max_csize_adj, c_segment_pages_compressed);
3466
3467 if (c_size == -1) {
3468 if (max_csize < PAGE_SIZE) {
3469 c_current_seg_filled(c_seg, current_chead);
3470 assert(*current_chead == NULL);
3471
3472 lck_mtx_unlock_always(&c_seg->c_lock);
3473 /* TODO: it may be worth requiring codecs to distinguish
3474 * between incompressible inputs and failures due to
3475 * budget exhaustion.
3476 */
3477 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3478 goto retry;
3479 }
3480 c_size = PAGE_SIZE;
3481
3482 if (incomp_copy == FALSE) {
3483 memcpy(&c_seg->c_store.c_buffer[cs->c_offset], src, c_size);
3484 }
3485
3486 OSAddAtomic(1, &c_segment_noncompressible_pages);
3487 } else if (c_size == 0) {
3488 int hash_index;
3489
3490 /*
3491 * special case - this is a page completely full of a single 32 bit value
3492 */
3493 hash_index = c_segment_sv_hash_insert(*(uint32_t *)(uintptr_t)src);
3494
3495 if (hash_index != -1) {
3496 slot_ptr->s_cindx = hash_index;
3497 slot_ptr->s_cseg = C_SV_CSEG_ID;
3498
3499 OSAddAtomic(1, &c_segment_svp_hash_succeeded);
3500 #if RECORD_THE_COMPRESSED_DATA
3501 c_compressed_record_data(src, 4);
3502 #endif
3503 goto sv_compression;
3504 }
3505 c_size = 4;
3506
3507 memcpy(&c_seg->c_store.c_buffer[cs->c_offset], src, c_size);
3508
3509 OSAddAtomic(1, &c_segment_svp_hash_failed);
3510 }
3511
3512 #if RECORD_THE_COMPRESSED_DATA
3513 c_compressed_record_data((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size);
3514 #endif
3515 #if CHECKSUM_THE_COMPRESSED_DATA
3516 cs->c_hash_compressed_data = vmc_hash((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size);
3517 #endif
3518 #if POPCOUNT_THE_COMPRESSED_DATA
3519 cs->c_pop_cdata = vmc_pop((uintptr_t) &c_seg->c_store.c_buffer[cs->c_offset], c_size);
3520 #endif
3521 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
3522
3523 PACK_C_SIZE(cs, c_size);
3524 c_seg->c_bytes_used += c_rounded_size;
3525 c_seg->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
3526 c_seg->c_slots_used++;
3527
3528 slot_ptr->s_cindx = c_seg->c_nextslot++;
3529 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
3530 slot_ptr->s_cseg = c_seg->c_mysegno + 1;
3531
3532 sv_compression:
3533 if (c_seg->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg->c_nextslot >= C_SLOT_MAX_INDEX) {
3534 c_current_seg_filled(c_seg, current_chead);
3535 assert(*current_chead == NULL);
3536 }
3537 lck_mtx_unlock_always(&c_seg->c_lock);
3538
3539 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3540
3541 #if RECORD_THE_COMPRESSED_DATA
3542 if ((c_compressed_record_cptr - c_compressed_record_sbuf) >= C_SEG_ALLOCSIZE) {
3543 c_compressed_record_write(c_compressed_record_sbuf, (int)(c_compressed_record_cptr - c_compressed_record_sbuf));
3544 c_compressed_record_cptr = c_compressed_record_sbuf;
3545 }
3546 #endif
3547 if (c_size) {
3548 OSAddAtomic64(c_size, &c_segment_compressed_bytes);
3549 OSAddAtomic64(c_rounded_size, &compressor_bytes_used);
3550 }
3551 OSAddAtomic64(PAGE_SIZE, &c_segment_input_bytes);
3552
3553 OSAddAtomic(1, &c_segment_pages_compressed);
3554 OSAddAtomic(1, &sample_period_compression_count);
3555
3556 KERNEL_DEBUG(0xe0400000 | DBG_FUNC_END, *current_chead, c_size, c_segment_input_bytes, c_segment_compressed_bytes, 0);
3557
3558 return 0;
3559 }
3560
3561 static inline void
3562 sv_decompress(int32_t *ddst, int32_t pattern)
3563 {
3564 // assert(__builtin_constant_p(PAGE_SIZE) != 0);
3565 #if defined(__x86_64__)
3566 memset_word(ddst, pattern, PAGE_SIZE / sizeof(int32_t));
3567 #elif defined(__arm64__)
3568 assert((PAGE_SIZE % 128) == 0);
3569 if (pattern == 0) {
3570 fill32_dczva((addr64_t)ddst, PAGE_SIZE);
3571 } else {
3572 fill32_nt((addr64_t)ddst, PAGE_SIZE, pattern);
3573 }
3574 #else
3575 size_t i;
3576
3577 /* Unroll the pattern fill loop 4x to encourage the
3578 * compiler to emit NEON stores, cf.
3579 * <rdar://problem/25839866> Loop autovectorization
3580 * anomalies.
3581 */
3582 /* * We use separate loops for each PAGE_SIZE
3583 * to allow the autovectorizer to engage, as PAGE_SIZE
3584 * may not be a constant.
3585 */
3586
3587 __unreachable_ok_push
3588 if (PAGE_SIZE == 4096) {
3589 for (i = 0; i < (4096U / sizeof(int32_t)); i += 4) {
3590 *ddst++ = pattern;
3591 *ddst++ = pattern;
3592 *ddst++ = pattern;
3593 *ddst++ = pattern;
3594 }
3595 } else {
3596 assert(PAGE_SIZE == 16384);
3597 for (i = 0; i < (int)(16384U / sizeof(int32_t)); i += 4) {
3598 *ddst++ = pattern;
3599 *ddst++ = pattern;
3600 *ddst++ = pattern;
3601 *ddst++ = pattern;
3602 }
3603 }
3604 __unreachable_ok_pop
3605 #endif
3606 }
3607
3608 static int
3609 c_decompress_page(char *dst, volatile c_slot_mapping_t slot_ptr, int flags, int *zeroslot)
3610 {
3611 c_slot_t cs;
3612 c_segment_t c_seg;
3613 uint32_t c_segno;
3614 int c_indx;
3615 int c_rounded_size;
3616 uint32_t c_size;
3617 int retval = 0;
3618 boolean_t need_unlock = TRUE;
3619 boolean_t consider_defragmenting = FALSE;
3620 boolean_t kdp_mode = FALSE;
3621
3622 if (__improbable(flags & C_KDP)) {
3623 if (not_in_kdp) {
3624 panic("C_KDP passed to decompress page from outside of debugger context");
3625 }
3626
3627 assert((flags & C_KEEP) == C_KEEP);
3628 assert((flags & C_DONT_BLOCK) == C_DONT_BLOCK);
3629
3630 if ((flags & (C_DONT_BLOCK | C_KEEP)) != (C_DONT_BLOCK | C_KEEP)) {
3631 return -2;
3632 }
3633
3634 kdp_mode = TRUE;
3635 *zeroslot = 0;
3636 }
3637
3638 ReTry:
3639 if (__probable(!kdp_mode)) {
3640 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3641 } else {
3642 if (kdp_lck_rw_lock_is_acquired_exclusive(&c_master_lock)) {
3643 return -2;
3644 }
3645 }
3646
3647 #if HIBERNATION
3648 /*
3649 * if hibernation is enabled, it indicates (via a call
3650 * to 'vm_decompressor_lock' that no further
3651 * decompressions are allowed once it reaches
3652 * the point of flushing all of the currently dirty
3653 * anonymous memory through the compressor and out
3654 * to disk... in this state we allow freeing of compressed
3655 * pages and must honor the C_DONT_BLOCK case
3656 */
3657 if (__improbable(dst && decompressions_blocked == TRUE)) {
3658 if (flags & C_DONT_BLOCK) {
3659 if (__probable(!kdp_mode)) {
3660 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3661 }
3662
3663 *zeroslot = 0;
3664 return -2;
3665 }
3666 /*
3667 * it's safe to atomically assert and block behind the
3668 * lock held in shared mode because "decompressions_blocked" is
3669 * only set and cleared and the thread_wakeup done when the lock
3670 * is held exclusively
3671 */
3672 assert_wait((event_t)&decompressions_blocked, THREAD_UNINT);
3673
3674 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3675
3676 thread_block(THREAD_CONTINUE_NULL);
3677
3678 goto ReTry;
3679 }
3680 #endif
3681 /* s_cseg is actually "segno+1" */
3682 c_segno = slot_ptr->s_cseg - 1;
3683
3684 if (__improbable(c_segno >= c_segments_available)) {
3685 panic("c_decompress_page: c_segno %d >= c_segments_available %d, slot_ptr(%p), slot_data(%x)",
3686 c_segno, c_segments_available, slot_ptr, *(int *)((void *)slot_ptr));
3687 }
3688
3689 if (__improbable(c_segments[c_segno].c_segno < c_segments_available)) {
3690 panic("c_decompress_page: c_segno %d is free, slot_ptr(%p), slot_data(%x)",
3691 c_segno, slot_ptr, *(int *)((void *)slot_ptr));
3692 }
3693
3694 c_seg = c_segments[c_segno].c_seg;
3695
3696 if (__probable(!kdp_mode)) {
3697 lck_mtx_lock_spin_always(&c_seg->c_lock);
3698 } else {
3699 if (kdp_lck_mtx_lock_spin_is_acquired(&c_seg->c_lock)) {
3700 return -2;
3701 }
3702 }
3703
3704 assert(c_seg->c_state != C_IS_EMPTY && c_seg->c_state != C_IS_FREE);
3705
3706 if (dst == NULL && c_seg->c_busy_swapping) {
3707 assert(c_seg->c_busy);
3708
3709 goto bypass_busy_check;
3710 }
3711 if (flags & C_DONT_BLOCK) {
3712 if (c_seg->c_busy || (C_SEG_IS_ONDISK(c_seg) && dst)) {
3713 *zeroslot = 0;
3714
3715 retval = -2;
3716 goto done;
3717 }
3718 }
3719 if (c_seg->c_busy) {
3720 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3721
3722 c_seg_wait_on_busy(c_seg);
3723
3724 goto ReTry;
3725 }
3726 bypass_busy_check:
3727
3728 c_indx = slot_ptr->s_cindx;
3729
3730 if (__improbable(c_indx >= c_seg->c_nextslot)) {
3731 panic("c_decompress_page: c_indx %d >= c_nextslot %d, c_seg(%p), slot_ptr(%p), slot_data(%x)",
3732 c_indx, c_seg->c_nextslot, c_seg, slot_ptr, *(int *)((void *)slot_ptr));
3733 }
3734
3735 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
3736
3737 c_size = UNPACK_C_SIZE(cs);
3738
3739 if (__improbable(c_size == 0)) {
3740 panic("c_decompress_page: c_size == 0, c_seg(%p), slot_ptr(%p), slot_data(%x)",
3741 c_seg, slot_ptr, *(int *)((void *)slot_ptr));
3742 }
3743
3744 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
3745
3746 if (dst) {
3747 uint32_t age_of_cseg;
3748 clock_sec_t cur_ts_sec;
3749 clock_nsec_t cur_ts_nsec;
3750
3751 if (C_SEG_IS_ONDISK(c_seg)) {
3752 assert(kdp_mode == FALSE);
3753 retval = c_seg_swapin(c_seg, FALSE, TRUE);
3754 assert(retval == 0);
3755
3756 retval = 1;
3757 }
3758 if (c_seg->c_state == C_ON_BAD_Q) {
3759 assert(c_seg->c_store.c_buffer == NULL);
3760 *zeroslot = 0;
3761
3762 retval = -1;
3763 goto done;
3764 }
3765
3766 #if POPCOUNT_THE_COMPRESSED_DATA
3767 unsigned csvpop;
3768 uintptr_t csvaddr = (uintptr_t) &c_seg->c_store.c_buffer[cs->c_offset];
3769 if (cs->c_pop_cdata != (csvpop = vmc_pop(csvaddr, c_size))) {
3770 panic("Compressed data popcount doesn't match original, bit distance: %d %p (phys: %p) %p %p 0x%x 0x%x 0x%x 0x%x", (csvpop - cs->c_pop_cdata), (void *)csvaddr, (void *) kvtophys(csvaddr), c_seg, cs, cs->c_offset, c_size, csvpop, cs->c_pop_cdata);
3771 }
3772 #endif
3773
3774 #if CHECKSUM_THE_COMPRESSED_DATA
3775 unsigned csvhash;
3776 if (cs->c_hash_compressed_data != (csvhash = vmc_hash((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size))) {
3777 panic("Compressed data doesn't match original %p %p %u %u %u", c_seg, cs, c_size, cs->c_hash_compressed_data, csvhash);
3778 }
3779 #endif
3780 if (c_rounded_size == PAGE_SIZE) {
3781 /*
3782 * page wasn't compressible... just copy it out
3783 */
3784 memcpy(dst, &c_seg->c_store.c_buffer[cs->c_offset], PAGE_SIZE);
3785 } else if (c_size == 4) {
3786 int32_t data;
3787 int32_t *dptr;
3788
3789 /*
3790 * page was populated with a single value
3791 * that didn't fit into our fast hash
3792 * so we packed it in as a single non-compressed value
3793 * that we need to populate the page with
3794 */
3795 dptr = (int32_t *)(uintptr_t)dst;
3796 data = *(int32_t *)(&c_seg->c_store.c_buffer[cs->c_offset]);
3797 sv_decompress(dptr, data);
3798 } else {
3799 uint32_t my_cpu_no;
3800 char *scratch_buf;
3801
3802 if (__probable(!kdp_mode)) {
3803 /*
3804 * we're behind the c_seg lock held in spin mode
3805 * which means pre-emption is disabled... therefore
3806 * the following sequence is atomic and safe
3807 */
3808 my_cpu_no = cpu_number();
3809
3810 assert(my_cpu_no < compressor_cpus);
3811
3812 scratch_buf = &compressor_scratch_bufs[my_cpu_no * vm_compressor_get_decode_scratch_size()];
3813 } else {
3814 scratch_buf = kdp_compressor_scratch_buf;
3815 }
3816
3817 if (vm_compressor_algorithm() != VM_COMPRESSOR_DEFAULT_CODEC) {
3818 #if defined(__arm__) || defined(__arm64__)
3819 uint16_t c_codec = cs->c_codec;
3820 metadecompressor((const uint8_t *) &c_seg->c_store.c_buffer[cs->c_offset],
3821 (uint8_t *)dst, c_size, c_codec, (void *)scratch_buf);
3822 #endif
3823 } else {
3824 #if defined(__arm64__)
3825 __unreachable_ok_push
3826 if (PAGE_SIZE == 4096) {
3827 WKdm_decompress_4k((WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
3828 (WK_word *)(uintptr_t)dst, (WK_word *)(uintptr_t)scratch_buf, c_size);
3829 } else {
3830 WKdm_decompress_16k((WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
3831 (WK_word *)(uintptr_t)dst, (WK_word *)(uintptr_t)scratch_buf, c_size);
3832 }
3833 __unreachable_ok_pop
3834 #else
3835 WKdm_decompress_new((WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
3836 (WK_word *)(uintptr_t)dst, (WK_word *)(uintptr_t)scratch_buf, c_size);
3837 #endif
3838 }
3839 }
3840
3841 #if CHECKSUM_THE_DATA
3842 if (cs->c_hash_data != vmc_hash(dst, PAGE_SIZE)) {
3843 #if defined(__arm__) || defined(__arm64__)
3844 int32_t *dinput = &c_seg->c_store.c_buffer[cs->c_offset];
3845 panic("decompressed data doesn't match original cs: %p, hash: 0x%x, offset: %d, c_size: %d, c_rounded_size: %d, codec: %d, header: 0x%x 0x%x 0x%x", cs, cs->c_hash_data, cs->c_offset, c_size, c_rounded_size, cs->c_codec, *dinput, *(dinput + 1), *(dinput + 2));
3846 #else
3847 panic("decompressed data doesn't match original cs: %p, hash: %d, offset: 0x%x, c_size: %d", cs, cs->c_hash_data, cs->c_offset, c_size);
3848 #endif
3849 }
3850 #endif
3851 if (c_seg->c_swappedin_ts == 0 && !kdp_mode) {
3852 clock_get_system_nanotime(&cur_ts_sec, &cur_ts_nsec);
3853
3854 age_of_cseg = (uint32_t)cur_ts_sec - c_seg->c_creation_ts;
3855 if (age_of_cseg < DECOMPRESSION_SAMPLE_MAX_AGE) {
3856 OSAddAtomic(1, &age_of_decompressions_during_sample_period[age_of_cseg]);
3857 } else {
3858 OSAddAtomic(1, &overage_decompressions_during_sample_period);
3859 }
3860
3861 OSAddAtomic(1, &sample_period_decompression_count);
3862 }
3863 }
3864 if (flags & C_KEEP) {
3865 *zeroslot = 0;
3866 goto done;
3867 }
3868 assert(kdp_mode == FALSE);
3869
3870 c_seg->c_bytes_unused += c_rounded_size;
3871 c_seg->c_bytes_used -= c_rounded_size;
3872
3873 assert(c_seg->c_slots_used);
3874 c_seg->c_slots_used--;
3875
3876 PACK_C_SIZE(cs, 0);
3877
3878 if (c_indx < c_seg->c_firstemptyslot) {
3879 c_seg->c_firstemptyslot = c_indx;
3880 }
3881
3882 OSAddAtomic(-1, &c_segment_pages_compressed);
3883
3884 if (c_seg->c_state != C_ON_BAD_Q && !(C_SEG_IS_ONDISK(c_seg))) {
3885 /*
3886 * C_SEG_IS_ONDISK == TRUE can occur when we're doing a
3887 * free of a compressed page (i.e. dst == NULL)
3888 */
3889 OSAddAtomic64(-c_rounded_size, &compressor_bytes_used);
3890 }
3891 if (c_seg->c_busy_swapping) {
3892 /*
3893 * bypass case for c_busy_swapping...
3894 * let the swapin/swapout paths deal with putting
3895 * the c_seg on the minor compaction queue if needed
3896 */
3897 assert(c_seg->c_busy);
3898 goto done;
3899 }
3900 assert(!c_seg->c_busy);
3901
3902 if (c_seg->c_state != C_IS_FILLING) {
3903 if (c_seg->c_bytes_used == 0) {
3904 if (!(C_SEG_IS_ONDISK(c_seg))) {
3905 int pages_populated;
3906
3907 pages_populated = (round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE;
3908 c_seg->c_populated_offset = C_SEG_BYTES_TO_OFFSET(0);
3909
3910 if (pages_populated) {
3911 assert(c_seg->c_state != C_ON_BAD_Q);
3912 assert(c_seg->c_store.c_buffer != NULL);
3913
3914 C_SEG_BUSY(c_seg);
3915 lck_mtx_unlock_always(&c_seg->c_lock);
3916
3917 kernel_memory_depopulate(compressor_map, (vm_offset_t) c_seg->c_store.c_buffer, pages_populated * PAGE_SIZE, KMA_COMPRESSOR);
3918
3919 lck_mtx_lock_spin_always(&c_seg->c_lock);
3920 C_SEG_WAKEUP_DONE(c_seg);
3921 }
3922 if (!c_seg->c_on_minorcompact_q && c_seg->c_state != C_ON_SWAPOUT_Q && c_seg->c_state != C_ON_SWAPIO_Q) {
3923 c_seg_need_delayed_compaction(c_seg, FALSE);
3924 }
3925 } else {
3926 if (c_seg->c_state != C_ON_SWAPPEDOUTSPARSE_Q) {
3927 c_seg_move_to_sparse_list(c_seg);
3928 consider_defragmenting = TRUE;
3929 }
3930 }
3931 } else if (c_seg->c_on_minorcompact_q) {
3932 assert(c_seg->c_state != C_ON_BAD_Q);
3933 assert(!C_SEG_IS_ON_DISK_OR_SOQ(c_seg));
3934
3935 if (C_SEG_SHOULD_MINORCOMPACT_NOW(c_seg)) {
3936 c_seg_try_minor_compaction_and_unlock(c_seg);
3937 need_unlock = FALSE;
3938 }
3939 } else if (!(C_SEG_IS_ONDISK(c_seg))) {
3940 if (c_seg->c_state != C_ON_BAD_Q && c_seg->c_state != C_ON_SWAPOUT_Q && c_seg->c_state != C_ON_SWAPIO_Q &&
3941 C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE) {
3942 c_seg_need_delayed_compaction(c_seg, FALSE);
3943 }
3944 } else if (c_seg->c_state != C_ON_SWAPPEDOUTSPARSE_Q && C_SEG_ONDISK_IS_SPARSE(c_seg)) {
3945 c_seg_move_to_sparse_list(c_seg);
3946 consider_defragmenting = TRUE;
3947 }
3948 }
3949 done:
3950 if (__improbable(kdp_mode)) {
3951 return retval;
3952 }
3953
3954 if (need_unlock == TRUE) {
3955 lck_mtx_unlock_always(&c_seg->c_lock);
3956 }
3957
3958 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3959
3960 if (consider_defragmenting == TRUE) {
3961 vm_swap_consider_defragmenting(VM_SWAP_FLAGS_NONE);
3962 }
3963
3964 #if CONFIG_EMBEDDED
3965 if ((c_minor_count && COMPRESSOR_NEEDS_TO_MINOR_COMPACT()) || vm_compressor_needs_to_major_compact()) {
3966 vm_wake_compactor_swapper();
3967 }
3968 #endif
3969
3970 return retval;
3971 }
3972
3973
3974 int
3975 vm_compressor_get(ppnum_t pn, int *slot, int flags)
3976 {
3977 c_slot_mapping_t slot_ptr;
3978 char *dst;
3979 int zeroslot = 1;
3980 int retval;
3981
3982 dst = pmap_map_compressor_page(pn);
3983 slot_ptr = (c_slot_mapping_t)slot;
3984
3985 assert(dst != NULL);
3986
3987 if (slot_ptr->s_cseg == C_SV_CSEG_ID) {
3988 int32_t data;
3989 int32_t *dptr;
3990
3991 /*
3992 * page was populated with a single value
3993 * that found a home in our hash table
3994 * grab that value from the hash and populate the page
3995 * that we need to populate the page with
3996 */
3997 dptr = (int32_t *)(uintptr_t)dst;
3998 data = c_segment_sv_hash_table[slot_ptr->s_cindx].he_data;
3999 sv_decompress(dptr, data);
4000 if (!(flags & C_KEEP)) {
4001 c_segment_sv_hash_drop_ref(slot_ptr->s_cindx);
4002
4003 OSAddAtomic(-1, &c_segment_pages_compressed);
4004 *slot = 0;
4005 }
4006 if (data) {
4007 OSAddAtomic(1, &c_segment_svp_nonzero_decompressions);
4008 } else {
4009 OSAddAtomic(1, &c_segment_svp_zero_decompressions);
4010 }
4011
4012 pmap_unmap_compressor_page(pn, dst);
4013 return 0;
4014 }
4015
4016 retval = c_decompress_page(dst, slot_ptr, flags, &zeroslot);
4017
4018 /*
4019 * zeroslot will be set to 0 by c_decompress_page if (flags & C_KEEP)
4020 * or (flags & C_DONT_BLOCK) and we found 'c_busy' or 'C_SEG_IS_ONDISK' to be TRUE
4021 */
4022 if (zeroslot) {
4023 *slot = 0;
4024 }
4025
4026 pmap_unmap_compressor_page(pn, dst);
4027
4028 /*
4029 * returns 0 if we successfully decompressed a page from a segment already in memory
4030 * returns 1 if we had to first swap in the segment, before successfully decompressing the page
4031 * returns -1 if we encountered an error swapping in the segment - decompression failed
4032 * returns -2 if (flags & C_DONT_BLOCK) and we found 'c_busy' or 'C_SEG_IS_ONDISK' to be true
4033 */
4034 return retval;
4035 }
4036
4037
4038 int
4039 vm_compressor_free(int *slot, int flags)
4040 {
4041 c_slot_mapping_t slot_ptr;
4042 int zeroslot = 1;
4043 int retval;
4044
4045 assert(flags == 0 || flags == C_DONT_BLOCK);
4046
4047 slot_ptr = (c_slot_mapping_t)slot;
4048
4049 if (slot_ptr->s_cseg == C_SV_CSEG_ID) {
4050 c_segment_sv_hash_drop_ref(slot_ptr->s_cindx);
4051 OSAddAtomic(-1, &c_segment_pages_compressed);
4052
4053 *slot = 0;
4054 return 0;
4055 }
4056 retval = c_decompress_page(NULL, slot_ptr, flags, &zeroslot);
4057 /*
4058 * returns 0 if we successfully freed the specified compressed page
4059 * returns -2 if (flags & C_DONT_BLOCK) and we found 'c_busy' set
4060 */
4061
4062 if (retval == 0) {
4063 *slot = 0;
4064 } else {
4065 assert(retval == -2);
4066 }
4067
4068 return retval;
4069 }
4070
4071
4072 int
4073 vm_compressor_put(ppnum_t pn, int *slot, void **current_chead, char *scratch_buf)
4074 {
4075 char *src;
4076 int retval;
4077
4078 src = pmap_map_compressor_page(pn);
4079 assert(src != NULL);
4080
4081 retval = c_compress_page(src, (c_slot_mapping_t)slot, (c_segment_t *)current_chead, scratch_buf);
4082 pmap_unmap_compressor_page(pn, src);
4083
4084 return retval;
4085 }
4086
4087 void
4088 vm_compressor_transfer(
4089 int *dst_slot_p,
4090 int *src_slot_p)
4091 {
4092 c_slot_mapping_t dst_slot, src_slot;
4093 c_segment_t c_seg;
4094 int c_indx;
4095 c_slot_t cs;
4096
4097 src_slot = (c_slot_mapping_t) src_slot_p;
4098
4099 if (src_slot->s_cseg == C_SV_CSEG_ID) {
4100 *dst_slot_p = *src_slot_p;
4101 *src_slot_p = 0;
4102 return;
4103 }
4104 dst_slot = (c_slot_mapping_t) dst_slot_p;
4105 Retry:
4106 PAGE_REPLACEMENT_DISALLOWED(TRUE);
4107 /* get segment for src_slot */
4108 c_seg = c_segments[src_slot->s_cseg - 1].c_seg;
4109 /* lock segment */
4110 lck_mtx_lock_spin_always(&c_seg->c_lock);
4111 /* wait if it's busy */
4112 if (c_seg->c_busy && !c_seg->c_busy_swapping) {
4113 PAGE_REPLACEMENT_DISALLOWED(FALSE);
4114 c_seg_wait_on_busy(c_seg);
4115 goto Retry;
4116 }
4117 /* find the c_slot */
4118 c_indx = src_slot->s_cindx;
4119 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
4120 /* point the c_slot back to dst_slot instead of src_slot */
4121 cs->c_packed_ptr = C_SLOT_PACK_PTR(dst_slot);
4122 /* transfer */
4123 *dst_slot_p = *src_slot_p;
4124 *src_slot_p = 0;
4125 lck_mtx_unlock_always(&c_seg->c_lock);
4126 PAGE_REPLACEMENT_DISALLOWED(FALSE);
4127 }
4128
4129 #if CONFIG_FREEZE
4130
4131 int freezer_finished_filling = 0;
4132
4133 void
4134 vm_compressor_finished_filling(
4135 void **current_chead)
4136 {
4137 c_segment_t c_seg;
4138
4139 if ((c_seg = *(c_segment_t *)current_chead) == NULL) {
4140 return;
4141 }
4142
4143 assert(c_seg->c_state == C_IS_FILLING);
4144
4145 lck_mtx_lock_spin_always(&c_seg->c_lock);
4146
4147 c_current_seg_filled(c_seg, (c_segment_t *)current_chead);
4148
4149 lck_mtx_unlock_always(&c_seg->c_lock);
4150
4151 freezer_finished_filling++;
4152 }
4153
4154
4155 /*
4156 * This routine is used to transfer the compressed chunks from
4157 * the c_seg/cindx pointed to by slot_p into a new c_seg headed
4158 * by the current_chead and a new cindx within that c_seg.
4159 *
4160 * Currently, this routine is only used by the "freezer backed by
4161 * compressor with swap" mode to create a series of c_segs that
4162 * only contain compressed data belonging to one task. So, we
4163 * move a task's previously compressed data into a set of new
4164 * c_segs which will also hold the task's yet to be compressed data.
4165 */
4166
4167 kern_return_t
4168 vm_compressor_relocate(
4169 void **current_chead,
4170 int *slot_p)
4171 {
4172 c_slot_mapping_t slot_ptr;
4173 c_slot_mapping_t src_slot;
4174 uint32_t c_rounded_size;
4175 uint32_t c_size;
4176 uint16_t dst_slot;
4177 c_slot_t c_dst;
4178 c_slot_t c_src;
4179 int c_indx;
4180 c_segment_t c_seg_dst = NULL;
4181 c_segment_t c_seg_src = NULL;
4182 kern_return_t kr = KERN_SUCCESS;
4183
4184
4185 src_slot = (c_slot_mapping_t) slot_p;
4186
4187 if (src_slot->s_cseg == C_SV_CSEG_ID) {
4188 /*
4189 * no need to relocate... this is a page full of a single
4190 * value which is hashed to a single entry not contained
4191 * in a c_segment_t
4192 */
4193 return kr;
4194 }
4195
4196 Relookup_dst:
4197 c_seg_dst = c_seg_allocate((c_segment_t *)current_chead);
4198 /*
4199 * returns with c_seg lock held
4200 * and PAGE_REPLACEMENT_DISALLOWED(TRUE)...
4201 * c_nextslot has been allocated and
4202 * c_store.c_buffer populated
4203 */
4204 if (c_seg_dst == NULL) {
4205 /*
4206 * Out of compression segments?
4207 */
4208 kr = KERN_RESOURCE_SHORTAGE;
4209 goto out;
4210 }
4211
4212 assert(c_seg_dst->c_busy == 0);
4213
4214 C_SEG_BUSY(c_seg_dst);
4215
4216 dst_slot = c_seg_dst->c_nextslot;
4217
4218 lck_mtx_unlock_always(&c_seg_dst->c_lock);
4219
4220 Relookup_src:
4221 c_seg_src = c_segments[src_slot->s_cseg - 1].c_seg;
4222
4223 assert(c_seg_dst != c_seg_src);
4224
4225 lck_mtx_lock_spin_always(&c_seg_src->c_lock);
4226
4227 if (C_SEG_IS_ONDISK(c_seg_src)) {
4228 /*
4229 * A "thaw" can mark a process as eligible for
4230 * another freeze cycle without bringing any of
4231 * its swapped out c_segs back from disk (because
4232 * that is done on-demand).
4233 *
4234 * If the src c_seg we find for our pre-compressed
4235 * data is already on-disk, then we are dealing
4236 * with an app's data that is already packed and
4237 * swapped out. Don't do anything.
4238 */
4239
4240 PAGE_REPLACEMENT_DISALLOWED(FALSE);
4241
4242 lck_mtx_unlock_always(&c_seg_src->c_lock);
4243
4244 c_seg_src = NULL;
4245
4246 goto out;
4247 }
4248
4249 if (c_seg_src->c_busy) {
4250 PAGE_REPLACEMENT_DISALLOWED(FALSE);
4251 c_seg_wait_on_busy(c_seg_src);
4252
4253 c_seg_src = NULL;
4254
4255 PAGE_REPLACEMENT_DISALLOWED(TRUE);
4256
4257 goto Relookup_src;
4258 }
4259
4260 C_SEG_BUSY(c_seg_src);
4261
4262 lck_mtx_unlock_always(&c_seg_src->c_lock);
4263
4264 PAGE_REPLACEMENT_DISALLOWED(FALSE);
4265
4266 /* find the c_slot */
4267 c_indx = src_slot->s_cindx;
4268
4269 c_src = C_SEG_SLOT_FROM_INDEX(c_seg_src, c_indx);
4270
4271 c_size = UNPACK_C_SIZE(c_src);
4272
4273 assert(c_size);
4274
4275 if (c_size > (uint32_t)(C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES((int32_t)c_seg_dst->c_nextoffset))) {
4276 /*
4277 * This segment is full. We need a new one.
4278 */
4279
4280 PAGE_REPLACEMENT_DISALLOWED(TRUE);
4281
4282 lck_mtx_lock_spin_always(&c_seg_src->c_lock);
4283 C_SEG_WAKEUP_DONE(c_seg_src);
4284 lck_mtx_unlock_always(&c_seg_src->c_lock);
4285
4286 c_seg_src = NULL;
4287
4288 lck_mtx_lock_spin_always(&c_seg_dst->c_lock);
4289
4290 assert(c_seg_dst->c_busy);
4291 assert(c_seg_dst->c_state == C_IS_FILLING);
4292 assert(!c_seg_dst->c_on_minorcompact_q);
4293
4294 c_current_seg_filled(c_seg_dst, (c_segment_t *)current_chead);
4295 assert(*current_chead == NULL);
4296
4297 C_SEG_WAKEUP_DONE(c_seg_dst);
4298
4299 lck_mtx_unlock_always(&c_seg_dst->c_lock);
4300
4301 c_seg_dst = NULL;
4302
4303 PAGE_REPLACEMENT_DISALLOWED(FALSE);
4304
4305 goto Relookup_dst;
4306 }
4307
4308 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, c_seg_dst->c_nextslot);
4309
4310 memcpy(&c_seg_dst->c_store.c_buffer[c_seg_dst->c_nextoffset], &c_seg_src->c_store.c_buffer[c_src->c_offset], c_size);
4311 //is platform alignment actually necessary since wkdm aligns its output?
4312 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
4313
4314 cslot_copy(c_dst, c_src);
4315 c_dst->c_offset = c_seg_dst->c_nextoffset;
4316
4317 if (c_seg_dst->c_firstemptyslot == c_seg_dst->c_nextslot) {
4318 c_seg_dst->c_firstemptyslot++;
4319 }
4320
4321 c_seg_dst->c_slots_used++;
4322 c_seg_dst->c_nextslot++;
4323 c_seg_dst->c_bytes_used += c_rounded_size;
4324 c_seg_dst->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
4325
4326
4327 PACK_C_SIZE(c_src, 0);
4328
4329 c_seg_src->c_bytes_used -= c_rounded_size;
4330 c_seg_src->c_bytes_unused += c_rounded_size;
4331
4332 assert(c_seg_src->c_slots_used);
4333 c_seg_src->c_slots_used--;
4334
4335 if (c_indx < c_seg_src->c_firstemptyslot) {
4336 c_seg_src->c_firstemptyslot = c_indx;
4337 }
4338
4339 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, dst_slot);
4340
4341 PAGE_REPLACEMENT_ALLOWED(TRUE);
4342 slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
4343 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
4344 slot_ptr->s_cseg = c_seg_dst->c_mysegno + 1;
4345 slot_ptr->s_cindx = dst_slot;
4346
4347 PAGE_REPLACEMENT_ALLOWED(FALSE);
4348
4349 out:
4350 if (c_seg_src) {
4351 lck_mtx_lock_spin_always(&c_seg_src->c_lock);
4352
4353 C_SEG_WAKEUP_DONE(c_seg_src);
4354
4355 if (c_seg_src->c_bytes_used == 0 && c_seg_src->c_state != C_IS_FILLING) {
4356 if (!c_seg_src->c_on_minorcompact_q) {
4357 c_seg_need_delayed_compaction(c_seg_src, FALSE);
4358 }
4359 }
4360
4361 lck_mtx_unlock_always(&c_seg_src->c_lock);
4362 }
4363
4364 if (c_seg_dst) {
4365 PAGE_REPLACEMENT_DISALLOWED(TRUE);
4366
4367 lck_mtx_lock_spin_always(&c_seg_dst->c_lock);
4368
4369 if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX_INDEX) {
4370 /*
4371 * Nearing or exceeded maximum slot and offset capacity.
4372 */
4373 assert(c_seg_dst->c_busy);
4374 assert(c_seg_dst->c_state == C_IS_FILLING);
4375 assert(!c_seg_dst->c_on_minorcompact_q);
4376
4377 c_current_seg_filled(c_seg_dst, (c_segment_t *)current_chead);
4378 assert(*current_chead == NULL);
4379 }
4380
4381 C_SEG_WAKEUP_DONE(c_seg_dst);
4382
4383 lck_mtx_unlock_always(&c_seg_dst->c_lock);
4384
4385 c_seg_dst = NULL;
4386
4387 PAGE_REPLACEMENT_DISALLOWED(FALSE);
4388 }
4389
4390 return kr;
4391 }
4392 #endif /* CONFIG_FREEZE */