2 * Copyright (c) 2000-2013 Apple Inc. All rights reserved.
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
29 #include <vm/vm_compressor.h>
31 #if CONFIG_PHANTOM_CACHE
32 #include <vm/vm_phantom_cache.h>
35 #include <vm/vm_map.h>
36 #include <vm/vm_pageout.h>
37 #include <vm/memory_object.h>
38 #include <vm/vm_compressor_algorithms.h>
39 #include <vm/vm_fault.h>
40 #include <vm/vm_protos.h>
41 #include <mach/mach_host.h> /* for host_info() */
42 #include <kern/ledger.h>
43 #include <kern/policy_internal.h>
44 #include <kern/thread_group.h>
45 #include <san/kasan.h>
47 #if defined(__x86_64__)
48 #include <i386/misc_protos.h>
50 #if defined(__arm64__)
51 #include <arm/machine_routines.h>
54 #include <IOKit/IOHibernatePrivate.h>
56 extern boolean_t vm_darkwake_mode
;
58 #if POPCOUNT_THE_COMPRESSED_DATA
59 boolean_t popcount_c_segs
= TRUE
;
61 static inline uint32_t
62 vmc_pop(uintptr_t ins
, int sz
)
66 if (__probable(popcount_c_segs
== FALSE
)) {
72 uint64_t *ins64
= (uint64_t *) ins
;
73 uint64_t *ins642
= (uint64_t *) (ins
+ 8);
74 rv1
= __builtin_popcountll(*ins64
);
75 rv2
= __builtin_popcountll(*ins642
);
82 uint32_t *ins32
= (uint32_t *) ins
;
83 rv
+= __builtin_popcount(*ins32
);
89 char *ins8
= (char *)ins
;
90 rv
+= __builtin_popcount(*ins8
);
98 #if VALIDATE_C_SEGMENTS
99 boolean_t validate_c_segs
= TRUE
;
102 * vm_compressor_mode has a heirarchy of control to set its value.
103 * boot-args are checked first, then device-tree, and finally
104 * the default value that is defined below. See vm_fault_init() for
105 * the boot-arg & device-tree code.
111 int vm_compressor_mode
= VM_PAGER_FREEZER_DEFAULT
;
113 void *freezer_chead
; /* The chead used to track c_segs allocated for the exclusive use of holding just one task's compressed memory.*/
114 char *freezer_compressor_scratch_buf
= NULL
;
116 extern int c_freezer_swapout_page_count
; /* This count keeps track of the # of compressed pages holding just one task's compressed memory on the swapout queue. This count is used during each freeze i.e. on a per-task basis.*/
118 #else /* CONFIG_FREEZE */
119 int vm_compressor_mode
= VM_PAGER_NOT_CONFIGURED
;
120 #endif /* CONFIG_FREEZE */
124 #else /* CONFIG_EMBEDDED */
125 int vm_compressor_mode
= VM_PAGER_COMPRESSOR_WITH_SWAP
;
128 #endif /* CONFIG_EMBEDDED */
130 int vm_compressor_is_active
= 0;
131 int vm_compression_limit
= 0;
132 int vm_compressor_available
= 0;
134 extern void vm_pageout_io_throttle(void);
136 #if CHECKSUM_THE_DATA || CHECKSUM_THE_SWAP || CHECKSUM_THE_COMPRESSED_DATA
137 extern unsigned int hash_string(char *cp
, int len
);
138 static unsigned int vmc_hash(char *, int);
139 boolean_t checksum_c_segs
= TRUE
;
142 vmc_hash(char *cp
, int len
)
144 if (__probable(checksum_c_segs
== FALSE
)) {
147 return hash_string(cp
, len
);
151 #define UNPACK_C_SIZE(cs) ((cs->c_size == (PAGE_SIZE-1)) ? PAGE_SIZE : cs->c_size)
152 #define PACK_C_SIZE(cs, size) (cs->c_size = ((size == PAGE_SIZE) ? PAGE_SIZE - 1 : size))
155 struct c_sv_hash_entry
{
158 uint32_t c_sv_he_ref
;
159 uint32_t c_sv_he_data
;
161 uint64_t c_sv_he_record
;
165 #define he_ref c_sv_he_un.c_sv_he.c_sv_he_ref
166 #define he_data c_sv_he_un.c_sv_he.c_sv_he_data
167 #define he_record c_sv_he_un.c_sv_he_record
169 #define C_SV_HASH_MAX_MISS 32
170 #define C_SV_HASH_SIZE ((1 << 10))
171 #define C_SV_HASH_MASK ((1 << 10) - 1)
172 #define C_SV_CSEG_ID ((1 << 22) - 1)
182 #define C_SLOT_PACK_PTR(ptr) (((uintptr_t)ptr - (uintptr_t) KERNEL_PMAP_HEAP_RANGE_START) >> 2)
183 #define C_SLOT_UNPACK_PTR(cslot) ((uintptr_t)(cslot->c_packed_ptr << 2) + (uintptr_t) KERNEL_PMAP_HEAP_RANGE_START)
186 uint32_t c_segment_count
= 0;
187 uint32_t c_segment_count_max
= 0;
189 uint64_t c_generation_id
= 0;
190 uint64_t c_generation_id_flush_barrier
;
193 #define HIBERNATE_FLUSHING_SECS_TO_COMPLETE 120
195 boolean_t hibernate_no_swapspace
= FALSE
;
196 clock_sec_t hibernate_flushing_deadline
= 0;
199 #if RECORD_THE_COMPRESSED_DATA
200 char *c_compressed_record_sbuf
;
201 char *c_compressed_record_ebuf
;
202 char *c_compressed_record_cptr
;
206 queue_head_t c_age_list_head
;
207 queue_head_t c_swappedin_list_head
;
208 queue_head_t c_swapout_list_head
;
209 queue_head_t c_swapio_list_head
;
210 queue_head_t c_swappedout_list_head
;
211 queue_head_t c_swappedout_sparse_list_head
;
212 queue_head_t c_major_list_head
;
213 queue_head_t c_filling_list_head
;
214 queue_head_t c_bad_list_head
;
216 uint32_t c_age_count
= 0;
217 uint32_t c_swappedin_count
= 0;
218 uint32_t c_swapout_count
= 0;
219 uint32_t c_swapio_count
= 0;
220 uint32_t c_swappedout_count
= 0;
221 uint32_t c_swappedout_sparse_count
= 0;
222 uint32_t c_major_count
= 0;
223 uint32_t c_filling_count
= 0;
224 uint32_t c_empty_count
= 0;
225 uint32_t c_bad_count
= 0;
228 queue_head_t c_minor_list_head
;
229 uint32_t c_minor_count
= 0;
231 int c_overage_swapped_count
= 0;
232 int c_overage_swapped_limit
= 0;
234 int c_seg_fixed_array_len
;
235 union c_segu
*c_segments
;
236 vm_offset_t c_buffers
;
237 vm_size_t c_buffers_size
;
238 caddr_t c_segments_next_page
;
239 boolean_t c_segments_busy
;
240 uint32_t c_segments_available
;
241 uint32_t c_segments_limit
;
242 uint32_t c_segments_nearing_limit
;
244 uint32_t c_segment_svp_in_hash
;
245 uint32_t c_segment_svp_hash_succeeded
;
246 uint32_t c_segment_svp_hash_failed
;
247 uint32_t c_segment_svp_zero_compressions
;
248 uint32_t c_segment_svp_nonzero_compressions
;
249 uint32_t c_segment_svp_zero_decompressions
;
250 uint32_t c_segment_svp_nonzero_decompressions
;
252 uint32_t c_segment_noncompressible_pages
;
254 uint32_t c_segment_pages_compressed
;
255 uint32_t c_segment_pages_compressed_limit
;
256 uint32_t c_segment_pages_compressed_nearing_limit
;
257 uint32_t c_free_segno_head
= (uint32_t)-1;
259 uint32_t vm_compressor_minorcompact_threshold_divisor
= 10;
260 uint32_t vm_compressor_majorcompact_threshold_divisor
= 10;
261 uint32_t vm_compressor_unthrottle_threshold_divisor
= 10;
262 uint32_t vm_compressor_catchup_threshold_divisor
= 10;
264 uint32_t vm_compressor_minorcompact_threshold_divisor_overridden
= 0;
265 uint32_t vm_compressor_majorcompact_threshold_divisor_overridden
= 0;
266 uint32_t vm_compressor_unthrottle_threshold_divisor_overridden
= 0;
267 uint32_t vm_compressor_catchup_threshold_divisor_overridden
= 0;
269 #define C_SEGMENTS_PER_PAGE (PAGE_SIZE / sizeof(union c_segu))
272 lck_grp_attr_t vm_compressor_lck_grp_attr
;
273 lck_attr_t vm_compressor_lck_attr
;
274 lck_grp_t vm_compressor_lck_grp
;
275 lck_mtx_t
*c_list_lock
;
276 lck_rw_t c_master_lock
;
277 boolean_t decompressions_blocked
= FALSE
;
279 zone_t compressor_segment_zone
;
280 int c_compressor_swap_trigger
= 0;
282 uint32_t compressor_cpus
;
283 char *compressor_scratch_bufs
;
284 char *kdp_compressor_scratch_buf
;
285 char *kdp_compressor_decompressed_page
;
286 addr64_t kdp_compressor_decompressed_page_paddr
;
287 ppnum_t kdp_compressor_decompressed_page_ppnum
;
289 clock_sec_t start_of_sample_period_sec
= 0;
290 clock_nsec_t start_of_sample_period_nsec
= 0;
291 clock_sec_t start_of_eval_period_sec
= 0;
292 clock_nsec_t start_of_eval_period_nsec
= 0;
293 uint32_t sample_period_decompression_count
= 0;
294 uint32_t sample_period_compression_count
= 0;
295 uint32_t last_eval_decompression_count
= 0;
296 uint32_t last_eval_compression_count
= 0;
298 #define DECOMPRESSION_SAMPLE_MAX_AGE (60 * 30)
300 boolean_t vm_swapout_ripe_segments
= FALSE
;
301 uint32_t vm_ripe_target_age
= (60 * 60 * 48);
303 uint32_t swapout_target_age
= 0;
304 uint32_t age_of_decompressions_during_sample_period
[DECOMPRESSION_SAMPLE_MAX_AGE
];
305 uint32_t overage_decompressions_during_sample_period
= 0;
308 void do_fastwake_warmup(queue_head_t
*, boolean_t
);
309 boolean_t fastwake_warmup
= FALSE
;
310 boolean_t fastwake_recording_in_progress
= FALSE
;
311 clock_sec_t dont_trim_until_ts
= 0;
313 uint64_t c_segment_warmup_count
;
314 uint64_t first_c_segment_to_warm_generation_id
= 0;
315 uint64_t last_c_segment_to_warm_generation_id
= 0;
316 boolean_t hibernate_flushing
= FALSE
;
318 int64_t c_segment_input_bytes
__attribute__((aligned(8))) = 0;
319 int64_t c_segment_compressed_bytes
__attribute__((aligned(8))) = 0;
320 int64_t compressor_bytes_used
__attribute__((aligned(8))) = 0;
323 struct c_sv_hash_entry c_segment_sv_hash_table
[C_SV_HASH_SIZE
] __attribute__ ((aligned(8)));
325 static boolean_t
compressor_needs_to_swap(void);
326 static void vm_compressor_swap_trigger_thread(void);
327 static void vm_compressor_do_delayed_compactions(boolean_t
);
328 static void vm_compressor_compact_and_swap(boolean_t
);
329 static void vm_compressor_age_swapped_in_segments(boolean_t
);
332 static void vm_compressor_take_paging_space_action(void);
335 void compute_swapout_target_age(void);
337 boolean_t
c_seg_major_compact(c_segment_t
, c_segment_t
);
338 boolean_t
c_seg_major_compact_ok(c_segment_t
, c_segment_t
);
340 int c_seg_minor_compaction_and_unlock(c_segment_t
, boolean_t
);
341 int c_seg_do_minor_compaction_and_unlock(c_segment_t
, boolean_t
, boolean_t
, boolean_t
);
342 void c_seg_try_minor_compaction_and_unlock(c_segment_t c_seg
);
344 void c_seg_move_to_sparse_list(c_segment_t
);
345 void c_seg_insert_into_q(queue_head_t
*, c_segment_t
);
347 uint64_t vm_available_memory(void);
348 uint64_t vm_compressor_pages_compressed(void);
351 * indicate the need to do a major compaction if
352 * the overall set of in-use compression segments
353 * becomes sparse... on systems that support pressure
354 * driven swapping, this will also cause swapouts to
357 static inline boolean_t
358 vm_compressor_needs_to_major_compact()
360 uint32_t incore_seg_count
;
362 incore_seg_count
= c_segment_count
- c_swappedout_count
- c_swappedout_sparse_count
;
364 if ((c_segment_count
>= (c_segments_nearing_limit
/ 8)) &&
365 ((incore_seg_count
* C_SEG_MAX_PAGES
) - VM_PAGE_COMPRESSOR_COUNT
) >
366 ((incore_seg_count
/ 8) * C_SEG_MAX_PAGES
)) {
374 vm_available_memory(void)
376 return ((uint64_t)AVAILABLE_NON_COMPRESSED_MEMORY
) * PAGE_SIZE_64
;
381 vm_compressor_pages_compressed(void)
383 return c_segment_pages_compressed
* PAGE_SIZE_64
;
388 vm_compressor_low_on_space(void)
390 if ((c_segment_pages_compressed
> c_segment_pages_compressed_nearing_limit
) ||
391 (c_segment_count
> c_segments_nearing_limit
)) {
400 vm_compressor_out_of_space(void)
402 if ((c_segment_pages_compressed
>= c_segment_pages_compressed_limit
) ||
403 (c_segment_count
>= c_segments_limit
)) {
412 vm_wants_task_throttled(task_t task
)
414 if (task
== kernel_task
) {
418 if (VM_CONFIG_SWAP_IS_ACTIVE
) {
419 if ((vm_compressor_low_on_space() || HARD_THROTTLE_LIMIT_REACHED()) &&
420 (unsigned int)pmap_compressed(task
->map
->pmap
) > (c_segment_pages_compressed
/ 4)) {
428 #if DEVELOPMENT || DEBUG
429 boolean_t kill_on_no_paging_space
= FALSE
; /* On compressor/swap exhaustion, kill the largest process regardless of
430 * its chosen process policy. Controlled by a boot-arg of the same name. */
431 #endif /* DEVELOPMENT || DEBUG */
435 static uint32_t no_paging_space_action_in_progress
= 0;
436 extern void memorystatus_send_low_swap_note(void);
439 vm_compressor_take_paging_space_action(void)
441 if (no_paging_space_action_in_progress
== 0) {
442 if (OSCompareAndSwap(0, 1, (UInt32
*)&no_paging_space_action_in_progress
)) {
443 if (no_paging_space_action()) {
444 #if DEVELOPMENT || DEBUG
445 if (kill_on_no_paging_space
== TRUE
) {
447 * Since we are choosing to always kill a process, we don't need the
448 * "out of application memory" dialog box in this mode. And, hence we won't
451 no_paging_space_action_in_progress
= 0;
454 #endif /* DEVELOPMENT || DEBUG */
455 memorystatus_send_low_swap_note();
458 no_paging_space_action_in_progress
= 0;
462 #endif /* !CONFIG_EMBEDDED */
466 vm_compressor_init_locks(void)
468 lck_grp_attr_setdefault(&vm_compressor_lck_grp_attr
);
469 lck_grp_init(&vm_compressor_lck_grp
, "vm_compressor", &vm_compressor_lck_grp_attr
);
470 lck_attr_setdefault(&vm_compressor_lck_attr
);
472 lck_rw_init(&c_master_lock
, &vm_compressor_lck_grp
, &vm_compressor_lck_attr
);
477 vm_decompressor_lock(void)
479 PAGE_REPLACEMENT_ALLOWED(TRUE
);
481 decompressions_blocked
= TRUE
;
483 PAGE_REPLACEMENT_ALLOWED(FALSE
);
487 vm_decompressor_unlock(void)
489 PAGE_REPLACEMENT_ALLOWED(TRUE
);
491 decompressions_blocked
= FALSE
;
493 PAGE_REPLACEMENT_ALLOWED(FALSE
);
495 thread_wakeup((event_t
)&decompressions_blocked
);
499 cslot_copy(c_slot_t cdst
, c_slot_t csrc
)
501 #if CHECKSUM_THE_DATA
502 cdst
->c_hash_data
= csrc
->c_hash_data
;
504 #if CHECKSUM_THE_COMPRESSED_DATA
505 cdst
->c_hash_compressed_data
= csrc
->c_hash_compressed_data
;
507 #if POPCOUNT_THE_COMPRESSED_DATA
508 cdst
->c_pop_cdata
= csrc
->c_pop_cdata
;
510 cdst
->c_size
= csrc
->c_size
;
511 cdst
->c_packed_ptr
= csrc
->c_packed_ptr
;
512 #if defined(__arm__) || defined(__arm64__)
513 cdst
->c_codec
= csrc
->c_codec
;
517 vm_map_t compressor_map
;
518 uint64_t compressor_pool_max_size
;
519 uint64_t compressor_pool_size
;
520 uint32_t compressor_pool_multiplier
;
522 #if DEVELOPMENT || DEBUG
524 * Compressor segments are write-protected in development/debug
525 * kernels to help debug memory corruption.
526 * In cases where performance is a concern, this can be disabled
527 * via the boot-arg "-disable_cseg_write_protection".
529 boolean_t write_protect_c_segs
= TRUE
;
530 int vm_compressor_test_seg_wp
;
531 uint32_t vm_ktrace_enabled
;
532 #endif /* DEVELOPMENT || DEBUG */
535 vm_compressor_init(void)
538 struct c_slot cs_dummy
;
539 c_slot_t cs
= &cs_dummy
;
540 int c_segment_min_size
;
541 int c_segment_padded_size
;
543 kern_return_t retval
= KERN_SUCCESS
;
544 vm_offset_t start_addr
= 0;
545 vm_size_t c_segments_arr_size
= 0, compressor_submap_size
= 0;
546 vm_map_kernel_flags_t vmk_flags
;
547 #if RECORD_THE_COMPRESSED_DATA
548 vm_size_t c_compressed_record_sbuf_size
= 0;
549 #endif /* RECORD_THE_COMPRESSED_DATA */
551 #if DEVELOPMENT || DEBUG
552 char bootarg_name
[32];
553 if (PE_parse_boot_argn("-kill_on_no_paging_space", bootarg_name
, sizeof(bootarg_name
))) {
554 kill_on_no_paging_space
= TRUE
;
556 if (PE_parse_boot_argn("-disable_cseg_write_protection", bootarg_name
, sizeof(bootarg_name
))) {
557 write_protect_c_segs
= FALSE
;
560 PE_parse_boot_argn("vm_compressor_validation", &vmcval
, sizeof(vmcval
));
562 if (kern_feature_override(KF_COMPRSV_OVRD
)) {
566 #if POPCOUNT_THE_COMPRESSED_DATA
567 popcount_c_segs
= FALSE
;
569 #if CHECKSUM_THE_DATA || CHECKSUM_THE_COMPRESSED_DATA
570 checksum_c_segs
= FALSE
;
572 #if VALIDATE_C_SEGMENTS
573 validate_c_segs
= FALSE
;
575 write_protect_c_segs
= FALSE
;
577 #endif /* DEVELOPMENT || DEBUG */
580 * ensure that any pointer that gets created from
581 * the vm_page zone can be packed properly
583 cs
->c_packed_ptr
= C_SLOT_PACK_PTR(zone_map_min_address
);
585 if (C_SLOT_UNPACK_PTR(cs
) != (uintptr_t)zone_map_min_address
) {
586 panic("C_SLOT_UNPACK_PTR failed on zone_map_min_address - %p", (void *)zone_map_min_address
);
589 cs
->c_packed_ptr
= C_SLOT_PACK_PTR(zone_map_max_address
);
591 if (C_SLOT_UNPACK_PTR(cs
) != (uintptr_t)zone_map_max_address
) {
592 panic("C_SLOT_UNPACK_PTR failed on zone_map_max_address - %p", (void *)zone_map_max_address
);
596 assert((C_SEGMENTS_PER_PAGE
* sizeof(union c_segu
)) == PAGE_SIZE
);
598 PE_parse_boot_argn("vm_compression_limit", &vm_compression_limit
, sizeof(vm_compression_limit
));
600 #ifdef CONFIG_EMBEDDED
601 #if XNU_TARGET_OS_WATCH
602 // rdar://problem/51012698
603 vm_compressor_minorcompact_threshold_divisor
= 40;
605 vm_compressor_minorcompact_threshold_divisor
= 20;
607 vm_compressor_majorcompact_threshold_divisor
= 30;
608 vm_compressor_unthrottle_threshold_divisor
= 40;
609 vm_compressor_catchup_threshold_divisor
= 60;
611 if (max_mem
<= (3ULL * 1024ULL * 1024ULL * 1024ULL)) {
612 vm_compressor_minorcompact_threshold_divisor
= 11;
613 vm_compressor_majorcompact_threshold_divisor
= 13;
614 vm_compressor_unthrottle_threshold_divisor
= 20;
615 vm_compressor_catchup_threshold_divisor
= 35;
617 vm_compressor_minorcompact_threshold_divisor
= 20;
618 vm_compressor_majorcompact_threshold_divisor
= 25;
619 vm_compressor_unthrottle_threshold_divisor
= 35;
620 vm_compressor_catchup_threshold_divisor
= 50;
624 * vm_page_init_lck_grp is now responsible for calling vm_compressor_init_locks
625 * c_master_lock needs to be available early so that "vm_page_find_contiguous" can
626 * use PAGE_REPLACEMENT_ALLOWED to coordinate with the compressor.
629 c_list_lock
= lck_mtx_alloc_init(&vm_compressor_lck_grp
, &vm_compressor_lck_attr
);
631 queue_init(&c_bad_list_head
);
632 queue_init(&c_age_list_head
);
633 queue_init(&c_minor_list_head
);
634 queue_init(&c_major_list_head
);
635 queue_init(&c_filling_list_head
);
636 queue_init(&c_swapout_list_head
);
637 queue_init(&c_swapio_list_head
);
638 queue_init(&c_swappedin_list_head
);
639 queue_init(&c_swappedout_list_head
);
640 queue_init(&c_swappedout_sparse_list_head
);
642 c_free_segno_head
= -1;
643 c_segments_available
= 0;
645 if (vm_compression_limit
) {
646 compressor_pool_size
= (uint64_t)vm_compression_limit
* PAGE_SIZE_64
;
649 compressor_pool_max_size
= C_SEG_MAX_LIMIT
;
650 compressor_pool_max_size
*= C_SEG_BUFSIZE
;
654 if (vm_compression_limit
== 0) {
655 if (max_mem
<= (4ULL * 1024ULL * 1024ULL * 1024ULL)) {
656 compressor_pool_size
= 16ULL * max_mem
;
657 } else if (max_mem
<= (8ULL * 1024ULL * 1024ULL * 1024ULL)) {
658 compressor_pool_size
= 8ULL * max_mem
;
659 } else if (max_mem
<= (32ULL * 1024ULL * 1024ULL * 1024ULL)) {
660 compressor_pool_size
= 4ULL * max_mem
;
662 compressor_pool_size
= 2ULL * max_mem
;
665 if (max_mem
<= (8ULL * 1024ULL * 1024ULL * 1024ULL)) {
666 compressor_pool_multiplier
= 1;
667 } else if (max_mem
<= (32ULL * 1024ULL * 1024ULL * 1024ULL)) {
668 compressor_pool_multiplier
= 2;
670 compressor_pool_multiplier
= 4;
673 #elif defined(__arm__)
675 #define VM_RESERVE_SIZE (1024 * 1024 * 256)
676 #define MAX_COMPRESSOR_POOL_SIZE (1024 * 1024 * 450)
678 if (compressor_pool_max_size
> MAX_COMPRESSOR_POOL_SIZE
) {
679 compressor_pool_max_size
= MAX_COMPRESSOR_POOL_SIZE
;
682 if (vm_compression_limit
== 0) {
683 compressor_pool_size
= ((kernel_map
->max_offset
- kernel_map
->min_offset
) - kernel_map
->size
) - VM_RESERVE_SIZE
;
685 compressor_pool_multiplier
= 1;
687 if (compressor_pool_max_size
> max_mem
) {
688 compressor_pool_max_size
= max_mem
;
691 if (vm_compression_limit
== 0) {
692 compressor_pool_size
= max_mem
;
694 compressor_pool_multiplier
= 1;
696 if (compressor_pool_size
> compressor_pool_max_size
) {
697 compressor_pool_size
= compressor_pool_max_size
;
701 c_segments_limit
= (uint32_t)(compressor_pool_size
/ (vm_size_t
)(C_SEG_ALLOCSIZE
));
702 c_segments_nearing_limit
= (uint32_t)(((uint64_t)c_segments_limit
* 98ULL) / 100ULL);
704 c_segment_pages_compressed_limit
= (c_segments_limit
* (C_SEG_BUFSIZE
/ PAGE_SIZE
) * compressor_pool_multiplier
);
706 if (c_segment_pages_compressed_limit
< (uint32_t)(max_mem
/ PAGE_SIZE
)) {
707 c_segment_pages_compressed_limit
= (uint32_t)(max_mem
/ PAGE_SIZE
);
710 c_segment_pages_compressed_nearing_limit
= (uint32_t)(((uint64_t)c_segment_pages_compressed_limit
* 98ULL) / 100ULL);
713 * Submap needs space for:
716 * - swap reclaimations -- C_SEG_BUFSIZE
718 c_segments_arr_size
= vm_map_round_page((sizeof(union c_segu
) * c_segments_limit
), VM_MAP_PAGE_MASK(kernel_map
));
719 c_buffers_size
= vm_map_round_page(((vm_size_t
)C_SEG_ALLOCSIZE
* (vm_size_t
)c_segments_limit
), VM_MAP_PAGE_MASK(kernel_map
));
721 compressor_submap_size
= c_segments_arr_size
+ c_buffers_size
+ C_SEG_BUFSIZE
;
723 #if RECORD_THE_COMPRESSED_DATA
724 c_compressed_record_sbuf_size
= (vm_size_t
)C_SEG_ALLOCSIZE
+ (PAGE_SIZE
* 2);
725 compressor_submap_size
+= c_compressed_record_sbuf_size
;
726 #endif /* RECORD_THE_COMPRESSED_DATA */
728 vmk_flags
= VM_MAP_KERNEL_FLAGS_NONE
;
729 vmk_flags
.vmkf_permanent
= TRUE
;
730 retval
= kmem_suballoc(kernel_map
, &start_addr
, compressor_submap_size
,
731 FALSE
, VM_FLAGS_ANYWHERE
, vmk_flags
, VM_KERN_MEMORY_COMPRESSOR
,
734 if (retval
!= KERN_SUCCESS
) {
735 if (++attempts
> 3) {
736 panic("vm_compressor_init: kmem_suballoc failed - 0x%llx", (uint64_t)compressor_submap_size
);
739 compressor_pool_size
= compressor_pool_size
/ 2;
741 kprintf("retrying creation of the compressor submap at 0x%llx bytes\n", compressor_pool_size
);
744 if (kernel_memory_allocate(compressor_map
, (vm_offset_t
*)(&c_segments
), (sizeof(union c_segu
) * c_segments_limit
), 0, KMA_KOBJECT
| KMA_VAONLY
| KMA_PERMANENT
, VM_KERN_MEMORY_COMPRESSOR
) != KERN_SUCCESS
) {
745 panic("vm_compressor_init: kernel_memory_allocate failed - c_segments\n");
747 if (kernel_memory_allocate(compressor_map
, &c_buffers
, c_buffers_size
, 0, KMA_COMPRESSOR
| KMA_VAONLY
| KMA_PERMANENT
, VM_KERN_MEMORY_COMPRESSOR
) != KERN_SUCCESS
) {
748 panic("vm_compressor_init: kernel_memory_allocate failed - c_buffers\n");
752 c_segment_min_size
= sizeof(struct c_segment
) + (C_SEG_SLOT_VAR_ARRAY_MIN_LEN
* sizeof(struct c_slot
));
754 for (c_segment_padded_size
= 128; c_segment_padded_size
< c_segment_min_size
; c_segment_padded_size
= c_segment_padded_size
<< 1) {
758 compressor_segment_zone
= zinit(c_segment_padded_size
, c_segments_limit
* c_segment_padded_size
, PAGE_SIZE
, "compressor_segment");
759 zone_change(compressor_segment_zone
, Z_CALLERACCT
, FALSE
);
760 zone_change(compressor_segment_zone
, Z_NOENCRYPT
, TRUE
);
762 c_seg_fixed_array_len
= (c_segment_padded_size
- sizeof(struct c_segment
)) / sizeof(struct c_slot
);
764 c_segments_busy
= FALSE
;
766 c_segments_next_page
= (caddr_t
)c_segments
;
767 vm_compressor_algorithm_init();
770 host_basic_info_data_t hinfo
;
771 mach_msg_type_number_t count
= HOST_BASIC_INFO_COUNT
;
774 host_info((host_t
)BSD_HOST
, HOST_BASIC_INFO
, (host_info_t
)&hinfo
, &count
);
776 compressor_cpus
= hinfo
.max_cpus
;
777 compressor_scratch_bufs
= kalloc_tag(compressor_cpus
* vm_compressor_get_decode_scratch_size(), VM_KERN_MEMORY_COMPRESSOR
);
779 kdp_compressor_scratch_buf
= kalloc_tag(vm_compressor_get_decode_scratch_size(), VM_KERN_MEMORY_COMPRESSOR
);
782 * kdp_compressor_decompressed_page must be page aligned because we access
783 * it through the physical apperture by page number. kalloc() does not
784 * guarantee alignment.
787 if (kernel_memory_allocate(kernel_map
, &addr
, PAGE_SIZE
, 0, KMA_KOBJECT
, VM_KERN_MEMORY_COMPRESSOR
) != KERN_SUCCESS
) {
788 panic("vm_compressor_init: kernel_memory_allocate failed - kdp_compressor_decompressed_page\n");
790 assert((addr
& PAGE_MASK
) == 0);
791 kdp_compressor_decompressed_page
= (void *)addr
;
792 kdp_compressor_decompressed_page_paddr
= kvtophys((vm_offset_t
)kdp_compressor_decompressed_page
);
793 kdp_compressor_decompressed_page_ppnum
= (ppnum_t
) atop(kdp_compressor_decompressed_page_paddr
);
796 freezer_compressor_scratch_buf
= kalloc_tag(vm_compressor_get_encode_scratch_size(), VM_KERN_MEMORY_COMPRESSOR
);
799 #if RECORD_THE_COMPRESSED_DATA
800 if (kernel_memory_allocate(compressor_map
, (vm_offset_t
*)&c_compressed_record_sbuf
, c_compressed_record_sbuf_size
, 0, KMA_KOBJECT
, VM_KERN_MEMORY_COMPRESSOR
) != KERN_SUCCESS
) {
801 panic("vm_compressor_init: kernel_memory_allocate failed - c_compressed_record_sbuf\n");
804 c_compressed_record_cptr
= c_compressed_record_sbuf
;
805 c_compressed_record_ebuf
= c_compressed_record_sbuf
+ c_compressed_record_sbuf_size
;
808 if (kernel_thread_start_priority((thread_continue_t
)vm_compressor_swap_trigger_thread
, NULL
,
809 BASEPRI_VM
, &thread
) != KERN_SUCCESS
) {
810 panic("vm_compressor_swap_trigger_thread: create failed");
812 thread_deallocate(thread
);
814 if (vm_pageout_internal_start() != KERN_SUCCESS
) {
815 panic("vm_compressor_init: Failed to start the internal pageout thread.\n");
817 if (VM_CONFIG_SWAP_IS_PRESENT
) {
818 vm_compressor_swap_init();
821 if (VM_CONFIG_COMPRESSOR_IS_ACTIVE
) {
822 vm_compressor_is_active
= 1;
826 memorystatus_freeze_enabled
= TRUE
;
827 #endif /* CONFIG_FREEZE */
829 vm_compressor_available
= 1;
831 vm_page_reactivate_all_throttled();
835 #if VALIDATE_C_SEGMENTS
838 c_seg_validate(c_segment_t c_seg
, boolean_t must_be_compact
)
842 uint32_t c_rounded_size
;
846 if (__probable(validate_c_segs
== FALSE
)) {
849 if (c_seg
->c_firstemptyslot
< c_seg
->c_nextslot
) {
850 c_indx
= c_seg
->c_firstemptyslot
;
851 cs
= C_SEG_SLOT_FROM_INDEX(c_seg
, c_indx
);
854 panic("c_seg_validate: no slot backing c_firstemptyslot");
858 panic("c_seg_validate: c_firstemptyslot has non-zero size (%d)\n", cs
->c_size
);
863 for (c_indx
= 0; c_indx
< c_seg
->c_nextslot
; c_indx
++) {
864 cs
= C_SEG_SLOT_FROM_INDEX(c_seg
, c_indx
);
866 c_size
= UNPACK_C_SIZE(cs
);
868 c_rounded_size
= (c_size
+ C_SEG_OFFSET_ALIGNMENT_MASK
) & ~C_SEG_OFFSET_ALIGNMENT_MASK
;
870 bytes_used
+= c_rounded_size
;
872 #if CHECKSUM_THE_COMPRESSED_DATA
874 if (c_size
&& cs
->c_hash_compressed_data
!= (csvhash
= vmc_hash((char *)&c_seg
->c_store
.c_buffer
[cs
->c_offset
], c_size
))) {
875 addr64_t csvphys
= kvtophys((vm_offset_t
)&c_seg
->c_store
.c_buffer
[cs
->c_offset
]);
876 panic("Compressed data doesn't match original %p phys: 0x%llx %d %p %d %d 0x%x 0x%x", c_seg
, csvphys
, cs
->c_offset
, cs
, c_indx
, c_size
, cs
->c_hash_compressed_data
, csvhash
);
879 #if POPCOUNT_THE_COMPRESSED_DATA
882 uintptr_t csvaddr
= (uintptr_t) &c_seg
->c_store
.c_buffer
[cs
->c_offset
];
883 if (cs
->c_pop_cdata
!= (csvpop
= vmc_pop(csvaddr
, c_size
))) {
884 panic("Compressed data popcount doesn't match original, bit distance: %d %p (phys: %p) %p %p 0x%llx 0x%x 0x%x 0x%x", (csvpop
- cs
->c_pop_cdata
), (void *)csvaddr
, (void *) kvtophys(csvaddr
), c_seg
, cs
, (uint64_t)cs
->c_offset
, c_size
, csvpop
, cs
->c_pop_cdata
);
890 if (bytes_used
!= c_seg
->c_bytes_used
) {
891 panic("c_seg_validate: bytes_used mismatch - found %d, segment has %d\n", bytes_used
, c_seg
->c_bytes_used
);
894 if (c_seg
->c_bytes_used
> C_SEG_OFFSET_TO_BYTES((int32_t)c_seg
->c_nextoffset
)) {
895 panic("c_seg_validate: c_bytes_used > c_nextoffset - c_nextoffset = %d, c_bytes_used = %d\n",
896 (int32_t)C_SEG_OFFSET_TO_BYTES((int32_t)c_seg
->c_nextoffset
), c_seg
->c_bytes_used
);
899 if (must_be_compact
) {
900 if (c_seg
->c_bytes_used
!= C_SEG_OFFSET_TO_BYTES((int32_t)c_seg
->c_nextoffset
)) {
901 panic("c_seg_validate: c_bytes_used doesn't match c_nextoffset - c_nextoffset = %d, c_bytes_used = %d\n",
902 (int32_t)C_SEG_OFFSET_TO_BYTES((int32_t)c_seg
->c_nextoffset
), c_seg
->c_bytes_used
);
911 c_seg_need_delayed_compaction(c_segment_t c_seg
, boolean_t c_list_lock_held
)
913 boolean_t clear_busy
= FALSE
;
915 if (c_list_lock_held
== FALSE
) {
916 if (!lck_mtx_try_lock_spin_always(c_list_lock
)) {
919 lck_mtx_unlock_always(&c_seg
->c_lock
);
920 lck_mtx_lock_spin_always(c_list_lock
);
921 lck_mtx_lock_spin_always(&c_seg
->c_lock
);
926 assert(c_seg
->c_state
!= C_IS_FILLING
);
928 if (!c_seg
->c_on_minorcompact_q
&& !(C_SEG_IS_ON_DISK_OR_SOQ(c_seg
))) {
929 queue_enter(&c_minor_list_head
, c_seg
, c_segment_t
, c_list
);
930 c_seg
->c_on_minorcompact_q
= 1;
933 if (c_list_lock_held
== FALSE
) {
934 lck_mtx_unlock_always(c_list_lock
);
937 if (clear_busy
== TRUE
) {
938 C_SEG_WAKEUP_DONE(c_seg
);
943 unsigned int c_seg_moved_to_sparse_list
= 0;
946 c_seg_move_to_sparse_list(c_segment_t c_seg
)
948 boolean_t clear_busy
= FALSE
;
950 if (!lck_mtx_try_lock_spin_always(c_list_lock
)) {
953 lck_mtx_unlock_always(&c_seg
->c_lock
);
954 lck_mtx_lock_spin_always(c_list_lock
);
955 lck_mtx_lock_spin_always(&c_seg
->c_lock
);
959 c_seg_switch_state(c_seg
, C_ON_SWAPPEDOUTSPARSE_Q
, FALSE
);
961 c_seg_moved_to_sparse_list
++;
963 lck_mtx_unlock_always(c_list_lock
);
965 if (clear_busy
== TRUE
) {
966 C_SEG_WAKEUP_DONE(c_seg
);
972 c_seg_insert_into_q(queue_head_t
*qhead
, c_segment_t c_seg
)
974 c_segment_t c_seg_next
;
976 if (queue_empty(qhead
)) {
977 queue_enter(qhead
, c_seg
, c_segment_t
, c_age_list
);
979 c_seg_next
= (c_segment_t
)queue_first(qhead
);
982 if (c_seg
->c_generation_id
< c_seg_next
->c_generation_id
) {
983 queue_insert_before(qhead
, c_seg
, c_seg_next
, c_segment_t
, c_age_list
);
986 c_seg_next
= (c_segment_t
) queue_next(&c_seg_next
->c_age_list
);
988 if (queue_end(qhead
, (queue_entry_t
) c_seg_next
)) {
989 queue_enter(qhead
, c_seg
, c_segment_t
, c_age_list
);
997 int try_minor_compaction_failed
= 0;
998 int try_minor_compaction_succeeded
= 0;
1001 c_seg_try_minor_compaction_and_unlock(c_segment_t c_seg
)
1003 assert(c_seg
->c_on_minorcompact_q
);
1005 * c_seg is currently on the delayed minor compaction
1006 * queue and we have c_seg locked... if we can get the
1007 * c_list_lock w/o blocking (if we blocked we could deadlock
1008 * because the lock order is c_list_lock then c_seg's lock)
1009 * we'll pull it from the delayed list and free it directly
1011 if (!lck_mtx_try_lock_spin_always(c_list_lock
)) {
1013 * c_list_lock is held, we need to bail
1015 try_minor_compaction_failed
++;
1017 lck_mtx_unlock_always(&c_seg
->c_lock
);
1019 try_minor_compaction_succeeded
++;
1022 c_seg_do_minor_compaction_and_unlock(c_seg
, TRUE
, FALSE
, FALSE
);
1028 c_seg_do_minor_compaction_and_unlock(c_segment_t c_seg
, boolean_t clear_busy
, boolean_t need_list_lock
, boolean_t disallow_page_replacement
)
1032 assert(c_seg
->c_busy
);
1033 assert(!C_SEG_IS_ON_DISK_OR_SOQ(c_seg
));
1036 * check for the case that can occur when we are not swapping
1037 * and this segment has been major compacted in the past
1038 * and moved to the majorcompact q to remove it from further
1039 * consideration... if the occupancy falls too low we need
1040 * to put it back on the age_q so that it will be considered
1041 * in the next major compaction sweep... if we don't do this
1042 * we will eventually run into the c_segments_limit
1044 if (c_seg
->c_state
== C_ON_MAJORCOMPACT_Q
&& C_SEG_SHOULD_MAJORCOMPACT_NOW(c_seg
)) {
1045 c_seg_switch_state(c_seg
, C_ON_AGE_Q
, FALSE
);
1047 if (!c_seg
->c_on_minorcompact_q
) {
1048 if (clear_busy
== TRUE
) {
1049 C_SEG_WAKEUP_DONE(c_seg
);
1052 lck_mtx_unlock_always(&c_seg
->c_lock
);
1056 queue_remove(&c_minor_list_head
, c_seg
, c_segment_t
, c_list
);
1057 c_seg
->c_on_minorcompact_q
= 0;
1060 lck_mtx_unlock_always(c_list_lock
);
1062 if (disallow_page_replacement
== TRUE
) {
1063 lck_mtx_unlock_always(&c_seg
->c_lock
);
1065 PAGE_REPLACEMENT_DISALLOWED(TRUE
);
1067 lck_mtx_lock_spin_always(&c_seg
->c_lock
);
1069 c_seg_freed
= c_seg_minor_compaction_and_unlock(c_seg
, clear_busy
);
1071 if (disallow_page_replacement
== TRUE
) {
1072 PAGE_REPLACEMENT_DISALLOWED(FALSE
);
1075 if (need_list_lock
== TRUE
) {
1076 lck_mtx_lock_spin_always(c_list_lock
);
1084 c_seg_wait_on_busy(c_segment_t c_seg
)
1086 c_seg
->c_wanted
= 1;
1087 assert_wait((event_t
) (c_seg
), THREAD_UNINT
);
1089 lck_mtx_unlock_always(&c_seg
->c_lock
);
1090 thread_block(THREAD_CONTINUE_NULL
);
1095 c_seg_switch_state(c_segment_t c_seg
, int new_state
, boolean_t insert_head
)
1097 int old_state
= c_seg
->c_state
;
1099 #if !CONFIG_EMBEDDED
1100 #if DEVELOPMENT || DEBUG
1101 if (new_state
!= C_IS_FILLING
) {
1102 LCK_MTX_ASSERT(&c_seg
->c_lock
, LCK_MTX_ASSERT_OWNED
);
1104 LCK_MTX_ASSERT(c_list_lock
, LCK_MTX_ASSERT_OWNED
);
1106 #endif /* !CONFIG_EMBEDDED */
1107 switch (old_state
) {
1109 assert(new_state
== C_IS_FILLING
|| new_state
== C_IS_FREE
);
1115 assert(new_state
== C_ON_AGE_Q
|| new_state
== C_ON_SWAPOUT_Q
);
1117 queue_remove(&c_filling_list_head
, c_seg
, c_segment_t
, c_age_list
);
1122 assert(new_state
== C_ON_SWAPOUT_Q
|| new_state
== C_ON_MAJORCOMPACT_Q
||
1123 new_state
== C_IS_FREE
);
1125 queue_remove(&c_age_list_head
, c_seg
, c_segment_t
, c_age_list
);
1129 case C_ON_SWAPPEDIN_Q
:
1130 assert(new_state
== C_ON_AGE_Q
|| new_state
== C_IS_FREE
);
1132 queue_remove(&c_swappedin_list_head
, c_seg
, c_segment_t
, c_age_list
);
1133 c_swappedin_count
--;
1136 case C_ON_SWAPOUT_Q
:
1137 assert(new_state
== C_ON_AGE_Q
|| new_state
== C_IS_FREE
|| new_state
== C_IS_EMPTY
|| new_state
== C_ON_SWAPIO_Q
);
1139 queue_remove(&c_swapout_list_head
, c_seg
, c_segment_t
, c_age_list
);
1140 thread_wakeup((event_t
)&compaction_swapper_running
);
1145 assert(new_state
== C_ON_SWAPPEDOUT_Q
|| new_state
== C_ON_SWAPPEDOUTSPARSE_Q
|| new_state
== C_ON_AGE_Q
);
1147 queue_remove(&c_swapio_list_head
, c_seg
, c_segment_t
, c_age_list
);
1151 case C_ON_SWAPPEDOUT_Q
:
1152 assert(new_state
== C_ON_SWAPPEDIN_Q
|| new_state
== C_ON_AGE_Q
||
1153 new_state
== C_ON_SWAPPEDOUTSPARSE_Q
||
1154 new_state
== C_ON_BAD_Q
|| new_state
== C_IS_EMPTY
|| new_state
== C_IS_FREE
);
1156 queue_remove(&c_swappedout_list_head
, c_seg
, c_segment_t
, c_age_list
);
1157 c_swappedout_count
--;
1160 case C_ON_SWAPPEDOUTSPARSE_Q
:
1161 assert(new_state
== C_ON_SWAPPEDIN_Q
|| new_state
== C_ON_AGE_Q
||
1162 new_state
== C_ON_BAD_Q
|| new_state
== C_IS_EMPTY
|| new_state
== C_IS_FREE
);
1164 queue_remove(&c_swappedout_sparse_list_head
, c_seg
, c_segment_t
, c_age_list
);
1165 c_swappedout_sparse_count
--;
1168 case C_ON_MAJORCOMPACT_Q
:
1169 assert(new_state
== C_ON_AGE_Q
|| new_state
== C_IS_FREE
);
1171 queue_remove(&c_major_list_head
, c_seg
, c_segment_t
, c_age_list
);
1176 assert(new_state
== C_IS_FREE
);
1178 queue_remove(&c_bad_list_head
, c_seg
, c_segment_t
, c_age_list
);
1183 panic("c_seg %p has bad c_state = %d\n", c_seg
, old_state
);
1186 switch (new_state
) {
1188 assert(old_state
!= C_IS_FILLING
);
1193 assert(old_state
== C_ON_SWAPOUT_Q
|| old_state
== C_ON_SWAPPEDOUT_Q
|| old_state
== C_ON_SWAPPEDOUTSPARSE_Q
);
1199 assert(old_state
== C_IS_EMPTY
);
1201 queue_enter(&c_filling_list_head
, c_seg
, c_segment_t
, c_age_list
);
1206 assert(old_state
== C_IS_FILLING
|| old_state
== C_ON_SWAPPEDIN_Q
||
1207 old_state
== C_ON_SWAPOUT_Q
|| old_state
== C_ON_SWAPIO_Q
||
1208 old_state
== C_ON_MAJORCOMPACT_Q
|| old_state
== C_ON_SWAPPEDOUT_Q
|| old_state
== C_ON_SWAPPEDOUTSPARSE_Q
);
1210 if (old_state
== C_IS_FILLING
) {
1211 queue_enter(&c_age_list_head
, c_seg
, c_segment_t
, c_age_list
);
1213 if (!queue_empty(&c_age_list_head
)) {
1214 c_segment_t c_first
;
1216 c_first
= (c_segment_t
)queue_first(&c_age_list_head
);
1217 c_seg
->c_creation_ts
= c_first
->c_creation_ts
;
1219 queue_enter_first(&c_age_list_head
, c_seg
, c_segment_t
, c_age_list
);
1224 case C_ON_SWAPPEDIN_Q
:
1225 assert(old_state
== C_ON_SWAPPEDOUT_Q
|| old_state
== C_ON_SWAPPEDOUTSPARSE_Q
);
1227 if (insert_head
== TRUE
) {
1228 queue_enter_first(&c_swappedin_list_head
, c_seg
, c_segment_t
, c_age_list
);
1230 queue_enter(&c_swappedin_list_head
, c_seg
, c_segment_t
, c_age_list
);
1232 c_swappedin_count
++;
1235 case C_ON_SWAPOUT_Q
:
1236 assert(old_state
== C_ON_AGE_Q
|| old_state
== C_IS_FILLING
);
1238 if (insert_head
== TRUE
) {
1239 queue_enter_first(&c_swapout_list_head
, c_seg
, c_segment_t
, c_age_list
);
1241 queue_enter(&c_swapout_list_head
, c_seg
, c_segment_t
, c_age_list
);
1247 assert(old_state
== C_ON_SWAPOUT_Q
);
1249 if (insert_head
== TRUE
) {
1250 queue_enter_first(&c_swapio_list_head
, c_seg
, c_segment_t
, c_age_list
);
1252 queue_enter(&c_swapio_list_head
, c_seg
, c_segment_t
, c_age_list
);
1257 case C_ON_SWAPPEDOUT_Q
:
1258 assert(old_state
== C_ON_SWAPIO_Q
);
1260 if (insert_head
== TRUE
) {
1261 queue_enter_first(&c_swappedout_list_head
, c_seg
, c_segment_t
, c_age_list
);
1263 queue_enter(&c_swappedout_list_head
, c_seg
, c_segment_t
, c_age_list
);
1265 c_swappedout_count
++;
1268 case C_ON_SWAPPEDOUTSPARSE_Q
:
1269 assert(old_state
== C_ON_SWAPIO_Q
|| old_state
== C_ON_SWAPPEDOUT_Q
);
1271 if (insert_head
== TRUE
) {
1272 queue_enter_first(&c_swappedout_sparse_list_head
, c_seg
, c_segment_t
, c_age_list
);
1274 queue_enter(&c_swappedout_sparse_list_head
, c_seg
, c_segment_t
, c_age_list
);
1277 c_swappedout_sparse_count
++;
1280 case C_ON_MAJORCOMPACT_Q
:
1281 assert(old_state
== C_ON_AGE_Q
);
1283 if (insert_head
== TRUE
) {
1284 queue_enter_first(&c_major_list_head
, c_seg
, c_segment_t
, c_age_list
);
1286 queue_enter(&c_major_list_head
, c_seg
, c_segment_t
, c_age_list
);
1292 assert(old_state
== C_ON_SWAPPEDOUT_Q
|| old_state
== C_ON_SWAPPEDOUTSPARSE_Q
);
1294 if (insert_head
== TRUE
) {
1295 queue_enter_first(&c_bad_list_head
, c_seg
, c_segment_t
, c_age_list
);
1297 queue_enter(&c_bad_list_head
, c_seg
, c_segment_t
, c_age_list
);
1303 panic("c_seg %p requesting bad c_state = %d\n", c_seg
, new_state
);
1305 c_seg
->c_state
= new_state
;
1311 c_seg_free(c_segment_t c_seg
)
1313 assert(c_seg
->c_busy
);
1315 lck_mtx_unlock_always(&c_seg
->c_lock
);
1316 lck_mtx_lock_spin_always(c_list_lock
);
1317 lck_mtx_lock_spin_always(&c_seg
->c_lock
);
1319 c_seg_free_locked(c_seg
);
1324 c_seg_free_locked(c_segment_t c_seg
)
1327 int pages_populated
= 0;
1328 int32_t *c_buffer
= NULL
;
1329 uint64_t c_swap_handle
= 0;
1331 assert(c_seg
->c_busy
);
1332 assert(c_seg
->c_slots_used
== 0);
1333 assert(!c_seg
->c_on_minorcompact_q
);
1334 assert(!c_seg
->c_busy_swapping
);
1336 if (c_seg
->c_overage_swap
== TRUE
) {
1337 c_overage_swapped_count
--;
1338 c_seg
->c_overage_swap
= FALSE
;
1340 if (!(C_SEG_IS_ONDISK(c_seg
))) {
1341 c_buffer
= c_seg
->c_store
.c_buffer
;
1343 c_swap_handle
= c_seg
->c_store
.c_swap_handle
;
1346 c_seg_switch_state(c_seg
, C_IS_FREE
, FALSE
);
1348 lck_mtx_unlock_always(c_list_lock
);
1351 pages_populated
= (round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg
->c_populated_offset
))) / PAGE_SIZE
;
1352 c_seg
->c_store
.c_buffer
= NULL
;
1354 c_seg
->c_store
.c_swap_handle
= (uint64_t)-1;
1357 lck_mtx_unlock_always(&c_seg
->c_lock
);
1360 if (pages_populated
) {
1361 kernel_memory_depopulate(compressor_map
, (vm_offset_t
) c_buffer
, pages_populated
* PAGE_SIZE
, KMA_COMPRESSOR
);
1363 } else if (c_swap_handle
) {
1365 * Free swap space on disk.
1367 vm_swap_free(c_swap_handle
);
1369 lck_mtx_lock_spin_always(&c_seg
->c_lock
);
1371 * c_seg must remain busy until
1372 * after the call to vm_swap_free
1374 C_SEG_WAKEUP_DONE(c_seg
);
1375 lck_mtx_unlock_always(&c_seg
->c_lock
);
1377 segno
= c_seg
->c_mysegno
;
1379 lck_mtx_lock_spin_always(c_list_lock
);
1381 * because the c_buffer is now associated with the segno,
1382 * we can't put the segno back on the free list until
1383 * after we have depopulated the c_buffer range, or
1384 * we run the risk of depopulating a range that is
1385 * now being used in one of the compressor heads
1387 c_segments
[segno
].c_segno
= c_free_segno_head
;
1388 c_free_segno_head
= segno
;
1391 lck_mtx_unlock_always(c_list_lock
);
1393 lck_mtx_destroy(&c_seg
->c_lock
, &vm_compressor_lck_grp
);
1395 if (c_seg
->c_slot_var_array_len
) {
1396 kfree(c_seg
->c_slot_var_array
, sizeof(struct c_slot
) * c_seg
->c_slot_var_array_len
);
1399 zfree(compressor_segment_zone
, c_seg
);
1402 #if DEVELOPMENT || DEBUG
1403 int c_seg_trim_page_count
= 0;
1407 c_seg_trim_tail(c_segment_t c_seg
)
1412 uint32_t c_rounded_size
;
1413 uint16_t current_nextslot
;
1414 uint32_t current_populated_offset
;
1416 if (c_seg
->c_bytes_used
== 0) {
1419 current_nextslot
= c_seg
->c_nextslot
;
1420 current_populated_offset
= c_seg
->c_populated_offset
;
1422 while (c_seg
->c_nextslot
) {
1423 cs
= C_SEG_SLOT_FROM_INDEX(c_seg
, (c_seg
->c_nextslot
- 1));
1425 c_size
= UNPACK_C_SIZE(cs
);
1428 if (current_nextslot
!= c_seg
->c_nextslot
) {
1429 c_rounded_size
= (c_size
+ C_SEG_OFFSET_ALIGNMENT_MASK
) & ~C_SEG_OFFSET_ALIGNMENT_MASK
;
1430 c_offset
= cs
->c_offset
+ C_SEG_BYTES_TO_OFFSET(c_rounded_size
);
1432 c_seg
->c_nextoffset
= c_offset
;
1433 c_seg
->c_populated_offset
= (c_offset
+ (C_SEG_BYTES_TO_OFFSET(PAGE_SIZE
) - 1)) &
1434 ~(C_SEG_BYTES_TO_OFFSET(PAGE_SIZE
) - 1);
1436 if (c_seg
->c_firstemptyslot
> c_seg
->c_nextslot
) {
1437 c_seg
->c_firstemptyslot
= c_seg
->c_nextslot
;
1439 #if DEVELOPMENT || DEBUG
1440 c_seg_trim_page_count
+= ((round_page_32(C_SEG_OFFSET_TO_BYTES(current_populated_offset
)) -
1441 round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg
->c_populated_offset
))) /
1447 c_seg
->c_nextslot
--;
1449 assert(c_seg
->c_nextslot
);
1454 c_seg_minor_compaction_and_unlock(c_segment_t c_seg
, boolean_t clear_busy
)
1456 c_slot_mapping_t slot_ptr
;
1457 uint32_t c_offset
= 0;
1458 uint32_t old_populated_offset
;
1459 uint32_t c_rounded_size
;
1466 assert(c_seg
->c_busy
);
1468 #if VALIDATE_C_SEGMENTS
1469 c_seg_validate(c_seg
, FALSE
);
1471 if (c_seg
->c_bytes_used
== 0) {
1475 lck_mtx_unlock_always(&c_seg
->c_lock
);
1477 if (c_seg
->c_firstemptyslot
>= c_seg
->c_nextslot
|| C_SEG_UNUSED_BYTES(c_seg
) < PAGE_SIZE
) {
1481 /* TODO: assert first emptyslot's c_size is actually 0 */
1483 #if DEVELOPMENT || DEBUG
1484 C_SEG_MAKE_WRITEABLE(c_seg
);
1487 #if VALIDATE_C_SEGMENTS
1488 c_seg
->c_was_minor_compacted
++;
1490 c_indx
= c_seg
->c_firstemptyslot
;
1491 c_dst
= C_SEG_SLOT_FROM_INDEX(c_seg
, c_indx
);
1493 old_populated_offset
= c_seg
->c_populated_offset
;
1494 c_offset
= c_dst
->c_offset
;
1496 for (i
= c_indx
+ 1; i
< c_seg
->c_nextslot
&& c_offset
< c_seg
->c_nextoffset
; i
++) {
1497 c_src
= C_SEG_SLOT_FROM_INDEX(c_seg
, i
);
1499 c_size
= UNPACK_C_SIZE(c_src
);
1505 c_rounded_size
= (c_size
+ C_SEG_OFFSET_ALIGNMENT_MASK
) & ~C_SEG_OFFSET_ALIGNMENT_MASK
;
1506 /* N.B.: This memcpy may be an overlapping copy */
1507 memcpy(&c_seg
->c_store
.c_buffer
[c_offset
], &c_seg
->c_store
.c_buffer
[c_src
->c_offset
], c_rounded_size
);
1509 cslot_copy(c_dst
, c_src
);
1510 c_dst
->c_offset
= c_offset
;
1512 slot_ptr
= (c_slot_mapping_t
)C_SLOT_UNPACK_PTR(c_dst
);
1513 slot_ptr
->s_cindx
= c_indx
;
1515 c_offset
+= C_SEG_BYTES_TO_OFFSET(c_rounded_size
);
1516 PACK_C_SIZE(c_src
, 0);
1519 c_dst
= C_SEG_SLOT_FROM_INDEX(c_seg
, c_indx
);
1521 c_seg
->c_firstemptyslot
= c_indx
;
1522 c_seg
->c_nextslot
= c_indx
;
1523 c_seg
->c_nextoffset
= c_offset
;
1524 c_seg
->c_populated_offset
= (c_offset
+ (C_SEG_BYTES_TO_OFFSET(PAGE_SIZE
) - 1)) & ~(C_SEG_BYTES_TO_OFFSET(PAGE_SIZE
) - 1);
1525 c_seg
->c_bytes_unused
= 0;
1527 #if VALIDATE_C_SEGMENTS
1528 c_seg_validate(c_seg
, TRUE
);
1530 if (old_populated_offset
> c_seg
->c_populated_offset
) {
1534 gc_size
= C_SEG_OFFSET_TO_BYTES(old_populated_offset
- c_seg
->c_populated_offset
);
1535 gc_ptr
= &c_seg
->c_store
.c_buffer
[c_seg
->c_populated_offset
];
1537 kernel_memory_depopulate(compressor_map
, (vm_offset_t
)gc_ptr
, gc_size
, KMA_COMPRESSOR
);
1540 #if DEVELOPMENT || DEBUG
1541 C_SEG_WRITE_PROTECT(c_seg
);
1545 if (clear_busy
== TRUE
) {
1546 lck_mtx_lock_spin_always(&c_seg
->c_lock
);
1547 C_SEG_WAKEUP_DONE(c_seg
);
1548 lck_mtx_unlock_always(&c_seg
->c_lock
);
1555 c_seg_alloc_nextslot(c_segment_t c_seg
)
1557 struct c_slot
*old_slot_array
= NULL
;
1558 struct c_slot
*new_slot_array
= NULL
;
1562 if (c_seg
->c_nextslot
< c_seg_fixed_array_len
) {
1566 if ((c_seg
->c_nextslot
- c_seg_fixed_array_len
) >= c_seg
->c_slot_var_array_len
) {
1567 oldlen
= c_seg
->c_slot_var_array_len
;
1568 old_slot_array
= c_seg
->c_slot_var_array
;
1571 newlen
= C_SEG_SLOT_VAR_ARRAY_MIN_LEN
;
1573 newlen
= oldlen
* 2;
1576 new_slot_array
= (struct c_slot
*)kalloc(sizeof(struct c_slot
) * newlen
);
1578 lck_mtx_lock_spin_always(&c_seg
->c_lock
);
1580 if (old_slot_array
) {
1581 memcpy((char *)new_slot_array
, (char *)old_slot_array
, sizeof(struct c_slot
) * oldlen
);
1584 c_seg
->c_slot_var_array_len
= newlen
;
1585 c_seg
->c_slot_var_array
= new_slot_array
;
1587 lck_mtx_unlock_always(&c_seg
->c_lock
);
1589 if (old_slot_array
) {
1590 kfree(old_slot_array
, sizeof(struct c_slot
) * oldlen
);
1598 uint64_t asked_permission
;
1599 uint64_t compactions
;
1600 uint64_t moved_slots
;
1601 uint64_t moved_bytes
;
1602 uint64_t wasted_space_in_swapouts
;
1603 uint64_t count_of_swapouts
;
1604 uint64_t count_of_freed_segs
;
1605 } c_seg_major_compact_stats
;
1608 #define C_MAJOR_COMPACTION_SIZE_APPROPRIATE ((C_SEG_BUFSIZE * 90) / 100)
1612 c_seg_major_compact_ok(
1613 c_segment_t c_seg_dst
,
1614 c_segment_t c_seg_src
)
1616 c_seg_major_compact_stats
.asked_permission
++;
1618 if (c_seg_src
->c_bytes_used
>= C_MAJOR_COMPACTION_SIZE_APPROPRIATE
&&
1619 c_seg_dst
->c_bytes_used
>= C_MAJOR_COMPACTION_SIZE_APPROPRIATE
) {
1623 if (c_seg_dst
->c_nextoffset
>= C_SEG_OFF_LIMIT
|| c_seg_dst
->c_nextslot
>= C_SLOT_MAX_INDEX
) {
1625 * destination segment is full... can't compact
1635 c_seg_major_compact(
1636 c_segment_t c_seg_dst
,
1637 c_segment_t c_seg_src
)
1639 c_slot_mapping_t slot_ptr
;
1640 uint32_t c_rounded_size
;
1646 boolean_t keep_compacting
= TRUE
;
1649 * segments are not locked but they are both marked c_busy
1650 * which keeps c_decompress from working on them...
1651 * we can safely allocate new pages, move compressed data
1652 * from c_seg_src to c_seg_dst and update both c_segment's
1653 * state w/o holding the master lock
1655 #if DEVELOPMENT || DEBUG
1656 C_SEG_MAKE_WRITEABLE(c_seg_dst
);
1659 #if VALIDATE_C_SEGMENTS
1660 c_seg_dst
->c_was_major_compacted
++;
1661 c_seg_src
->c_was_major_donor
++;
1663 c_seg_major_compact_stats
.compactions
++;
1665 dst_slot
= c_seg_dst
->c_nextslot
;
1667 for (i
= 0; i
< c_seg_src
->c_nextslot
; i
++) {
1668 c_src
= C_SEG_SLOT_FROM_INDEX(c_seg_src
, i
);
1670 c_size
= UNPACK_C_SIZE(c_src
);
1673 /* BATCH: move what we have so far; */
1677 if (C_SEG_OFFSET_TO_BYTES(c_seg_dst
->c_populated_offset
- c_seg_dst
->c_nextoffset
) < (unsigned) c_size
) {
1678 int size_to_populate
;
1681 size_to_populate
= C_SEG_BUFSIZE
- C_SEG_OFFSET_TO_BYTES(c_seg_dst
->c_populated_offset
);
1683 if (size_to_populate
== 0) {
1685 keep_compacting
= FALSE
;
1688 if (size_to_populate
> C_SEG_MAX_POPULATE_SIZE
) {
1689 size_to_populate
= C_SEG_MAX_POPULATE_SIZE
;
1692 kernel_memory_populate(compressor_map
,
1693 (vm_offset_t
) &c_seg_dst
->c_store
.c_buffer
[c_seg_dst
->c_populated_offset
],
1696 VM_KERN_MEMORY_COMPRESSOR
);
1698 c_seg_dst
->c_populated_offset
+= C_SEG_BYTES_TO_OFFSET(size_to_populate
);
1699 assert(C_SEG_OFFSET_TO_BYTES(c_seg_dst
->c_populated_offset
) <= C_SEG_BUFSIZE
);
1701 c_seg_alloc_nextslot(c_seg_dst
);
1703 c_dst
= C_SEG_SLOT_FROM_INDEX(c_seg_dst
, c_seg_dst
->c_nextslot
);
1705 memcpy(&c_seg_dst
->c_store
.c_buffer
[c_seg_dst
->c_nextoffset
], &c_seg_src
->c_store
.c_buffer
[c_src
->c_offset
], c_size
);
1707 c_rounded_size
= (c_size
+ C_SEG_OFFSET_ALIGNMENT_MASK
) & ~C_SEG_OFFSET_ALIGNMENT_MASK
;
1709 c_seg_major_compact_stats
.moved_slots
++;
1710 c_seg_major_compact_stats
.moved_bytes
+= c_size
;
1712 cslot_copy(c_dst
, c_src
);
1713 c_dst
->c_offset
= c_seg_dst
->c_nextoffset
;
1715 if (c_seg_dst
->c_firstemptyslot
== c_seg_dst
->c_nextslot
) {
1716 c_seg_dst
->c_firstemptyslot
++;
1718 c_seg_dst
->c_slots_used
++;
1719 c_seg_dst
->c_nextslot
++;
1720 c_seg_dst
->c_bytes_used
+= c_rounded_size
;
1721 c_seg_dst
->c_nextoffset
+= C_SEG_BYTES_TO_OFFSET(c_rounded_size
);
1723 PACK_C_SIZE(c_src
, 0);
1725 c_seg_src
->c_bytes_used
-= c_rounded_size
;
1726 c_seg_src
->c_bytes_unused
+= c_rounded_size
;
1727 c_seg_src
->c_firstemptyslot
= 0;
1729 assert(c_seg_src
->c_slots_used
);
1730 c_seg_src
->c_slots_used
--;
1732 if (c_seg_dst
->c_nextoffset
>= C_SEG_OFF_LIMIT
|| c_seg_dst
->c_nextslot
>= C_SLOT_MAX_INDEX
) {
1733 /* dest segment is now full */
1734 keep_compacting
= FALSE
;
1738 #if DEVELOPMENT || DEBUG
1739 C_SEG_WRITE_PROTECT(c_seg_dst
);
1741 if (dst_slot
< c_seg_dst
->c_nextslot
) {
1742 PAGE_REPLACEMENT_ALLOWED(TRUE
);
1744 * we've now locked out c_decompress from
1745 * converting the slot passed into it into
1746 * a c_segment_t which allows us to use
1747 * the backptr to change which c_segment and
1748 * index the slot points to
1750 while (dst_slot
< c_seg_dst
->c_nextslot
) {
1751 c_dst
= C_SEG_SLOT_FROM_INDEX(c_seg_dst
, dst_slot
);
1753 slot_ptr
= (c_slot_mapping_t
)C_SLOT_UNPACK_PTR(c_dst
);
1754 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
1755 slot_ptr
->s_cseg
= c_seg_dst
->c_mysegno
+ 1;
1756 slot_ptr
->s_cindx
= dst_slot
++;
1758 PAGE_REPLACEMENT_ALLOWED(FALSE
);
1760 return keep_compacting
;
1765 vm_compressor_compute_elapsed_msecs(clock_sec_t end_sec
, clock_nsec_t end_nsec
, clock_sec_t start_sec
, clock_nsec_t start_nsec
)
1768 uint64_t start_msecs
;
1770 end_msecs
= (end_sec
* 1000) + end_nsec
/ 1000000;
1771 start_msecs
= (start_sec
* 1000) + start_nsec
/ 1000000;
1773 return end_msecs
- start_msecs
;
1778 uint32_t compressor_eval_period_in_msecs
= 250;
1779 uint32_t compressor_sample_min_in_msecs
= 500;
1780 uint32_t compressor_sample_max_in_msecs
= 10000;
1781 uint32_t compressor_thrashing_threshold_per_10msecs
= 50;
1782 uint32_t compressor_thrashing_min_per_10msecs
= 20;
1784 /* When true, reset sample data next chance we get. */
1785 static boolean_t compressor_need_sample_reset
= FALSE
;
1789 compute_swapout_target_age(void)
1791 clock_sec_t cur_ts_sec
;
1792 clock_nsec_t cur_ts_nsec
;
1793 uint32_t min_operations_needed_in_this_sample
;
1794 uint64_t elapsed_msecs_in_eval
;
1795 uint64_t elapsed_msecs_in_sample
;
1796 boolean_t need_eval_reset
= FALSE
;
1798 clock_get_system_nanotime(&cur_ts_sec
, &cur_ts_nsec
);
1800 elapsed_msecs_in_sample
= vm_compressor_compute_elapsed_msecs(cur_ts_sec
, cur_ts_nsec
, start_of_sample_period_sec
, start_of_sample_period_nsec
);
1802 if (compressor_need_sample_reset
||
1803 elapsed_msecs_in_sample
>= compressor_sample_max_in_msecs
) {
1804 compressor_need_sample_reset
= TRUE
;
1805 need_eval_reset
= TRUE
;
1808 elapsed_msecs_in_eval
= vm_compressor_compute_elapsed_msecs(cur_ts_sec
, cur_ts_nsec
, start_of_eval_period_sec
, start_of_eval_period_nsec
);
1810 if (elapsed_msecs_in_eval
< compressor_eval_period_in_msecs
) {
1813 need_eval_reset
= TRUE
;
1815 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_START
, elapsed_msecs_in_eval
, sample_period_compression_count
, sample_period_decompression_count
, 0, 0);
1817 min_operations_needed_in_this_sample
= (compressor_thrashing_min_per_10msecs
* (uint32_t)elapsed_msecs_in_eval
) / 10;
1819 if ((sample_period_compression_count
- last_eval_compression_count
) < min_operations_needed_in_this_sample
||
1820 (sample_period_decompression_count
- last_eval_decompression_count
) < min_operations_needed_in_this_sample
) {
1821 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END
, sample_period_compression_count
- last_eval_compression_count
,
1822 sample_period_decompression_count
- last_eval_decompression_count
, 0, 1, 0);
1824 swapout_target_age
= 0;
1826 compressor_need_sample_reset
= TRUE
;
1827 need_eval_reset
= TRUE
;
1830 last_eval_compression_count
= sample_period_compression_count
;
1831 last_eval_decompression_count
= sample_period_decompression_count
;
1833 if (elapsed_msecs_in_sample
< compressor_sample_min_in_msecs
) {
1834 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END
, swapout_target_age
, 0, 0, 5, 0);
1837 if (sample_period_decompression_count
> ((compressor_thrashing_threshold_per_10msecs
* elapsed_msecs_in_sample
) / 10)) {
1838 uint64_t running_total
;
1839 uint64_t working_target
;
1840 uint64_t aging_target
;
1841 uint32_t oldest_age_of_csegs_sampled
= 0;
1842 uint64_t working_set_approximation
= 0;
1844 swapout_target_age
= 0;
1846 working_target
= (sample_period_decompression_count
/ 100) * 95; /* 95 percent */
1847 aging_target
= (sample_period_decompression_count
/ 100) * 1; /* 1 percent */
1850 for (oldest_age_of_csegs_sampled
= 0; oldest_age_of_csegs_sampled
< DECOMPRESSION_SAMPLE_MAX_AGE
; oldest_age_of_csegs_sampled
++) {
1851 running_total
+= age_of_decompressions_during_sample_period
[oldest_age_of_csegs_sampled
];
1853 working_set_approximation
+= oldest_age_of_csegs_sampled
* age_of_decompressions_during_sample_period
[oldest_age_of_csegs_sampled
];
1855 if (running_total
>= working_target
) {
1859 if (oldest_age_of_csegs_sampled
< DECOMPRESSION_SAMPLE_MAX_AGE
) {
1860 working_set_approximation
= (working_set_approximation
* 1000) / elapsed_msecs_in_sample
;
1862 if (working_set_approximation
< VM_PAGE_COMPRESSOR_COUNT
) {
1863 running_total
= overage_decompressions_during_sample_period
;
1865 for (oldest_age_of_csegs_sampled
= DECOMPRESSION_SAMPLE_MAX_AGE
- 1; oldest_age_of_csegs_sampled
; oldest_age_of_csegs_sampled
--) {
1866 running_total
+= age_of_decompressions_during_sample_period
[oldest_age_of_csegs_sampled
];
1868 if (running_total
>= aging_target
) {
1872 swapout_target_age
= (uint32_t)cur_ts_sec
- oldest_age_of_csegs_sampled
;
1874 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END
, swapout_target_age
, working_set_approximation
, VM_PAGE_COMPRESSOR_COUNT
, 2, 0);
1876 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END
, working_set_approximation
, VM_PAGE_COMPRESSOR_COUNT
, 0, 3, 0);
1879 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END
, working_target
, running_total
, 0, 4, 0);
1882 compressor_need_sample_reset
= TRUE
;
1883 need_eval_reset
= TRUE
;
1885 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END
, sample_period_decompression_count
, (compressor_thrashing_threshold_per_10msecs
* elapsed_msecs_in_sample
) / 10, 0, 6, 0);
1888 if (compressor_need_sample_reset
== TRUE
) {
1889 bzero(age_of_decompressions_during_sample_period
, sizeof(age_of_decompressions_during_sample_period
));
1890 overage_decompressions_during_sample_period
= 0;
1892 start_of_sample_period_sec
= cur_ts_sec
;
1893 start_of_sample_period_nsec
= cur_ts_nsec
;
1894 sample_period_decompression_count
= 0;
1895 sample_period_compression_count
= 0;
1896 last_eval_decompression_count
= 0;
1897 last_eval_compression_count
= 0;
1898 compressor_need_sample_reset
= FALSE
;
1900 if (need_eval_reset
== TRUE
) {
1901 start_of_eval_period_sec
= cur_ts_sec
;
1902 start_of_eval_period_nsec
= cur_ts_nsec
;
1907 int compaction_swapper_init_now
= 0;
1908 int compaction_swapper_running
= 0;
1909 int compaction_swapper_awakened
= 0;
1910 int compaction_swapper_abort
= 0;
1914 boolean_t
memorystatus_kill_on_VM_compressor_thrashing(boolean_t
);
1915 boolean_t
memorystatus_kill_on_VM_compressor_space_shortage(boolean_t
);
1916 boolean_t
memorystatus_kill_on_FC_thrashing(boolean_t
);
1917 int compressor_thrashing_induced_jetsam
= 0;
1918 int filecache_thrashing_induced_jetsam
= 0;
1919 static boolean_t vm_compressor_thrashing_detected
= FALSE
;
1920 #endif /* CONFIG_JETSAM */
1923 compressor_needs_to_swap(void)
1925 boolean_t should_swap
= FALSE
;
1927 if (vm_swapout_ripe_segments
== TRUE
&& c_overage_swapped_count
< c_overage_swapped_limit
) {
1933 clock_get_system_nanotime(&now
, &nsec
);
1936 lck_mtx_lock_spin_always(c_list_lock
);
1938 if (!queue_empty(&c_age_list_head
)) {
1939 c_seg
= (c_segment_t
) queue_first(&c_age_list_head
);
1941 age
= now
- c_seg
->c_creation_ts
;
1943 lck_mtx_unlock_always(c_list_lock
);
1945 if (age
>= vm_ripe_target_age
) {
1949 if (VM_CONFIG_SWAP_IS_ACTIVE
) {
1950 if (COMPRESSOR_NEEDS_TO_SWAP()) {
1953 if (VM_PAGE_Q_THROTTLED(&vm_pageout_queue_external
) && vm_page_anonymous_count
< (vm_page_inactive_count
/ 20)) {
1956 if (vm_page_free_count
< (vm_page_free_reserved
- (COMPRESSOR_FREE_RESERVED_LIMIT
* 2))) {
1960 compute_swapout_target_age();
1962 if (swapout_target_age
) {
1965 lck_mtx_lock_spin_always(c_list_lock
);
1967 if (!queue_empty(&c_age_list_head
)) {
1968 c_seg
= (c_segment_t
) queue_first(&c_age_list_head
);
1970 if (c_seg
->c_creation_ts
> swapout_target_age
) {
1971 swapout_target_age
= 0;
1974 lck_mtx_unlock_always(c_list_lock
);
1976 #if CONFIG_PHANTOM_CACHE
1977 if (vm_phantom_cache_check_pressure()) {
1981 if (swapout_target_age
) {
1986 if (should_swap
|| vm_compressor_low_on_space() == TRUE
) {
1987 if (vm_compressor_thrashing_detected
== FALSE
) {
1988 vm_compressor_thrashing_detected
= TRUE
;
1990 if (swapout_target_age
|| vm_compressor_low_on_space() == TRUE
) {
1991 if (swapout_target_age
) {
1992 /* The compressor is thrashing. */
1993 memorystatus_kill_on_VM_compressor_thrashing(TRUE
/* async */);
1995 /* The compressor is running low on space. */
1996 memorystatus_kill_on_VM_compressor_space_shortage(TRUE
/* async */);
1998 compressor_thrashing_induced_jetsam
++;
2000 memorystatus_kill_on_FC_thrashing(TRUE
/* async */);
2001 filecache_thrashing_induced_jetsam
++;
2005 * let the jetsam take precedence over
2006 * any major compactions we might have
2007 * been able to do... otherwise we run
2008 * the risk of doing major compactions
2009 * on segments we're about to free up
2010 * due to the jetsam activity.
2012 should_swap
= FALSE
;
2015 #endif /* CONFIG_JETSAM */
2017 if (should_swap
== FALSE
) {
2019 * vm_compressor_needs_to_major_compact returns true only if we're
2020 * about to run out of available compressor segments... in this
2021 * case, we absolutely need to run a major compaction even if
2022 * we've just kicked off a jetsam or we don't otherwise need to
2023 * swap... terminating objects releases
2024 * pages back to the uncompressed cache, but does not guarantee
2025 * that we will free up even a single compression segment
2027 should_swap
= vm_compressor_needs_to_major_compact();
2031 * returning TRUE when swap_supported == FALSE
2032 * will cause the major compaction engine to
2033 * run, but will not trigger any swapping...
2034 * segments that have been major compacted
2035 * will be moved to the majorcompact queue
2042 * This function is called from the jetsam thread after killing something to
2043 * mitigate thrashing.
2045 * We need to restart our thrashing detection heuristics since memory pressure
2046 * has potentially changed significantly, and we don't want to detect on old
2047 * data from before the jetsam.
2050 vm_thrashing_jetsam_done(void)
2052 vm_compressor_thrashing_detected
= FALSE
;
2054 /* Were we compressor-thrashing or filecache-thrashing? */
2055 if (swapout_target_age
) {
2056 swapout_target_age
= 0;
2057 compressor_need_sample_reset
= TRUE
;
2059 #if CONFIG_PHANTOM_CACHE
2061 vm_phantom_cache_restart_sample();
2065 #endif /* CONFIG_JETSAM */
2067 uint32_t vm_wake_compactor_swapper_calls
= 0;
2068 uint32_t vm_run_compactor_already_running
= 0;
2069 uint32_t vm_run_compactor_empty_minor_q
= 0;
2070 uint32_t vm_run_compactor_did_compact
= 0;
2071 uint32_t vm_run_compactor_waited
= 0;
2074 vm_run_compactor(void)
2076 if (c_segment_count
== 0) {
2080 lck_mtx_lock_spin_always(c_list_lock
);
2082 if (c_minor_count
== 0) {
2083 vm_run_compactor_empty_minor_q
++;
2085 lck_mtx_unlock_always(c_list_lock
);
2088 if (compaction_swapper_running
) {
2089 if (vm_pageout_state
.vm_restricted_to_single_processor
== FALSE
) {
2090 vm_run_compactor_already_running
++;
2092 lck_mtx_unlock_always(c_list_lock
);
2095 vm_run_compactor_waited
++;
2097 assert_wait((event_t
)&compaction_swapper_running
, THREAD_UNINT
);
2099 lck_mtx_unlock_always(c_list_lock
);
2101 thread_block(THREAD_CONTINUE_NULL
);
2105 vm_run_compactor_did_compact
++;
2107 fastwake_warmup
= FALSE
;
2108 compaction_swapper_running
= 1;
2110 vm_compressor_do_delayed_compactions(FALSE
);
2112 compaction_swapper_running
= 0;
2114 lck_mtx_unlock_always(c_list_lock
);
2116 thread_wakeup((event_t
)&compaction_swapper_running
);
2121 vm_wake_compactor_swapper(void)
2123 if (compaction_swapper_running
|| compaction_swapper_awakened
|| c_segment_count
== 0) {
2127 if (c_minor_count
|| vm_compressor_needs_to_major_compact()) {
2128 lck_mtx_lock_spin_always(c_list_lock
);
2130 fastwake_warmup
= FALSE
;
2132 if (compaction_swapper_running
== 0 && compaction_swapper_awakened
== 0) {
2133 vm_wake_compactor_swapper_calls
++;
2135 compaction_swapper_awakened
= 1;
2136 thread_wakeup((event_t
)&c_compressor_swap_trigger
);
2138 lck_mtx_unlock_always(c_list_lock
);
2144 vm_consider_swapping()
2146 c_segment_t c_seg
, c_seg_next
;
2150 assert(VM_CONFIG_SWAP_IS_PRESENT
);
2152 lck_mtx_lock_spin_always(c_list_lock
);
2154 compaction_swapper_abort
= 1;
2156 while (compaction_swapper_running
) {
2157 assert_wait((event_t
)&compaction_swapper_running
, THREAD_UNINT
);
2159 lck_mtx_unlock_always(c_list_lock
);
2161 thread_block(THREAD_CONTINUE_NULL
);
2163 lck_mtx_lock_spin_always(c_list_lock
);
2165 compaction_swapper_abort
= 0;
2166 compaction_swapper_running
= 1;
2168 vm_swapout_ripe_segments
= TRUE
;
2170 if (!queue_empty(&c_major_list_head
)) {
2171 clock_get_system_nanotime(&now
, &nsec
);
2173 c_seg
= (c_segment_t
)queue_first(&c_major_list_head
);
2175 while (!queue_end(&c_major_list_head
, (queue_entry_t
)c_seg
)) {
2176 if (c_overage_swapped_count
>= c_overage_swapped_limit
) {
2180 c_seg_next
= (c_segment_t
) queue_next(&c_seg
->c_age_list
);
2182 if ((now
- c_seg
->c_creation_ts
) >= vm_ripe_target_age
) {
2183 lck_mtx_lock_spin_always(&c_seg
->c_lock
);
2185 c_seg_switch_state(c_seg
, C_ON_AGE_Q
, FALSE
);
2187 lck_mtx_unlock_always(&c_seg
->c_lock
);
2192 vm_compressor_compact_and_swap(FALSE
);
2194 compaction_swapper_running
= 0;
2196 vm_swapout_ripe_segments
= FALSE
;
2198 lck_mtx_unlock_always(c_list_lock
);
2200 thread_wakeup((event_t
)&compaction_swapper_running
);
2205 vm_consider_waking_compactor_swapper(void)
2207 boolean_t need_wakeup
= FALSE
;
2209 if (c_segment_count
== 0) {
2213 if (compaction_swapper_running
|| compaction_swapper_awakened
) {
2217 if (!compaction_swapper_inited
&& !compaction_swapper_init_now
) {
2218 compaction_swapper_init_now
= 1;
2222 if (c_minor_count
&& (COMPRESSOR_NEEDS_TO_MINOR_COMPACT())) {
2224 } else if (compressor_needs_to_swap()) {
2226 } else if (c_minor_count
) {
2227 uint64_t total_bytes
;
2229 total_bytes
= compressor_object
->resident_page_count
* PAGE_SIZE_64
;
2231 if ((total_bytes
- compressor_bytes_used
) > total_bytes
/ 10) {
2235 if (need_wakeup
== TRUE
) {
2236 lck_mtx_lock_spin_always(c_list_lock
);
2238 fastwake_warmup
= FALSE
;
2240 if (compaction_swapper_running
== 0 && compaction_swapper_awakened
== 0) {
2241 memoryshot(VM_WAKEUP_COMPACTOR_SWAPPER
, DBG_FUNC_NONE
);
2243 compaction_swapper_awakened
= 1;
2244 thread_wakeup((event_t
)&c_compressor_swap_trigger
);
2246 lck_mtx_unlock_always(c_list_lock
);
2251 #define C_SWAPOUT_LIMIT 4
2252 #define DELAYED_COMPACTIONS_PER_PASS 30
2255 vm_compressor_do_delayed_compactions(boolean_t flush_all
)
2258 int number_compacted
= 0;
2259 boolean_t needs_to_swap
= FALSE
;
2262 #if !CONFIG_EMBEDDED
2263 LCK_MTX_ASSERT(c_list_lock
, LCK_MTX_ASSERT_OWNED
);
2264 #endif /* !CONFIG_EMBEDDED */
2266 while (!queue_empty(&c_minor_list_head
) && needs_to_swap
== FALSE
) {
2267 c_seg
= (c_segment_t
)queue_first(&c_minor_list_head
);
2269 lck_mtx_lock_spin_always(&c_seg
->c_lock
);
2271 if (c_seg
->c_busy
) {
2272 lck_mtx_unlock_always(c_list_lock
);
2273 c_seg_wait_on_busy(c_seg
);
2274 lck_mtx_lock_spin_always(c_list_lock
);
2280 c_seg_do_minor_compaction_and_unlock(c_seg
, TRUE
, FALSE
, TRUE
);
2282 if (VM_CONFIG_SWAP_IS_ACTIVE
&& (number_compacted
++ > DELAYED_COMPACTIONS_PER_PASS
)) {
2283 if ((flush_all
== TRUE
|| compressor_needs_to_swap() == TRUE
) && c_swapout_count
< C_SWAPOUT_LIMIT
) {
2284 needs_to_swap
= TRUE
;
2287 number_compacted
= 0;
2289 lck_mtx_lock_spin_always(c_list_lock
);
2294 #define C_SEGMENT_SWAPPEDIN_AGE_LIMIT 10
2297 vm_compressor_age_swapped_in_segments(boolean_t flush_all
)
2303 clock_get_system_nanotime(&now
, &nsec
);
2305 while (!queue_empty(&c_swappedin_list_head
)) {
2306 c_seg
= (c_segment_t
)queue_first(&c_swappedin_list_head
);
2308 if (flush_all
== FALSE
&& (now
- c_seg
->c_swappedin_ts
) < C_SEGMENT_SWAPPEDIN_AGE_LIMIT
) {
2312 lck_mtx_lock_spin_always(&c_seg
->c_lock
);
2314 c_seg_switch_state(c_seg
, C_ON_AGE_Q
, FALSE
);
2316 lck_mtx_unlock_always(&c_seg
->c_lock
);
2321 extern int vm_num_swap_files
;
2322 extern int vm_num_pinned_swap_files
;
2323 extern int vm_swappin_enabled
;
2325 extern unsigned int vm_swapfile_total_segs_used
;
2326 extern unsigned int vm_swapfile_total_segs_alloced
;
2330 vm_compressor_flush(void)
2332 uint64_t vm_swap_put_failures_at_start
;
2333 wait_result_t wait_result
= 0;
2334 AbsoluteTime startTime
, endTime
;
2335 clock_sec_t now_sec
;
2336 clock_nsec_t now_nsec
;
2339 HIBLOG("vm_compressor_flush - starting\n");
2341 clock_get_uptime(&startTime
);
2343 lck_mtx_lock_spin_always(c_list_lock
);
2345 fastwake_warmup
= FALSE
;
2346 compaction_swapper_abort
= 1;
2348 while (compaction_swapper_running
) {
2349 assert_wait((event_t
)&compaction_swapper_running
, THREAD_UNINT
);
2351 lck_mtx_unlock_always(c_list_lock
);
2353 thread_block(THREAD_CONTINUE_NULL
);
2355 lck_mtx_lock_spin_always(c_list_lock
);
2357 compaction_swapper_abort
= 0;
2358 compaction_swapper_running
= 1;
2360 hibernate_flushing
= TRUE
;
2361 hibernate_no_swapspace
= FALSE
;
2362 c_generation_id_flush_barrier
= c_generation_id
+ 1000;
2364 clock_get_system_nanotime(&now_sec
, &now_nsec
);
2365 hibernate_flushing_deadline
= now_sec
+ HIBERNATE_FLUSHING_SECS_TO_COMPLETE
;
2367 vm_swap_put_failures_at_start
= vm_swap_put_failures
;
2369 vm_compressor_compact_and_swap(TRUE
);
2371 while (!queue_empty(&c_swapout_list_head
)) {
2372 assert_wait_timeout((event_t
) &compaction_swapper_running
, THREAD_INTERRUPTIBLE
, 5000, 1000 * NSEC_PER_USEC
);
2374 lck_mtx_unlock_always(c_list_lock
);
2376 wait_result
= thread_block(THREAD_CONTINUE_NULL
);
2378 lck_mtx_lock_spin_always(c_list_lock
);
2380 if (wait_result
== THREAD_TIMED_OUT
) {
2384 hibernate_flushing
= FALSE
;
2385 compaction_swapper_running
= 0;
2387 if (vm_swap_put_failures
> vm_swap_put_failures_at_start
) {
2388 HIBLOG("vm_compressor_flush failed to clean %llu segments - vm_page_compressor_count(%d)\n",
2389 vm_swap_put_failures
- vm_swap_put_failures_at_start
, VM_PAGE_COMPRESSOR_COUNT
);
2392 lck_mtx_unlock_always(c_list_lock
);
2394 thread_wakeup((event_t
)&compaction_swapper_running
);
2396 clock_get_uptime(&endTime
);
2397 SUB_ABSOLUTETIME(&endTime
, &startTime
);
2398 absolutetime_to_nanoseconds(endTime
, &nsec
);
2400 HIBLOG("vm_compressor_flush completed - took %qd msecs - vm_num_swap_files = %d, vm_num_pinned_swap_files = %d, vm_swappin_enabled = %d\n",
2401 nsec
/ 1000000ULL, vm_num_swap_files
, vm_num_pinned_swap_files
, vm_swappin_enabled
);
2405 int compaction_swap_trigger_thread_awakened
= 0;
2408 vm_compressor_swap_trigger_thread(void)
2410 current_thread()->options
|= TH_OPT_VMPRIV
;
2413 * compaction_swapper_init_now is set when the first call to
2414 * vm_consider_waking_compactor_swapper is made from
2415 * vm_pageout_scan... since this function is called upon
2416 * thread creation, we want to make sure to delay adjusting
2417 * the tuneables until we are awakened via vm_pageout_scan
2418 * so that we are at a point where the vm_swapfile_open will
2419 * be operating on the correct directory (in case the default
2420 * of /var/vm/ is overridden by the dymanic_pager
2422 if (compaction_swapper_init_now
) {
2423 vm_compaction_swapper_do_init();
2425 if (vm_pageout_state
.vm_restricted_to_single_processor
== TRUE
) {
2426 thread_vm_bind_group_add();
2428 thread_set_thread_name(current_thread(), "VM_cswap_trigger");
2429 compaction_swapper_init_now
= 0;
2431 lck_mtx_lock_spin_always(c_list_lock
);
2433 compaction_swap_trigger_thread_awakened
++;
2434 compaction_swapper_awakened
= 0;
2436 if (compaction_swapper_running
== 0) {
2437 compaction_swapper_running
= 1;
2439 vm_compressor_compact_and_swap(FALSE
);
2441 compaction_swapper_running
= 0;
2443 assert_wait((event_t
)&c_compressor_swap_trigger
, THREAD_UNINT
);
2445 if (compaction_swapper_running
== 0) {
2446 thread_wakeup((event_t
)&compaction_swapper_running
);
2449 lck_mtx_unlock_always(c_list_lock
);
2451 thread_block((thread_continue_t
)vm_compressor_swap_trigger_thread
);
2458 vm_compressor_record_warmup_start(void)
2462 lck_mtx_lock_spin_always(c_list_lock
);
2464 if (first_c_segment_to_warm_generation_id
== 0) {
2465 if (!queue_empty(&c_age_list_head
)) {
2466 c_seg
= (c_segment_t
)queue_last(&c_age_list_head
);
2468 first_c_segment_to_warm_generation_id
= c_seg
->c_generation_id
;
2470 first_c_segment_to_warm_generation_id
= 0;
2473 fastwake_recording_in_progress
= TRUE
;
2475 lck_mtx_unlock_always(c_list_lock
);
2480 vm_compressor_record_warmup_end(void)
2484 lck_mtx_lock_spin_always(c_list_lock
);
2486 if (fastwake_recording_in_progress
== TRUE
) {
2487 if (!queue_empty(&c_age_list_head
)) {
2488 c_seg
= (c_segment_t
)queue_last(&c_age_list_head
);
2490 last_c_segment_to_warm_generation_id
= c_seg
->c_generation_id
;
2492 last_c_segment_to_warm_generation_id
= first_c_segment_to_warm_generation_id
;
2495 fastwake_recording_in_progress
= FALSE
;
2497 HIBLOG("vm_compressor_record_warmup (%qd - %qd)\n", first_c_segment_to_warm_generation_id
, last_c_segment_to_warm_generation_id
);
2499 lck_mtx_unlock_always(c_list_lock
);
2503 #define DELAY_TRIM_ON_WAKE_SECS 25
2506 vm_compressor_delay_trim(void)
2511 clock_get_system_nanotime(&sec
, &nsec
);
2512 dont_trim_until_ts
= sec
+ DELAY_TRIM_ON_WAKE_SECS
;
2517 vm_compressor_do_warmup(void)
2519 lck_mtx_lock_spin_always(c_list_lock
);
2521 if (first_c_segment_to_warm_generation_id
== last_c_segment_to_warm_generation_id
) {
2522 first_c_segment_to_warm_generation_id
= last_c_segment_to_warm_generation_id
= 0;
2524 lck_mtx_unlock_always(c_list_lock
);
2528 if (compaction_swapper_running
== 0 && compaction_swapper_awakened
== 0) {
2529 fastwake_warmup
= TRUE
;
2531 compaction_swapper_awakened
= 1;
2532 thread_wakeup((event_t
)&c_compressor_swap_trigger
);
2534 lck_mtx_unlock_always(c_list_lock
);
2538 do_fastwake_warmup_all(void)
2540 lck_mtx_lock_spin_always(c_list_lock
);
2542 if (queue_empty(&c_swappedout_list_head
) && queue_empty(&c_swappedout_sparse_list_head
)) {
2543 lck_mtx_unlock_always(c_list_lock
);
2547 fastwake_warmup
= TRUE
;
2549 do_fastwake_warmup(&c_swappedout_list_head
, TRUE
);
2551 do_fastwake_warmup(&c_swappedout_sparse_list_head
, TRUE
);
2553 fastwake_warmup
= FALSE
;
2555 lck_mtx_unlock_always(c_list_lock
);
2559 do_fastwake_warmup(queue_head_t
*c_queue
, boolean_t consider_all_cseg
)
2561 c_segment_t c_seg
= NULL
;
2562 AbsoluteTime startTime
, endTime
;
2566 HIBLOG("vm_compressor_fastwake_warmup (%qd - %qd) - starting\n", first_c_segment_to_warm_generation_id
, last_c_segment_to_warm_generation_id
);
2568 clock_get_uptime(&startTime
);
2570 lck_mtx_unlock_always(c_list_lock
);
2572 proc_set_thread_policy(current_thread(),
2573 TASK_POLICY_INTERNAL
, TASK_POLICY_IO
, THROTTLE_LEVEL_COMPRESSOR_TIER2
);
2575 PAGE_REPLACEMENT_DISALLOWED(TRUE
);
2577 lck_mtx_lock_spin_always(c_list_lock
);
2579 while (!queue_empty(c_queue
) && fastwake_warmup
== TRUE
) {
2580 c_seg
= (c_segment_t
) queue_first(c_queue
);
2582 if (consider_all_cseg
== FALSE
) {
2583 if (c_seg
->c_generation_id
< first_c_segment_to_warm_generation_id
||
2584 c_seg
->c_generation_id
> last_c_segment_to_warm_generation_id
) {
2588 if (vm_page_free_count
< (AVAILABLE_MEMORY
/ 4)) {
2593 lck_mtx_lock_spin_always(&c_seg
->c_lock
);
2594 lck_mtx_unlock_always(c_list_lock
);
2596 if (c_seg
->c_busy
) {
2597 PAGE_REPLACEMENT_DISALLOWED(FALSE
);
2598 c_seg_wait_on_busy(c_seg
);
2599 PAGE_REPLACEMENT_DISALLOWED(TRUE
);
2601 if (c_seg_swapin(c_seg
, TRUE
, FALSE
) == 0) {
2602 lck_mtx_unlock_always(&c_seg
->c_lock
);
2604 c_segment_warmup_count
++;
2606 PAGE_REPLACEMENT_DISALLOWED(FALSE
);
2607 vm_pageout_io_throttle();
2608 PAGE_REPLACEMENT_DISALLOWED(TRUE
);
2610 lck_mtx_lock_spin_always(c_list_lock
);
2612 lck_mtx_unlock_always(c_list_lock
);
2614 PAGE_REPLACEMENT_DISALLOWED(FALSE
);
2616 proc_set_thread_policy(current_thread(),
2617 TASK_POLICY_INTERNAL
, TASK_POLICY_IO
, THROTTLE_LEVEL_COMPRESSOR_TIER0
);
2619 clock_get_uptime(&endTime
);
2620 SUB_ABSOLUTETIME(&endTime
, &startTime
);
2621 absolutetime_to_nanoseconds(endTime
, &nsec
);
2623 HIBLOG("vm_compressor_fastwake_warmup completed - took %qd msecs\n", nsec
/ 1000000ULL);
2625 lck_mtx_lock_spin_always(c_list_lock
);
2627 if (consider_all_cseg
== FALSE
) {
2628 first_c_segment_to_warm_generation_id
= last_c_segment_to_warm_generation_id
= 0;
2634 vm_compressor_compact_and_swap(boolean_t flush_all
)
2636 c_segment_t c_seg
, c_seg_next
;
2637 boolean_t keep_compacting
;
2642 if (fastwake_warmup
== TRUE
) {
2643 uint64_t starting_warmup_count
;
2645 starting_warmup_count
= c_segment_warmup_count
;
2647 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE
, 11) | DBG_FUNC_START
, c_segment_warmup_count
,
2648 first_c_segment_to_warm_generation_id
, last_c_segment_to_warm_generation_id
, 0, 0);
2649 do_fastwake_warmup(&c_swappedout_list_head
, FALSE
);
2650 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE
, 11) | DBG_FUNC_END
, c_segment_warmup_count
, c_segment_warmup_count
- starting_warmup_count
, 0, 0, 0);
2652 fastwake_warmup
= FALSE
;
2656 * it's possible for the c_age_list_head to be empty if we
2657 * hit our limits for growing the compressor pool and we subsequently
2658 * hibernated... on the next hibernation we could see the queue as
2659 * empty and not proceeed even though we have a bunch of segments on
2660 * the swapped in queue that need to be dealt with.
2662 vm_compressor_do_delayed_compactions(flush_all
);
2664 vm_compressor_age_swapped_in_segments(flush_all
);
2667 * we only need to grab the timestamp once per
2668 * invocation of this function since the
2669 * timescale we're interested in is measured
2672 clock_get_system_nanotime(&now
, &nsec
);
2674 while (!queue_empty(&c_age_list_head
) && compaction_swapper_abort
== 0) {
2675 if (hibernate_flushing
== TRUE
) {
2678 if (hibernate_should_abort()) {
2679 HIBLOG("vm_compressor_flush - hibernate_should_abort returned TRUE\n");
2682 if (hibernate_no_swapspace
== TRUE
) {
2683 HIBLOG("vm_compressor_flush - out of swap space\n");
2686 if (vm_swap_files_pinned() == FALSE
) {
2687 HIBLOG("vm_compressor_flush - unpinned swap files\n");
2690 if (hibernate_in_progress_with_pinned_swap
== TRUE
&&
2691 (vm_swapfile_total_segs_alloced
== vm_swapfile_total_segs_used
)) {
2692 HIBLOG("vm_compressor_flush - out of pinned swap space\n");
2695 clock_get_system_nanotime(&sec
, &nsec
);
2697 if (sec
> hibernate_flushing_deadline
) {
2698 HIBLOG("vm_compressor_flush - failed to finish before deadline\n");
2702 if (c_swapout_count
>= C_SWAPOUT_LIMIT
) {
2703 assert_wait_timeout((event_t
) &compaction_swapper_running
, THREAD_INTERRUPTIBLE
, 100, 1000 * NSEC_PER_USEC
);
2705 lck_mtx_unlock_always(c_list_lock
);
2707 thread_block(THREAD_CONTINUE_NULL
);
2709 lck_mtx_lock_spin_always(c_list_lock
);
2714 vm_compressor_do_delayed_compactions(flush_all
);
2716 vm_compressor_age_swapped_in_segments(flush_all
);
2718 if (c_swapout_count
>= C_SWAPOUT_LIMIT
) {
2720 * we timed out on the above thread_block
2721 * let's loop around and try again
2722 * the timeout allows us to continue
2723 * to do minor compactions to make
2724 * more memory available
2730 * Swap out segments?
2732 if (flush_all
== FALSE
) {
2733 boolean_t needs_to_swap
;
2735 lck_mtx_unlock_always(c_list_lock
);
2737 needs_to_swap
= compressor_needs_to_swap();
2739 #if !CONFIG_EMBEDDED
2740 if (needs_to_swap
== TRUE
&& vm_swap_low_on_space()) {
2741 vm_compressor_take_paging_space_action();
2743 #endif /* !CONFIG_EMBEDDED */
2745 lck_mtx_lock_spin_always(c_list_lock
);
2747 if (needs_to_swap
== FALSE
) {
2751 if (queue_empty(&c_age_list_head
)) {
2754 c_seg
= (c_segment_t
) queue_first(&c_age_list_head
);
2756 assert(c_seg
->c_state
== C_ON_AGE_Q
);
2758 if (flush_all
== TRUE
&& c_seg
->c_generation_id
> c_generation_id_flush_barrier
) {
2762 lck_mtx_lock_spin_always(&c_seg
->c_lock
);
2764 if (c_seg
->c_busy
) {
2765 lck_mtx_unlock_always(c_list_lock
);
2766 c_seg_wait_on_busy(c_seg
);
2767 lck_mtx_lock_spin_always(c_list_lock
);
2773 if (c_seg_do_minor_compaction_and_unlock(c_seg
, FALSE
, TRUE
, TRUE
)) {
2775 * found an empty c_segment and freed it
2776 * so go grab the next guy in the queue
2778 c_seg_major_compact_stats
.count_of_freed_segs
++;
2784 keep_compacting
= TRUE
;
2786 while (keep_compacting
== TRUE
) {
2787 assert(c_seg
->c_busy
);
2789 /* look for another segment to consolidate */
2791 c_seg_next
= (c_segment_t
) queue_next(&c_seg
->c_age_list
);
2793 if (queue_end(&c_age_list_head
, (queue_entry_t
)c_seg_next
)) {
2797 assert(c_seg_next
->c_state
== C_ON_AGE_Q
);
2799 if (c_seg_major_compact_ok(c_seg
, c_seg_next
) == FALSE
) {
2803 lck_mtx_lock_spin_always(&c_seg_next
->c_lock
);
2805 if (c_seg_next
->c_busy
) {
2806 lck_mtx_unlock_always(c_list_lock
);
2807 c_seg_wait_on_busy(c_seg_next
);
2808 lck_mtx_lock_spin_always(c_list_lock
);
2812 /* grab that segment */
2813 C_SEG_BUSY(c_seg_next
);
2815 if (c_seg_do_minor_compaction_and_unlock(c_seg_next
, FALSE
, TRUE
, TRUE
)) {
2817 * found an empty c_segment and freed it
2818 * so we can't continue to use c_seg_next
2820 c_seg_major_compact_stats
.count_of_freed_segs
++;
2824 /* unlock the list ... */
2825 lck_mtx_unlock_always(c_list_lock
);
2827 /* do the major compaction */
2829 keep_compacting
= c_seg_major_compact(c_seg
, c_seg_next
);
2831 PAGE_REPLACEMENT_DISALLOWED(TRUE
);
2833 lck_mtx_lock_spin_always(&c_seg_next
->c_lock
);
2835 * run a minor compaction on the donor segment
2836 * since we pulled at least some of it's
2837 * data into our target... if we've emptied
2838 * it, now is a good time to free it which
2839 * c_seg_minor_compaction_and_unlock also takes care of
2841 * by passing TRUE, we ask for c_busy to be cleared
2842 * and c_wanted to be taken care of
2844 if (c_seg_minor_compaction_and_unlock(c_seg_next
, TRUE
)) {
2845 c_seg_major_compact_stats
.count_of_freed_segs
++;
2848 PAGE_REPLACEMENT_DISALLOWED(FALSE
);
2850 /* relock the list */
2851 lck_mtx_lock_spin_always(c_list_lock
);
2852 } /* major compaction */
2854 lck_mtx_lock_spin_always(&c_seg
->c_lock
);
2856 assert(c_seg
->c_busy
);
2857 assert(!c_seg
->c_on_minorcompact_q
);
2859 if (VM_CONFIG_SWAP_IS_ACTIVE
) {
2861 * This mode of putting a generic c_seg on the swapout list is
2862 * only supported when we have general swapping enabled
2864 c_seg_switch_state(c_seg
, C_ON_SWAPOUT_Q
, FALSE
);
2866 if ((vm_swapout_ripe_segments
== TRUE
&& c_overage_swapped_count
< c_overage_swapped_limit
)) {
2867 assert(VM_CONFIG_SWAP_IS_PRESENT
);
2869 * we are running compressor sweeps with swap-behind
2870 * make sure the c_seg has aged enough before swapping it
2873 if ((now
- c_seg
->c_creation_ts
) >= vm_ripe_target_age
) {
2874 c_seg
->c_overage_swap
= TRUE
;
2875 c_overage_swapped_count
++;
2876 c_seg_switch_state(c_seg
, C_ON_SWAPOUT_Q
, FALSE
);
2880 if (c_seg
->c_state
== C_ON_AGE_Q
) {
2882 * this c_seg didn't get moved to the swapout queue
2883 * so we need to move it out of the way...
2884 * we just did a major compaction on it so put it
2887 c_seg_switch_state(c_seg
, C_ON_MAJORCOMPACT_Q
, FALSE
);
2889 c_seg_major_compact_stats
.wasted_space_in_swapouts
+= C_SEG_BUFSIZE
- c_seg
->c_bytes_used
;
2890 c_seg_major_compact_stats
.count_of_swapouts
++;
2892 C_SEG_WAKEUP_DONE(c_seg
);
2894 lck_mtx_unlock_always(&c_seg
->c_lock
);
2896 if (c_swapout_count
) {
2897 lck_mtx_unlock_always(c_list_lock
);
2899 thread_wakeup((event_t
)&c_swapout_list_head
);
2901 lck_mtx_lock_spin_always(c_list_lock
);
2908 c_seg_allocate(c_segment_t
*current_chead
)
2912 int size_to_populate
;
2914 #if !CONFIG_EMBEDDED
2915 if (vm_compressor_low_on_space()) {
2916 vm_compressor_take_paging_space_action();
2918 #endif /* !CONFIG_EMBEDDED */
2920 if ((c_seg
= *current_chead
) == NULL
) {
2923 lck_mtx_lock_spin_always(c_list_lock
);
2925 while (c_segments_busy
== TRUE
) {
2926 assert_wait((event_t
) (&c_segments_busy
), THREAD_UNINT
);
2928 lck_mtx_unlock_always(c_list_lock
);
2930 thread_block(THREAD_CONTINUE_NULL
);
2932 lck_mtx_lock_spin_always(c_list_lock
);
2934 if (c_free_segno_head
== (uint32_t)-1) {
2935 uint32_t c_segments_available_new
;
2937 if (c_segments_available
>= c_segments_limit
|| c_segment_pages_compressed
>= c_segment_pages_compressed_limit
) {
2938 lck_mtx_unlock_always(c_list_lock
);
2942 c_segments_busy
= TRUE
;
2943 lck_mtx_unlock_always(c_list_lock
);
2945 kernel_memory_populate(compressor_map
, (vm_offset_t
)c_segments_next_page
,
2946 PAGE_SIZE
, KMA_KOBJECT
, VM_KERN_MEMORY_COMPRESSOR
);
2947 c_segments_next_page
+= PAGE_SIZE
;
2949 c_segments_available_new
= c_segments_available
+ C_SEGMENTS_PER_PAGE
;
2951 if (c_segments_available_new
> c_segments_limit
) {
2952 c_segments_available_new
= c_segments_limit
;
2955 for (c_segno
= c_segments_available
+ 1; c_segno
< c_segments_available_new
; c_segno
++) {
2956 c_segments
[c_segno
- 1].c_segno
= c_segno
;
2959 lck_mtx_lock_spin_always(c_list_lock
);
2961 c_segments
[c_segno
- 1].c_segno
= c_free_segno_head
;
2962 c_free_segno_head
= c_segments_available
;
2963 c_segments_available
= c_segments_available_new
;
2965 c_segments_busy
= FALSE
;
2966 thread_wakeup((event_t
) (&c_segments_busy
));
2968 c_segno
= c_free_segno_head
;
2969 assert(c_segno
>= 0 && c_segno
< c_segments_limit
);
2971 c_free_segno_head
= (uint32_t)c_segments
[c_segno
].c_segno
;
2974 * do the rest of the bookkeeping now while we're still behind
2975 * the list lock and grab our generation id now into a local
2976 * so that we can install it once we have the c_seg allocated
2979 if (c_segment_count
> c_segment_count_max
) {
2980 c_segment_count_max
= c_segment_count
;
2983 lck_mtx_unlock_always(c_list_lock
);
2985 c_seg
= (c_segment_t
)zalloc(compressor_segment_zone
);
2986 bzero((char *)c_seg
, sizeof(struct c_segment
));
2988 c_seg
->c_store
.c_buffer
= (int32_t *)C_SEG_BUFFER_ADDRESS(c_segno
);
2990 lck_mtx_init(&c_seg
->c_lock
, &vm_compressor_lck_grp
, &vm_compressor_lck_attr
);
2992 c_seg
->c_state
= C_IS_EMPTY
;
2993 c_seg
->c_firstemptyslot
= C_SLOT_MAX_INDEX
;
2994 c_seg
->c_mysegno
= c_segno
;
2996 lck_mtx_lock_spin_always(c_list_lock
);
2998 c_seg_switch_state(c_seg
, C_IS_FILLING
, FALSE
);
2999 c_segments
[c_segno
].c_seg
= c_seg
;
3000 assert(c_segments
[c_segno
].c_segno
> c_segments_available
);
3001 lck_mtx_unlock_always(c_list_lock
);
3003 *current_chead
= c_seg
;
3005 #if DEVELOPMENT || DEBUG
3006 C_SEG_MAKE_WRITEABLE(c_seg
);
3009 c_seg_alloc_nextslot(c_seg
);
3011 size_to_populate
= C_SEG_ALLOCSIZE
- C_SEG_OFFSET_TO_BYTES(c_seg
->c_populated_offset
);
3013 if (size_to_populate
) {
3014 min_needed
= PAGE_SIZE
+ (C_SEG_ALLOCSIZE
- C_SEG_BUFSIZE
);
3016 if (C_SEG_OFFSET_TO_BYTES(c_seg
->c_populated_offset
- c_seg
->c_nextoffset
) < (unsigned) min_needed
) {
3017 if (size_to_populate
> C_SEG_MAX_POPULATE_SIZE
) {
3018 size_to_populate
= C_SEG_MAX_POPULATE_SIZE
;
3021 OSAddAtomic64(size_to_populate
/ PAGE_SIZE
, &vm_pageout_vminfo
.vm_compressor_pages_grabbed
);
3023 kernel_memory_populate(compressor_map
,
3024 (vm_offset_t
) &c_seg
->c_store
.c_buffer
[c_seg
->c_populated_offset
],
3027 VM_KERN_MEMORY_COMPRESSOR
);
3029 size_to_populate
= 0;
3032 PAGE_REPLACEMENT_DISALLOWED(TRUE
);
3034 lck_mtx_lock_spin_always(&c_seg
->c_lock
);
3036 if (size_to_populate
) {
3037 c_seg
->c_populated_offset
+= C_SEG_BYTES_TO_OFFSET(size_to_populate
);
3043 #if DEVELOPMENT || DEBUG
3045 extern boolean_t memorystatus_freeze_to_memory
;
3046 #endif /* CONFIG_FREEZE */
3047 #endif /* DEVELOPMENT || DEBUG */
3050 c_current_seg_filled(c_segment_t c_seg
, c_segment_t
*current_chead
)
3052 uint32_t unused_bytes
;
3053 uint32_t offset_to_depopulate
;
3054 int new_state
= C_ON_AGE_Q
;
3057 boolean_t head_insert
= FALSE
;
3059 unused_bytes
= trunc_page_32(C_SEG_OFFSET_TO_BYTES(c_seg
->c_populated_offset
- c_seg
->c_nextoffset
));
3062 offset_to_depopulate
= C_SEG_BYTES_TO_OFFSET(round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg
->c_nextoffset
)));
3065 * release the extra physical page(s) at the end of the segment
3067 lck_mtx_unlock_always(&c_seg
->c_lock
);
3069 kernel_memory_depopulate(
3071 (vm_offset_t
) &c_seg
->c_store
.c_buffer
[offset_to_depopulate
],
3075 lck_mtx_lock_spin_always(&c_seg
->c_lock
);
3077 c_seg
->c_populated_offset
= offset_to_depopulate
;
3079 assert(C_SEG_OFFSET_TO_BYTES(c_seg
->c_populated_offset
) <= C_SEG_BUFSIZE
);
3081 #if DEVELOPMENT || DEBUG
3083 boolean_t c_seg_was_busy
= FALSE
;
3085 if (!c_seg
->c_busy
) {
3088 c_seg_was_busy
= TRUE
;
3091 lck_mtx_unlock_always(&c_seg
->c_lock
);
3093 C_SEG_WRITE_PROTECT(c_seg
);
3095 lck_mtx_lock_spin_always(&c_seg
->c_lock
);
3097 if (c_seg_was_busy
== FALSE
) {
3098 C_SEG_WAKEUP_DONE(c_seg
);
3104 if (current_chead
== (c_segment_t
*)&freezer_chead
&&
3105 VM_CONFIG_SWAP_IS_PRESENT
&&
3106 VM_CONFIG_FREEZER_SWAP_IS_ACTIVE
3107 #if DEVELOPMENT || DEBUG
3108 && !memorystatus_freeze_to_memory
3109 #endif /* DEVELOPMENT || DEBUG */
3111 new_state
= C_ON_SWAPOUT_Q
;
3113 #endif /* CONFIG_FREEZE */
3115 if (vm_darkwake_mode
== TRUE
) {
3116 new_state
= C_ON_SWAPOUT_Q
;
3120 clock_get_system_nanotime(&sec
, &nsec
);
3121 c_seg
->c_creation_ts
= (uint32_t)sec
;
3123 lck_mtx_lock_spin_always(c_list_lock
);
3125 c_seg
->c_generation_id
= c_generation_id
++;
3126 c_seg_switch_state(c_seg
, new_state
, head_insert
);
3129 if (c_seg
->c_state
== C_ON_SWAPOUT_Q
) {
3131 * darkwake and freezer can't co-exist together
3132 * We'll need to fix this accounting as a start.
3134 assert(vm_darkwake_mode
== FALSE
);
3135 c_freezer_swapout_page_count
+= (C_SEG_OFFSET_TO_BYTES(c_seg
->c_populated_offset
)) / PAGE_SIZE_64
;
3137 #endif /* CONFIG_FREEZE */
3139 if (c_seg
->c_state
== C_ON_AGE_Q
&& C_SEG_UNUSED_BYTES(c_seg
) >= PAGE_SIZE
) {
3140 c_seg_need_delayed_compaction(c_seg
, TRUE
);
3143 lck_mtx_unlock_always(c_list_lock
);
3145 if (c_seg
->c_state
== C_ON_SWAPOUT_Q
) {
3146 thread_wakeup((event_t
)&c_swapout_list_head
);
3149 *current_chead
= NULL
;
3154 * returns with c_seg locked
3157 c_seg_swapin_requeue(c_segment_t c_seg
, boolean_t has_data
, boolean_t minor_compact_ok
, boolean_t age_on_swapin_q
)
3162 clock_get_system_nanotime(&sec
, &nsec
);
3164 lck_mtx_lock_spin_always(c_list_lock
);
3165 lck_mtx_lock_spin_always(&c_seg
->c_lock
);
3167 assert(c_seg
->c_busy_swapping
);
3168 assert(c_seg
->c_busy
);
3170 c_seg
->c_busy_swapping
= 0;
3172 if (c_seg
->c_overage_swap
== TRUE
) {
3173 c_overage_swapped_count
--;
3174 c_seg
->c_overage_swap
= FALSE
;
3176 if (has_data
== TRUE
) {
3177 if (age_on_swapin_q
== TRUE
) {
3178 c_seg_switch_state(c_seg
, C_ON_SWAPPEDIN_Q
, FALSE
);
3180 c_seg_switch_state(c_seg
, C_ON_AGE_Q
, FALSE
);
3183 if (minor_compact_ok
== TRUE
&& !c_seg
->c_on_minorcompact_q
&& C_SEG_UNUSED_BYTES(c_seg
) >= PAGE_SIZE
) {
3184 c_seg_need_delayed_compaction(c_seg
, TRUE
);
3187 c_seg
->c_store
.c_buffer
= (int32_t*) NULL
;
3188 c_seg
->c_populated_offset
= C_SEG_BYTES_TO_OFFSET(0);
3190 c_seg_switch_state(c_seg
, C_ON_BAD_Q
, FALSE
);
3192 c_seg
->c_swappedin_ts
= (uint32_t)sec
;
3194 lck_mtx_unlock_always(c_list_lock
);
3200 * c_seg has to be locked and is returned locked if the c_seg isn't freed
3201 * PAGE_REPLACMENT_DISALLOWED has to be TRUE on entry and is returned TRUE
3202 * c_seg_swapin returns 1 if the c_seg was freed, 0 otherwise
3206 c_seg_swapin(c_segment_t c_seg
, boolean_t force_minor_compaction
, boolean_t age_on_swapin_q
)
3208 vm_offset_t addr
= 0;
3209 uint32_t io_size
= 0;
3212 assert(C_SEG_IS_ONDISK(c_seg
));
3214 #if !CHECKSUM_THE_SWAP
3215 c_seg_trim_tail(c_seg
);
3217 io_size
= round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg
->c_populated_offset
));
3218 f_offset
= c_seg
->c_store
.c_swap_handle
;
3221 c_seg
->c_busy_swapping
= 1;
3224 * This thread is likely going to block for I/O.
3225 * Make sure it is ready to run when the I/O completes because
3226 * it needs to clear the busy bit on the c_seg so that other
3227 * waiting threads can make progress too. To do that, boost
3228 * the rwlock_count so that the priority is boosted.
3230 set_thread_rwlock_boost();
3231 lck_mtx_unlock_always(&c_seg
->c_lock
);
3233 PAGE_REPLACEMENT_DISALLOWED(FALSE
);
3235 addr
= (vm_offset_t
)C_SEG_BUFFER_ADDRESS(c_seg
->c_mysegno
);
3236 c_seg
->c_store
.c_buffer
= (int32_t*) addr
;
3238 kernel_memory_populate(compressor_map
, addr
, io_size
, KMA_COMPRESSOR
, VM_KERN_MEMORY_COMPRESSOR
);
3240 if (vm_swap_get(c_seg
, f_offset
, io_size
) != KERN_SUCCESS
) {
3241 PAGE_REPLACEMENT_DISALLOWED(TRUE
);
3243 kernel_memory_depopulate(compressor_map
, addr
, io_size
, KMA_COMPRESSOR
);
3245 c_seg_swapin_requeue(c_seg
, FALSE
, TRUE
, age_on_swapin_q
);
3248 vm_swap_decrypt(c_seg
);
3249 #endif /* ENCRYPTED_SWAP */
3251 #if CHECKSUM_THE_SWAP
3252 if (c_seg
->cseg_swap_size
!= io_size
) {
3253 panic("swapin size doesn't match swapout size");
3256 if (c_seg
->cseg_hash
!= vmc_hash((char*) c_seg
->c_store
.c_buffer
, (int)io_size
)) {
3257 panic("c_seg_swapin - Swap hash mismatch\n");
3259 #endif /* CHECKSUM_THE_SWAP */
3261 PAGE_REPLACEMENT_DISALLOWED(TRUE
);
3263 c_seg_swapin_requeue(c_seg
, TRUE
, force_minor_compaction
== TRUE
? FALSE
: TRUE
, age_on_swapin_q
);
3265 OSAddAtomic64(c_seg
->c_bytes_used
, &compressor_bytes_used
);
3267 if (force_minor_compaction
== TRUE
) {
3268 if (c_seg_minor_compaction_and_unlock(c_seg
, FALSE
)) {
3270 * c_seg was completely empty so it was freed,
3271 * so be careful not to reference it again
3273 * Drop the rwlock_count so that the thread priority
3274 * is returned back to where it is supposed to be.
3276 clear_thread_rwlock_boost();
3280 lck_mtx_lock_spin_always(&c_seg
->c_lock
);
3283 C_SEG_WAKEUP_DONE(c_seg
);
3286 * Drop the rwlock_count so that the thread priority
3287 * is returned back to where it is supposed to be.
3289 clear_thread_rwlock_boost();
3296 c_segment_sv_hash_drop_ref(int hash_indx
)
3298 struct c_sv_hash_entry o_sv_he
, n_sv_he
;
3301 o_sv_he
.he_record
= c_segment_sv_hash_table
[hash_indx
].he_record
;
3303 n_sv_he
.he_ref
= o_sv_he
.he_ref
- 1;
3304 n_sv_he
.he_data
= o_sv_he
.he_data
;
3306 if (OSCompareAndSwap64((UInt64
)o_sv_he
.he_record
, (UInt64
)n_sv_he
.he_record
, (UInt64
*) &c_segment_sv_hash_table
[hash_indx
].he_record
) == TRUE
) {
3307 if (n_sv_he
.he_ref
== 0) {
3308 OSAddAtomic(-1, &c_segment_svp_in_hash
);
3317 c_segment_sv_hash_insert(uint32_t data
)
3321 struct c_sv_hash_entry o_sv_he
, n_sv_he
;
3322 boolean_t got_ref
= FALSE
;
3325 OSAddAtomic(1, &c_segment_svp_zero_compressions
);
3327 OSAddAtomic(1, &c_segment_svp_nonzero_compressions
);
3330 hash_sindx
= data
& C_SV_HASH_MASK
;
3332 for (misses
= 0; misses
< C_SV_HASH_MAX_MISS
; misses
++) {
3333 o_sv_he
.he_record
= c_segment_sv_hash_table
[hash_sindx
].he_record
;
3335 while (o_sv_he
.he_data
== data
|| o_sv_he
.he_ref
== 0) {
3336 n_sv_he
.he_ref
= o_sv_he
.he_ref
+ 1;
3337 n_sv_he
.he_data
= data
;
3339 if (OSCompareAndSwap64((UInt64
)o_sv_he
.he_record
, (UInt64
)n_sv_he
.he_record
, (UInt64
*) &c_segment_sv_hash_table
[hash_sindx
].he_record
) == TRUE
) {
3340 if (n_sv_he
.he_ref
== 1) {
3341 OSAddAtomic(1, &c_segment_svp_in_hash
);
3346 o_sv_he
.he_record
= c_segment_sv_hash_table
[hash_sindx
].he_record
;
3348 if (got_ref
== TRUE
) {
3353 if (hash_sindx
== C_SV_HASH_SIZE
) {
3357 if (got_ref
== FALSE
) {
3365 #if RECORD_THE_COMPRESSED_DATA
3368 c_compressed_record_data(char *src
, int c_size
)
3370 if ((c_compressed_record_cptr
+ c_size
+ 4) >= c_compressed_record_ebuf
) {
3371 panic("c_compressed_record_cptr >= c_compressed_record_ebuf");
3374 *(int *)((void *)c_compressed_record_cptr
) = c_size
;
3376 c_compressed_record_cptr
+= 4;
3378 memcpy(c_compressed_record_cptr
, src
, c_size
);
3379 c_compressed_record_cptr
+= c_size
;
3385 c_compress_page(char *src
, c_slot_mapping_t slot_ptr
, c_segment_t
*current_chead
, char *scratch_buf
)
3388 int c_rounded_size
= 0;
3393 KERNEL_DEBUG(0xe0400000 | DBG_FUNC_START
, *current_chead
, 0, 0, 0, 0);
3395 if ((c_seg
= c_seg_allocate(current_chead
)) == NULL
) {
3399 * returns with c_seg lock held
3400 * and PAGE_REPLACEMENT_DISALLOWED(TRUE)...
3401 * c_nextslot has been allocated and
3402 * c_store.c_buffer populated
3404 assert(c_seg
->c_state
== C_IS_FILLING
);
3406 cs
= C_SEG_SLOT_FROM_INDEX(c_seg
, c_seg
->c_nextslot
);
3408 cs
->c_packed_ptr
= C_SLOT_PACK_PTR(slot_ptr
);
3409 assert(slot_ptr
== (c_slot_mapping_t
)C_SLOT_UNPACK_PTR(cs
));
3411 cs
->c_offset
= c_seg
->c_nextoffset
;
3413 max_csize
= C_SEG_BUFSIZE
- C_SEG_OFFSET_TO_BYTES((int32_t)cs
->c_offset
);
3415 if (max_csize
> PAGE_SIZE
) {
3416 max_csize
= PAGE_SIZE
;
3419 #if CHECKSUM_THE_DATA
3420 cs
->c_hash_data
= vmc_hash(src
, PAGE_SIZE
);
3422 boolean_t incomp_copy
= FALSE
;
3423 int max_csize_adj
= (max_csize
- 4);
3425 if (vm_compressor_algorithm() != VM_COMPRESSOR_DEFAULT_CODEC
) {
3426 #if defined(__arm__) || defined(__arm64__)
3427 uint16_t ccodec
= CINVALID
;
3429 if (max_csize
>= C_SEG_OFFSET_ALIGNMENT_BOUNDARY
) {
3430 c_size
= metacompressor((const uint8_t *) src
,
3431 (uint8_t *) &c_seg
->c_store
.c_buffer
[cs
->c_offset
],
3432 max_csize_adj
, &ccodec
,
3433 scratch_buf
, &incomp_copy
);
3434 #if C_SEG_OFFSET_ALIGNMENT_BOUNDARY > 4
3435 if (c_size
> max_csize_adj
) {
3442 assert(ccodec
== CCWK
|| ccodec
== CCLZ4
);
3443 cs
->c_codec
= ccodec
;
3446 #if defined(__arm__) || defined(__arm64__)
3449 #if defined(__arm64__)
3450 __unreachable_ok_push
3451 if (PAGE_SIZE
== 4096) {
3452 c_size
= WKdm_compress_4k((WK_word
*)(uintptr_t)src
, (WK_word
*)(uintptr_t)&c_seg
->c_store
.c_buffer
[cs
->c_offset
],
3453 (WK_word
*)(uintptr_t)scratch_buf
, max_csize_adj
);
3455 c_size
= WKdm_compress_16k((WK_word
*)(uintptr_t)src
, (WK_word
*)(uintptr_t)&c_seg
->c_store
.c_buffer
[cs
->c_offset
],
3456 (WK_word
*)(uintptr_t)scratch_buf
, max_csize_adj
);
3458 __unreachable_ok_pop
3460 c_size
= WKdm_compress_new((const WK_word
*)(uintptr_t)src
, (WK_word
*)(uintptr_t)&c_seg
->c_store
.c_buffer
[cs
->c_offset
],
3461 (WK_word
*)(uintptr_t)scratch_buf
, max_csize_adj
);
3464 assertf(((c_size
<= max_csize_adj
) && (c_size
>= -1)),
3465 "c_size invalid (%d, %d), cur compressions: %d", c_size
, max_csize_adj
, c_segment_pages_compressed
);
3468 if (max_csize
< PAGE_SIZE
) {
3469 c_current_seg_filled(c_seg
, current_chead
);
3470 assert(*current_chead
== NULL
);
3472 lck_mtx_unlock_always(&c_seg
->c_lock
);
3473 /* TODO: it may be worth requiring codecs to distinguish
3474 * between incompressible inputs and failures due to
3475 * budget exhaustion.
3477 PAGE_REPLACEMENT_DISALLOWED(FALSE
);
3482 if (incomp_copy
== FALSE
) {
3483 memcpy(&c_seg
->c_store
.c_buffer
[cs
->c_offset
], src
, c_size
);
3486 OSAddAtomic(1, &c_segment_noncompressible_pages
);
3487 } else if (c_size
== 0) {
3491 * special case - this is a page completely full of a single 32 bit value
3493 hash_index
= c_segment_sv_hash_insert(*(uint32_t *)(uintptr_t)src
);
3495 if (hash_index
!= -1) {
3496 slot_ptr
->s_cindx
= hash_index
;
3497 slot_ptr
->s_cseg
= C_SV_CSEG_ID
;
3499 OSAddAtomic(1, &c_segment_svp_hash_succeeded
);
3500 #if RECORD_THE_COMPRESSED_DATA
3501 c_compressed_record_data(src
, 4);
3503 goto sv_compression
;
3507 memcpy(&c_seg
->c_store
.c_buffer
[cs
->c_offset
], src
, c_size
);
3509 OSAddAtomic(1, &c_segment_svp_hash_failed
);
3512 #if RECORD_THE_COMPRESSED_DATA
3513 c_compressed_record_data((char *)&c_seg
->c_store
.c_buffer
[cs
->c_offset
], c_size
);
3515 #if CHECKSUM_THE_COMPRESSED_DATA
3516 cs
->c_hash_compressed_data
= vmc_hash((char *)&c_seg
->c_store
.c_buffer
[cs
->c_offset
], c_size
);
3518 #if POPCOUNT_THE_COMPRESSED_DATA
3519 cs
->c_pop_cdata
= vmc_pop((uintptr_t) &c_seg
->c_store
.c_buffer
[cs
->c_offset
], c_size
);
3521 c_rounded_size
= (c_size
+ C_SEG_OFFSET_ALIGNMENT_MASK
) & ~C_SEG_OFFSET_ALIGNMENT_MASK
;
3523 PACK_C_SIZE(cs
, c_size
);
3524 c_seg
->c_bytes_used
+= c_rounded_size
;
3525 c_seg
->c_nextoffset
+= C_SEG_BYTES_TO_OFFSET(c_rounded_size
);
3526 c_seg
->c_slots_used
++;
3528 slot_ptr
->s_cindx
= c_seg
->c_nextslot
++;
3529 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
3530 slot_ptr
->s_cseg
= c_seg
->c_mysegno
+ 1;
3533 if (c_seg
->c_nextoffset
>= C_SEG_OFF_LIMIT
|| c_seg
->c_nextslot
>= C_SLOT_MAX_INDEX
) {
3534 c_current_seg_filled(c_seg
, current_chead
);
3535 assert(*current_chead
== NULL
);
3537 lck_mtx_unlock_always(&c_seg
->c_lock
);
3539 PAGE_REPLACEMENT_DISALLOWED(FALSE
);
3541 #if RECORD_THE_COMPRESSED_DATA
3542 if ((c_compressed_record_cptr
- c_compressed_record_sbuf
) >= C_SEG_ALLOCSIZE
) {
3543 c_compressed_record_write(c_compressed_record_sbuf
, (int)(c_compressed_record_cptr
- c_compressed_record_sbuf
));
3544 c_compressed_record_cptr
= c_compressed_record_sbuf
;
3548 OSAddAtomic64(c_size
, &c_segment_compressed_bytes
);
3549 OSAddAtomic64(c_rounded_size
, &compressor_bytes_used
);
3551 OSAddAtomic64(PAGE_SIZE
, &c_segment_input_bytes
);
3553 OSAddAtomic(1, &c_segment_pages_compressed
);
3554 OSAddAtomic(1, &sample_period_compression_count
);
3556 KERNEL_DEBUG(0xe0400000 | DBG_FUNC_END
, *current_chead
, c_size
, c_segment_input_bytes
, c_segment_compressed_bytes
, 0);
3562 sv_decompress(int32_t *ddst
, int32_t pattern
)
3564 // assert(__builtin_constant_p(PAGE_SIZE) != 0);
3565 #if defined(__x86_64__)
3566 memset_word(ddst
, pattern
, PAGE_SIZE
/ sizeof(int32_t));
3567 #elif defined(__arm64__)
3568 assert((PAGE_SIZE
% 128) == 0);
3570 fill32_dczva((addr64_t
)ddst
, PAGE_SIZE
);
3572 fill32_nt((addr64_t
)ddst
, PAGE_SIZE
, pattern
);
3577 /* Unroll the pattern fill loop 4x to encourage the
3578 * compiler to emit NEON stores, cf.
3579 * <rdar://problem/25839866> Loop autovectorization
3582 /* * We use separate loops for each PAGE_SIZE
3583 * to allow the autovectorizer to engage, as PAGE_SIZE
3584 * may not be a constant.
3587 __unreachable_ok_push
3588 if (PAGE_SIZE
== 4096) {
3589 for (i
= 0; i
< (4096U / sizeof(int32_t)); i
+= 4) {
3596 assert(PAGE_SIZE
== 16384);
3597 for (i
= 0; i
< (int)(16384U / sizeof(int32_t)); i
+= 4) {
3604 __unreachable_ok_pop
3609 c_decompress_page(char *dst
, volatile c_slot_mapping_t slot_ptr
, int flags
, int *zeroslot
)
3618 boolean_t need_unlock
= TRUE
;
3619 boolean_t consider_defragmenting
= FALSE
;
3620 boolean_t kdp_mode
= FALSE
;
3622 if (__improbable(flags
& C_KDP
)) {
3624 panic("C_KDP passed to decompress page from outside of debugger context");
3627 assert((flags
& C_KEEP
) == C_KEEP
);
3628 assert((flags
& C_DONT_BLOCK
) == C_DONT_BLOCK
);
3630 if ((flags
& (C_DONT_BLOCK
| C_KEEP
)) != (C_DONT_BLOCK
| C_KEEP
)) {
3639 if (__probable(!kdp_mode
)) {
3640 PAGE_REPLACEMENT_DISALLOWED(TRUE
);
3642 if (kdp_lck_rw_lock_is_acquired_exclusive(&c_master_lock
)) {
3649 * if hibernation is enabled, it indicates (via a call
3650 * to 'vm_decompressor_lock' that no further
3651 * decompressions are allowed once it reaches
3652 * the point of flushing all of the currently dirty
3653 * anonymous memory through the compressor and out
3654 * to disk... in this state we allow freeing of compressed
3655 * pages and must honor the C_DONT_BLOCK case
3657 if (__improbable(dst
&& decompressions_blocked
== TRUE
)) {
3658 if (flags
& C_DONT_BLOCK
) {
3659 if (__probable(!kdp_mode
)) {
3660 PAGE_REPLACEMENT_DISALLOWED(FALSE
);
3667 * it's safe to atomically assert and block behind the
3668 * lock held in shared mode because "decompressions_blocked" is
3669 * only set and cleared and the thread_wakeup done when the lock
3670 * is held exclusively
3672 assert_wait((event_t
)&decompressions_blocked
, THREAD_UNINT
);
3674 PAGE_REPLACEMENT_DISALLOWED(FALSE
);
3676 thread_block(THREAD_CONTINUE_NULL
);
3681 /* s_cseg is actually "segno+1" */
3682 c_segno
= slot_ptr
->s_cseg
- 1;
3684 if (__improbable(c_segno
>= c_segments_available
)) {
3685 panic("c_decompress_page: c_segno %d >= c_segments_available %d, slot_ptr(%p), slot_data(%x)",
3686 c_segno
, c_segments_available
, slot_ptr
, *(int *)((void *)slot_ptr
));
3689 if (__improbable(c_segments
[c_segno
].c_segno
< c_segments_available
)) {
3690 panic("c_decompress_page: c_segno %d is free, slot_ptr(%p), slot_data(%x)",
3691 c_segno
, slot_ptr
, *(int *)((void *)slot_ptr
));
3694 c_seg
= c_segments
[c_segno
].c_seg
;
3696 if (__probable(!kdp_mode
)) {
3697 lck_mtx_lock_spin_always(&c_seg
->c_lock
);
3699 if (kdp_lck_mtx_lock_spin_is_acquired(&c_seg
->c_lock
)) {
3704 assert(c_seg
->c_state
!= C_IS_EMPTY
&& c_seg
->c_state
!= C_IS_FREE
);
3706 if (dst
== NULL
&& c_seg
->c_busy_swapping
) {
3707 assert(c_seg
->c_busy
);
3709 goto bypass_busy_check
;
3711 if (flags
& C_DONT_BLOCK
) {
3712 if (c_seg
->c_busy
|| (C_SEG_IS_ONDISK(c_seg
) && dst
)) {
3719 if (c_seg
->c_busy
) {
3720 PAGE_REPLACEMENT_DISALLOWED(FALSE
);
3722 c_seg_wait_on_busy(c_seg
);
3728 c_indx
= slot_ptr
->s_cindx
;
3730 if (__improbable(c_indx
>= c_seg
->c_nextslot
)) {
3731 panic("c_decompress_page: c_indx %d >= c_nextslot %d, c_seg(%p), slot_ptr(%p), slot_data(%x)",
3732 c_indx
, c_seg
->c_nextslot
, c_seg
, slot_ptr
, *(int *)((void *)slot_ptr
));
3735 cs
= C_SEG_SLOT_FROM_INDEX(c_seg
, c_indx
);
3737 c_size
= UNPACK_C_SIZE(cs
);
3739 if (__improbable(c_size
== 0)) {
3740 panic("c_decompress_page: c_size == 0, c_seg(%p), slot_ptr(%p), slot_data(%x)",
3741 c_seg
, slot_ptr
, *(int *)((void *)slot_ptr
));
3744 c_rounded_size
= (c_size
+ C_SEG_OFFSET_ALIGNMENT_MASK
) & ~C_SEG_OFFSET_ALIGNMENT_MASK
;
3747 uint32_t age_of_cseg
;
3748 clock_sec_t cur_ts_sec
;
3749 clock_nsec_t cur_ts_nsec
;
3751 if (C_SEG_IS_ONDISK(c_seg
)) {
3752 assert(kdp_mode
== FALSE
);
3753 retval
= c_seg_swapin(c_seg
, FALSE
, TRUE
);
3754 assert(retval
== 0);
3758 if (c_seg
->c_state
== C_ON_BAD_Q
) {
3759 assert(c_seg
->c_store
.c_buffer
== NULL
);
3766 #if POPCOUNT_THE_COMPRESSED_DATA
3768 uintptr_t csvaddr
= (uintptr_t) &c_seg
->c_store
.c_buffer
[cs
->c_offset
];
3769 if (cs
->c_pop_cdata
!= (csvpop
= vmc_pop(csvaddr
, c_size
))) {
3770 panic("Compressed data popcount doesn't match original, bit distance: %d %p (phys: %p) %p %p 0x%x 0x%x 0x%x 0x%x", (csvpop
- cs
->c_pop_cdata
), (void *)csvaddr
, (void *) kvtophys(csvaddr
), c_seg
, cs
, cs
->c_offset
, c_size
, csvpop
, cs
->c_pop_cdata
);
3774 #if CHECKSUM_THE_COMPRESSED_DATA
3776 if (cs
->c_hash_compressed_data
!= (csvhash
= vmc_hash((char *)&c_seg
->c_store
.c_buffer
[cs
->c_offset
], c_size
))) {
3777 panic("Compressed data doesn't match original %p %p %u %u %u", c_seg
, cs
, c_size
, cs
->c_hash_compressed_data
, csvhash
);
3780 if (c_rounded_size
== PAGE_SIZE
) {
3782 * page wasn't compressible... just copy it out
3784 memcpy(dst
, &c_seg
->c_store
.c_buffer
[cs
->c_offset
], PAGE_SIZE
);
3785 } else if (c_size
== 4) {
3790 * page was populated with a single value
3791 * that didn't fit into our fast hash
3792 * so we packed it in as a single non-compressed value
3793 * that we need to populate the page with
3795 dptr
= (int32_t *)(uintptr_t)dst
;
3796 data
= *(int32_t *)(&c_seg
->c_store
.c_buffer
[cs
->c_offset
]);
3797 sv_decompress(dptr
, data
);
3802 if (__probable(!kdp_mode
)) {
3804 * we're behind the c_seg lock held in spin mode
3805 * which means pre-emption is disabled... therefore
3806 * the following sequence is atomic and safe
3808 my_cpu_no
= cpu_number();
3810 assert(my_cpu_no
< compressor_cpus
);
3812 scratch_buf
= &compressor_scratch_bufs
[my_cpu_no
* vm_compressor_get_decode_scratch_size()];
3814 scratch_buf
= kdp_compressor_scratch_buf
;
3817 if (vm_compressor_algorithm() != VM_COMPRESSOR_DEFAULT_CODEC
) {
3818 #if defined(__arm__) || defined(__arm64__)
3819 uint16_t c_codec
= cs
->c_codec
;
3820 metadecompressor((const uint8_t *) &c_seg
->c_store
.c_buffer
[cs
->c_offset
],
3821 (uint8_t *)dst
, c_size
, c_codec
, (void *)scratch_buf
);
3824 #if defined(__arm64__)
3825 __unreachable_ok_push
3826 if (PAGE_SIZE
== 4096) {
3827 WKdm_decompress_4k((WK_word
*)(uintptr_t)&c_seg
->c_store
.c_buffer
[cs
->c_offset
],
3828 (WK_word
*)(uintptr_t)dst
, (WK_word
*)(uintptr_t)scratch_buf
, c_size
);
3830 WKdm_decompress_16k((WK_word
*)(uintptr_t)&c_seg
->c_store
.c_buffer
[cs
->c_offset
],
3831 (WK_word
*)(uintptr_t)dst
, (WK_word
*)(uintptr_t)scratch_buf
, c_size
);
3833 __unreachable_ok_pop
3835 WKdm_decompress_new((WK_word
*)(uintptr_t)&c_seg
->c_store
.c_buffer
[cs
->c_offset
],
3836 (WK_word
*)(uintptr_t)dst
, (WK_word
*)(uintptr_t)scratch_buf
, c_size
);
3841 #if CHECKSUM_THE_DATA
3842 if (cs
->c_hash_data
!= vmc_hash(dst
, PAGE_SIZE
)) {
3843 #if defined(__arm__) || defined(__arm64__)
3844 int32_t *dinput
= &c_seg
->c_store
.c_buffer
[cs
->c_offset
];
3845 panic("decompressed data doesn't match original cs: %p, hash: 0x%x, offset: %d, c_size: %d, c_rounded_size: %d, codec: %d, header: 0x%x 0x%x 0x%x", cs
, cs
->c_hash_data
, cs
->c_offset
, c_size
, c_rounded_size
, cs
->c_codec
, *dinput
, *(dinput
+ 1), *(dinput
+ 2));
3847 panic("decompressed data doesn't match original cs: %p, hash: %d, offset: 0x%x, c_size: %d", cs
, cs
->c_hash_data
, cs
->c_offset
, c_size
);
3851 if (c_seg
->c_swappedin_ts
== 0 && !kdp_mode
) {
3852 clock_get_system_nanotime(&cur_ts_sec
, &cur_ts_nsec
);
3854 age_of_cseg
= (uint32_t)cur_ts_sec
- c_seg
->c_creation_ts
;
3855 if (age_of_cseg
< DECOMPRESSION_SAMPLE_MAX_AGE
) {
3856 OSAddAtomic(1, &age_of_decompressions_during_sample_period
[age_of_cseg
]);
3858 OSAddAtomic(1, &overage_decompressions_during_sample_period
);
3861 OSAddAtomic(1, &sample_period_decompression_count
);
3864 if (flags
& C_KEEP
) {
3868 assert(kdp_mode
== FALSE
);
3870 c_seg
->c_bytes_unused
+= c_rounded_size
;
3871 c_seg
->c_bytes_used
-= c_rounded_size
;
3873 assert(c_seg
->c_slots_used
);
3874 c_seg
->c_slots_used
--;
3878 if (c_indx
< c_seg
->c_firstemptyslot
) {
3879 c_seg
->c_firstemptyslot
= c_indx
;
3882 OSAddAtomic(-1, &c_segment_pages_compressed
);
3884 if (c_seg
->c_state
!= C_ON_BAD_Q
&& !(C_SEG_IS_ONDISK(c_seg
))) {
3886 * C_SEG_IS_ONDISK == TRUE can occur when we're doing a
3887 * free of a compressed page (i.e. dst == NULL)
3889 OSAddAtomic64(-c_rounded_size
, &compressor_bytes_used
);
3891 if (c_seg
->c_busy_swapping
) {
3893 * bypass case for c_busy_swapping...
3894 * let the swapin/swapout paths deal with putting
3895 * the c_seg on the minor compaction queue if needed
3897 assert(c_seg
->c_busy
);
3900 assert(!c_seg
->c_busy
);
3902 if (c_seg
->c_state
!= C_IS_FILLING
) {
3903 if (c_seg
->c_bytes_used
== 0) {
3904 if (!(C_SEG_IS_ONDISK(c_seg
))) {
3905 int pages_populated
;
3907 pages_populated
= (round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg
->c_populated_offset
))) / PAGE_SIZE
;
3908 c_seg
->c_populated_offset
= C_SEG_BYTES_TO_OFFSET(0);
3910 if (pages_populated
) {
3911 assert(c_seg
->c_state
!= C_ON_BAD_Q
);
3912 assert(c_seg
->c_store
.c_buffer
!= NULL
);
3915 lck_mtx_unlock_always(&c_seg
->c_lock
);
3917 kernel_memory_depopulate(compressor_map
, (vm_offset_t
) c_seg
->c_store
.c_buffer
, pages_populated
* PAGE_SIZE
, KMA_COMPRESSOR
);
3919 lck_mtx_lock_spin_always(&c_seg
->c_lock
);
3920 C_SEG_WAKEUP_DONE(c_seg
);
3922 if (!c_seg
->c_on_minorcompact_q
&& c_seg
->c_state
!= C_ON_SWAPOUT_Q
&& c_seg
->c_state
!= C_ON_SWAPIO_Q
) {
3923 c_seg_need_delayed_compaction(c_seg
, FALSE
);
3926 if (c_seg
->c_state
!= C_ON_SWAPPEDOUTSPARSE_Q
) {
3927 c_seg_move_to_sparse_list(c_seg
);
3928 consider_defragmenting
= TRUE
;
3931 } else if (c_seg
->c_on_minorcompact_q
) {
3932 assert(c_seg
->c_state
!= C_ON_BAD_Q
);
3933 assert(!C_SEG_IS_ON_DISK_OR_SOQ(c_seg
));
3935 if (C_SEG_SHOULD_MINORCOMPACT_NOW(c_seg
)) {
3936 c_seg_try_minor_compaction_and_unlock(c_seg
);
3937 need_unlock
= FALSE
;
3939 } else if (!(C_SEG_IS_ONDISK(c_seg
))) {
3940 if (c_seg
->c_state
!= C_ON_BAD_Q
&& c_seg
->c_state
!= C_ON_SWAPOUT_Q
&& c_seg
->c_state
!= C_ON_SWAPIO_Q
&&
3941 C_SEG_UNUSED_BYTES(c_seg
) >= PAGE_SIZE
) {
3942 c_seg_need_delayed_compaction(c_seg
, FALSE
);
3944 } else if (c_seg
->c_state
!= C_ON_SWAPPEDOUTSPARSE_Q
&& C_SEG_ONDISK_IS_SPARSE(c_seg
)) {
3945 c_seg_move_to_sparse_list(c_seg
);
3946 consider_defragmenting
= TRUE
;
3950 if (__improbable(kdp_mode
)) {
3954 if (need_unlock
== TRUE
) {
3955 lck_mtx_unlock_always(&c_seg
->c_lock
);
3958 PAGE_REPLACEMENT_DISALLOWED(FALSE
);
3960 if (consider_defragmenting
== TRUE
) {
3961 vm_swap_consider_defragmenting(VM_SWAP_FLAGS_NONE
);
3965 if ((c_minor_count
&& COMPRESSOR_NEEDS_TO_MINOR_COMPACT()) || vm_compressor_needs_to_major_compact()) {
3966 vm_wake_compactor_swapper();
3975 vm_compressor_get(ppnum_t pn
, int *slot
, int flags
)
3977 c_slot_mapping_t slot_ptr
;
3982 dst
= pmap_map_compressor_page(pn
);
3983 slot_ptr
= (c_slot_mapping_t
)slot
;
3985 assert(dst
!= NULL
);
3987 if (slot_ptr
->s_cseg
== C_SV_CSEG_ID
) {
3992 * page was populated with a single value
3993 * that found a home in our hash table
3994 * grab that value from the hash and populate the page
3995 * that we need to populate the page with
3997 dptr
= (int32_t *)(uintptr_t)dst
;
3998 data
= c_segment_sv_hash_table
[slot_ptr
->s_cindx
].he_data
;
3999 sv_decompress(dptr
, data
);
4000 if (!(flags
& C_KEEP
)) {
4001 c_segment_sv_hash_drop_ref(slot_ptr
->s_cindx
);
4003 OSAddAtomic(-1, &c_segment_pages_compressed
);
4007 OSAddAtomic(1, &c_segment_svp_nonzero_decompressions
);
4009 OSAddAtomic(1, &c_segment_svp_zero_decompressions
);
4012 pmap_unmap_compressor_page(pn
, dst
);
4016 retval
= c_decompress_page(dst
, slot_ptr
, flags
, &zeroslot
);
4019 * zeroslot will be set to 0 by c_decompress_page if (flags & C_KEEP)
4020 * or (flags & C_DONT_BLOCK) and we found 'c_busy' or 'C_SEG_IS_ONDISK' to be TRUE
4026 pmap_unmap_compressor_page(pn
, dst
);
4029 * returns 0 if we successfully decompressed a page from a segment already in memory
4030 * returns 1 if we had to first swap in the segment, before successfully decompressing the page
4031 * returns -1 if we encountered an error swapping in the segment - decompression failed
4032 * returns -2 if (flags & C_DONT_BLOCK) and we found 'c_busy' or 'C_SEG_IS_ONDISK' to be true
4039 vm_compressor_free(int *slot
, int flags
)
4041 c_slot_mapping_t slot_ptr
;
4045 assert(flags
== 0 || flags
== C_DONT_BLOCK
);
4047 slot_ptr
= (c_slot_mapping_t
)slot
;
4049 if (slot_ptr
->s_cseg
== C_SV_CSEG_ID
) {
4050 c_segment_sv_hash_drop_ref(slot_ptr
->s_cindx
);
4051 OSAddAtomic(-1, &c_segment_pages_compressed
);
4056 retval
= c_decompress_page(NULL
, slot_ptr
, flags
, &zeroslot
);
4058 * returns 0 if we successfully freed the specified compressed page
4059 * returns -2 if (flags & C_DONT_BLOCK) and we found 'c_busy' set
4065 assert(retval
== -2);
4073 vm_compressor_put(ppnum_t pn
, int *slot
, void **current_chead
, char *scratch_buf
)
4078 src
= pmap_map_compressor_page(pn
);
4079 assert(src
!= NULL
);
4081 retval
= c_compress_page(src
, (c_slot_mapping_t
)slot
, (c_segment_t
*)current_chead
, scratch_buf
);
4082 pmap_unmap_compressor_page(pn
, src
);
4088 vm_compressor_transfer(
4092 c_slot_mapping_t dst_slot
, src_slot
;
4097 src_slot
= (c_slot_mapping_t
) src_slot_p
;
4099 if (src_slot
->s_cseg
== C_SV_CSEG_ID
) {
4100 *dst_slot_p
= *src_slot_p
;
4104 dst_slot
= (c_slot_mapping_t
) dst_slot_p
;
4106 PAGE_REPLACEMENT_DISALLOWED(TRUE
);
4107 /* get segment for src_slot */
4108 c_seg
= c_segments
[src_slot
->s_cseg
- 1].c_seg
;
4110 lck_mtx_lock_spin_always(&c_seg
->c_lock
);
4111 /* wait if it's busy */
4112 if (c_seg
->c_busy
&& !c_seg
->c_busy_swapping
) {
4113 PAGE_REPLACEMENT_DISALLOWED(FALSE
);
4114 c_seg_wait_on_busy(c_seg
);
4117 /* find the c_slot */
4118 c_indx
= src_slot
->s_cindx
;
4119 cs
= C_SEG_SLOT_FROM_INDEX(c_seg
, c_indx
);
4120 /* point the c_slot back to dst_slot instead of src_slot */
4121 cs
->c_packed_ptr
= C_SLOT_PACK_PTR(dst_slot
);
4123 *dst_slot_p
= *src_slot_p
;
4125 lck_mtx_unlock_always(&c_seg
->c_lock
);
4126 PAGE_REPLACEMENT_DISALLOWED(FALSE
);
4131 int freezer_finished_filling
= 0;
4134 vm_compressor_finished_filling(
4135 void **current_chead
)
4139 if ((c_seg
= *(c_segment_t
*)current_chead
) == NULL
) {
4143 assert(c_seg
->c_state
== C_IS_FILLING
);
4145 lck_mtx_lock_spin_always(&c_seg
->c_lock
);
4147 c_current_seg_filled(c_seg
, (c_segment_t
*)current_chead
);
4149 lck_mtx_unlock_always(&c_seg
->c_lock
);
4151 freezer_finished_filling
++;
4156 * This routine is used to transfer the compressed chunks from
4157 * the c_seg/cindx pointed to by slot_p into a new c_seg headed
4158 * by the current_chead and a new cindx within that c_seg.
4160 * Currently, this routine is only used by the "freezer backed by
4161 * compressor with swap" mode to create a series of c_segs that
4162 * only contain compressed data belonging to one task. So, we
4163 * move a task's previously compressed data into a set of new
4164 * c_segs which will also hold the task's yet to be compressed data.
4168 vm_compressor_relocate(
4169 void **current_chead
,
4172 c_slot_mapping_t slot_ptr
;
4173 c_slot_mapping_t src_slot
;
4174 uint32_t c_rounded_size
;
4180 c_segment_t c_seg_dst
= NULL
;
4181 c_segment_t c_seg_src
= NULL
;
4182 kern_return_t kr
= KERN_SUCCESS
;
4185 src_slot
= (c_slot_mapping_t
) slot_p
;
4187 if (src_slot
->s_cseg
== C_SV_CSEG_ID
) {
4189 * no need to relocate... this is a page full of a single
4190 * value which is hashed to a single entry not contained
4197 c_seg_dst
= c_seg_allocate((c_segment_t
*)current_chead
);
4199 * returns with c_seg lock held
4200 * and PAGE_REPLACEMENT_DISALLOWED(TRUE)...
4201 * c_nextslot has been allocated and
4202 * c_store.c_buffer populated
4204 if (c_seg_dst
== NULL
) {
4206 * Out of compression segments?
4208 kr
= KERN_RESOURCE_SHORTAGE
;
4212 assert(c_seg_dst
->c_busy
== 0);
4214 C_SEG_BUSY(c_seg_dst
);
4216 dst_slot
= c_seg_dst
->c_nextslot
;
4218 lck_mtx_unlock_always(&c_seg_dst
->c_lock
);
4221 c_seg_src
= c_segments
[src_slot
->s_cseg
- 1].c_seg
;
4223 assert(c_seg_dst
!= c_seg_src
);
4225 lck_mtx_lock_spin_always(&c_seg_src
->c_lock
);
4227 if (C_SEG_IS_ONDISK(c_seg_src
)) {
4229 * A "thaw" can mark a process as eligible for
4230 * another freeze cycle without bringing any of
4231 * its swapped out c_segs back from disk (because
4232 * that is done on-demand).
4234 * If the src c_seg we find for our pre-compressed
4235 * data is already on-disk, then we are dealing
4236 * with an app's data that is already packed and
4237 * swapped out. Don't do anything.
4240 PAGE_REPLACEMENT_DISALLOWED(FALSE
);
4242 lck_mtx_unlock_always(&c_seg_src
->c_lock
);
4249 if (c_seg_src
->c_busy
) {
4250 PAGE_REPLACEMENT_DISALLOWED(FALSE
);
4251 c_seg_wait_on_busy(c_seg_src
);
4255 PAGE_REPLACEMENT_DISALLOWED(TRUE
);
4260 C_SEG_BUSY(c_seg_src
);
4262 lck_mtx_unlock_always(&c_seg_src
->c_lock
);
4264 PAGE_REPLACEMENT_DISALLOWED(FALSE
);
4266 /* find the c_slot */
4267 c_indx
= src_slot
->s_cindx
;
4269 c_src
= C_SEG_SLOT_FROM_INDEX(c_seg_src
, c_indx
);
4271 c_size
= UNPACK_C_SIZE(c_src
);
4275 if (c_size
> (uint32_t)(C_SEG_BUFSIZE
- C_SEG_OFFSET_TO_BYTES((int32_t)c_seg_dst
->c_nextoffset
))) {
4277 * This segment is full. We need a new one.
4280 PAGE_REPLACEMENT_DISALLOWED(TRUE
);
4282 lck_mtx_lock_spin_always(&c_seg_src
->c_lock
);
4283 C_SEG_WAKEUP_DONE(c_seg_src
);
4284 lck_mtx_unlock_always(&c_seg_src
->c_lock
);
4288 lck_mtx_lock_spin_always(&c_seg_dst
->c_lock
);
4290 assert(c_seg_dst
->c_busy
);
4291 assert(c_seg_dst
->c_state
== C_IS_FILLING
);
4292 assert(!c_seg_dst
->c_on_minorcompact_q
);
4294 c_current_seg_filled(c_seg_dst
, (c_segment_t
*)current_chead
);
4295 assert(*current_chead
== NULL
);
4297 C_SEG_WAKEUP_DONE(c_seg_dst
);
4299 lck_mtx_unlock_always(&c_seg_dst
->c_lock
);
4303 PAGE_REPLACEMENT_DISALLOWED(FALSE
);
4308 c_dst
= C_SEG_SLOT_FROM_INDEX(c_seg_dst
, c_seg_dst
->c_nextslot
);
4310 memcpy(&c_seg_dst
->c_store
.c_buffer
[c_seg_dst
->c_nextoffset
], &c_seg_src
->c_store
.c_buffer
[c_src
->c_offset
], c_size
);
4311 //is platform alignment actually necessary since wkdm aligns its output?
4312 c_rounded_size
= (c_size
+ C_SEG_OFFSET_ALIGNMENT_MASK
) & ~C_SEG_OFFSET_ALIGNMENT_MASK
;
4314 cslot_copy(c_dst
, c_src
);
4315 c_dst
->c_offset
= c_seg_dst
->c_nextoffset
;
4317 if (c_seg_dst
->c_firstemptyslot
== c_seg_dst
->c_nextslot
) {
4318 c_seg_dst
->c_firstemptyslot
++;
4321 c_seg_dst
->c_slots_used
++;
4322 c_seg_dst
->c_nextslot
++;
4323 c_seg_dst
->c_bytes_used
+= c_rounded_size
;
4324 c_seg_dst
->c_nextoffset
+= C_SEG_BYTES_TO_OFFSET(c_rounded_size
);
4327 PACK_C_SIZE(c_src
, 0);
4329 c_seg_src
->c_bytes_used
-= c_rounded_size
;
4330 c_seg_src
->c_bytes_unused
+= c_rounded_size
;
4332 assert(c_seg_src
->c_slots_used
);
4333 c_seg_src
->c_slots_used
--;
4335 if (c_indx
< c_seg_src
->c_firstemptyslot
) {
4336 c_seg_src
->c_firstemptyslot
= c_indx
;
4339 c_dst
= C_SEG_SLOT_FROM_INDEX(c_seg_dst
, dst_slot
);
4341 PAGE_REPLACEMENT_ALLOWED(TRUE
);
4342 slot_ptr
= (c_slot_mapping_t
)C_SLOT_UNPACK_PTR(c_dst
);
4343 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
4344 slot_ptr
->s_cseg
= c_seg_dst
->c_mysegno
+ 1;
4345 slot_ptr
->s_cindx
= dst_slot
;
4347 PAGE_REPLACEMENT_ALLOWED(FALSE
);
4351 lck_mtx_lock_spin_always(&c_seg_src
->c_lock
);
4353 C_SEG_WAKEUP_DONE(c_seg_src
);
4355 if (c_seg_src
->c_bytes_used
== 0 && c_seg_src
->c_state
!= C_IS_FILLING
) {
4356 if (!c_seg_src
->c_on_minorcompact_q
) {
4357 c_seg_need_delayed_compaction(c_seg_src
, FALSE
);
4361 lck_mtx_unlock_always(&c_seg_src
->c_lock
);
4365 PAGE_REPLACEMENT_DISALLOWED(TRUE
);
4367 lck_mtx_lock_spin_always(&c_seg_dst
->c_lock
);
4369 if (c_seg_dst
->c_nextoffset
>= C_SEG_OFF_LIMIT
|| c_seg_dst
->c_nextslot
>= C_SLOT_MAX_INDEX
) {
4371 * Nearing or exceeded maximum slot and offset capacity.
4373 assert(c_seg_dst
->c_busy
);
4374 assert(c_seg_dst
->c_state
== C_IS_FILLING
);
4375 assert(!c_seg_dst
->c_on_minorcompact_q
);
4377 c_current_seg_filled(c_seg_dst
, (c_segment_t
*)current_chead
);
4378 assert(*current_chead
== NULL
);
4381 C_SEG_WAKEUP_DONE(c_seg_dst
);
4383 lck_mtx_unlock_always(&c_seg_dst
->c_lock
);
4387 PAGE_REPLACEMENT_DISALLOWED(FALSE
);
4392 #endif /* CONFIG_FREEZE */