]> git.saurik.com Git - apple/xnu.git/blame - osfmk/vm/vm_compressor.c
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-4570.71.2.tar.gz
[apple/xnu.git] / osfmk / vm / vm_compressor.c
CommitLineData
39236c6e
A		 1/*
2 * Copyright (c) 2000-2013 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <vm/vm_compressor.h>
fe8ab488
A		 30
31#if CONFIG_PHANTOM_CACHE
32#include <vm/vm_phantom_cache.h>
33#endif
34
39236c6e
A		 35#include <vm/vm_map.h>
36#include <vm/vm_pageout.h>
37#include <vm/memory_object.h>
39037602
A		 38#include <vm/vm_compressor_algorithms.h>
39#include <vm/vm_fault.h>
5ba3f43e 40#include <vm/vm_protos.h>
39236c6e
A		 41#include <mach/mach_host.h> /* for host_info() */
42#include <kern/ledger.h>
39037602 43#include <kern/policy_internal.h>
5ba3f43e
A		 44#include <kern/thread_group.h>
45#include <san/kasan.h>
39236c6e 46
5ba3f43e 47#if !CONFIG_EMBEDDED
3e170ce0 48#include <i386/misc_protos.h>
5ba3f43e 49#endif
3e170ce0 50
39236c6e
A		 51#include <IOKit/IOHibernatePrivate.h>
52
5ba3f43e
A		 53#if POPCOUNT_THE_COMPRESSED_DATA
54boolean_t popcount_c_segs = TRUE;
55
56static inline uint32_t vmc_pop(uintptr_t ins, int sz) {
57 uint32_t rv = 0;
58
59 if (__probable(popcount_c_segs == FALSE)) {
60 return 0xDEAD707C;
61 }
62
63 while (sz >= 16) {
64 uint32_t rv1, rv2;
65 uint64_t *ins64 = (uint64_t *) ins;
66 uint64_t *ins642 = (uint64_t *) (ins + 8);
67 rv1 = __builtin_popcountll(*ins64);
68 rv2 = __builtin_popcountll(*ins642);
69 rv += rv1 + rv2;
70 sz -= 16;
71 ins += 16;
72 }
73
74 while (sz >= 4) {
75 uint32_t *ins32 = (uint32_t *) ins;
76 rv += __builtin_popcount(*ins32);
77 sz -= 4;
78 ins += 4;
79 }
80
81 while (sz > 0) {
82 char *ins8 = (char *)ins;
83 rv += __builtin_popcount(*ins8);
84 sz--;
85 ins++;
86 }
87 return rv;
88}
89#endif
90
39236c6e
A		 91/*
92 * vm_compressor_mode has a heirarchy of control to set its value.
93 * boot-args are checked first, then device-tree, and finally
94 * the default value that is defined below. See vm_fault_init() for
95 * the boot-arg & device-tree code.
96 */
97
5ba3f43e
A		 98#if CONFIG_EMBEDDED
99
100#if CONFIG_FREEZE
101int vm_compressor_mode = VM_PAGER_FREEZER_DEFAULT;
102
103void *freezer_chead; /* The chead used to track c_segs allocated for the exclusive use of holding just one task's compressed memory.*/
104char *freezer_compressor_scratch_buf = NULL;
105
106#define VM_MAX_FREEZER_CSEG_SWAP_COUNT 64 /* The maximum number of c_segs holding just one task's compressed memory that can be swapped out to disk.*/
107extern int c_freezer_swapout_count; /* This count keeps track of the # of c_segs holding just one task's compressed memory on the swapout queue. This count is used during each freeze i.e. on a per-task basis.*/
108
109#else /* CONFIG_FREEZE */
110int vm_compressor_mode = VM_PAGER_NOT_CONFIGURED;
111#endif /* CONFIG_FREEZE */
112
113int vm_scale = 1;
114
115#else /* CONFIG_EMBEDDED */
39236c6e
A		 116int vm_compressor_mode = VM_PAGER_COMPRESSOR_WITH_SWAP;
117int vm_scale = 16;
118
5ba3f43e 119#endif /* CONFIG_EMBEDDED */
39236c6e 120
04b8595b 121int vm_compressor_is_active = 0;
39236c6e 122int vm_compression_limit = 0;
3e170ce0 123int vm_compressor_available = 0;
39236c6e 124
39236c6e
A		 125extern void vm_pageout_io_throttle(void);
126
127#if CHECKSUM_THE_DATA || CHECKSUM_THE_SWAP || CHECKSUM_THE_COMPRESSED_DATA
128extern unsigned int hash_string(char *cp, int len);
5ba3f43e
A		 129static unsigned int vmc_hash(char *, int);
130boolean_t checksum_c_segs = TRUE;
131
132unsigned int vmc_hash(char *cp, int len) {
133 if (__probable(checksum_c_segs == FALSE)) {
134 return 0xDEAD7A37;
135 }
136 return hash_string(cp, len);
137}
39236c6e
A		 138#endif
139
3e170ce0
A		 140#define UNPACK_C_SIZE(cs) ((cs->c_size == (PAGE_SIZE-1)) ? PAGE_SIZE : cs->c_size)
141#define PACK_C_SIZE(cs, size) (cs->c_size = ((size == PAGE_SIZE) ? PAGE_SIZE - 1 : size))
fe8ab488 142
39236c6e 143
3e170ce0
A		 144struct c_sv_hash_entry {
145 union {
146 struct {
147 uint32_t c_sv_he_ref;
148 uint32_t c_sv_he_data;
149 } c_sv_he;
150 uint64_t c_sv_he_record;
151
152 } c_sv_he_un;
39236c6e
A		 153};
154
3e170ce0
A		 155#define he_ref c_sv_he_un.c_sv_he.c_sv_he_ref
156#define he_data c_sv_he_un.c_sv_he.c_sv_he_data
157#define he_record c_sv_he_un.c_sv_he_record
158
159#define C_SV_HASH_MAX_MISS 32
160#define C_SV_HASH_SIZE ((1 << 10))
161#define C_SV_HASH_MASK ((1 << 10) - 1)
162#define C_SV_CSEG_ID ((1 << 22) - 1)
39236c6e
A		 163
164
39236c6e
A		 165union c_segu {
166 c_segment_t c_seg;
39037602 167 uintptr_t c_segno;
39236c6e
A		 168};
169
170
171
39037602
A		 172#define C_SLOT_PACK_PTR(ptr) (((uintptr_t)ptr - (uintptr_t) KERNEL_PMAP_HEAP_RANGE_START) >> 2)
173#define C_SLOT_UNPACK_PTR(cslot) ((uintptr_t)(cslot->c_packed_ptr << 2) + (uintptr_t) KERNEL_PMAP_HEAP_RANGE_START)
39236c6e
A		 174
175
176uint32_t c_segment_count = 0;
3e170ce0 177uint32_t c_segment_count_max = 0;
39236c6e
A		 178
179uint64_t c_generation_id = 0;
180uint64_t c_generation_id_flush_barrier;
181
182
183#define HIBERNATE_FLUSHING_SECS_TO_COMPLETE 120
184
185boolean_t hibernate_no_swapspace = FALSE;
186clock_sec_t hibernate_flushing_deadline = 0;
187
188
3e170ce0
A		 189#if RECORD_THE_COMPRESSED_DATA
190char *c_compressed_record_sbuf;
191char *c_compressed_record_ebuf;
192char *c_compressed_record_cptr;
39236c6e
A		 193#endif
194
3e170ce0 195
39236c6e
A		 196queue_head_t c_age_list_head;
197queue_head_t c_swapout_list_head;
198queue_head_t c_swappedin_list_head;
199queue_head_t c_swappedout_list_head;
200queue_head_t c_swappedout_sparse_list_head;
3e170ce0
A		 201queue_head_t c_major_list_head;
202queue_head_t c_filling_list_head;
203queue_head_t c_bad_list_head;
39236c6e
A		 204
205uint32_t c_age_count = 0;
206uint32_t c_swapout_count = 0;
207uint32_t c_swappedin_count = 0;
208uint32_t c_swappedout_count = 0;
209uint32_t c_swappedout_sparse_count = 0;
3e170ce0
A		 210uint32_t c_major_count = 0;
211uint32_t c_filling_count = 0;
212uint32_t c_empty_count = 0;
213uint32_t c_bad_count = 0;
214
39236c6e
A		 215
216queue_head_t c_minor_list_head;
217uint32_t c_minor_count = 0;
218
3e170ce0
A		 219int c_overage_swapped_count = 0;
220int c_overage_swapped_limit = 0;
221
222int c_seg_fixed_array_len;
39236c6e 223union c_segu *c_segments;
3e170ce0
A		 224vm_offset_t c_buffers;
225vm_size_t c_buffers_size;
39236c6e
A		 226caddr_t c_segments_next_page;
227boolean_t c_segments_busy;
228uint32_t c_segments_available;
229uint32_t c_segments_limit;
fe8ab488 230uint32_t c_segments_nearing_limit;
3e170ce0
A		 231
232uint32_t c_segment_svp_in_hash;
233uint32_t c_segment_svp_hash_succeeded;
234uint32_t c_segment_svp_hash_failed;
235uint32_t c_segment_svp_zero_compressions;
236uint32_t c_segment_svp_nonzero_compressions;
237uint32_t c_segment_svp_zero_decompressions;
238uint32_t c_segment_svp_nonzero_decompressions;
239
240uint32_t c_segment_noncompressible_pages;
241
39236c6e
A		 242uint32_t c_segment_pages_compressed;
243uint32_t c_segment_pages_compressed_limit;
fe8ab488 244uint32_t c_segment_pages_compressed_nearing_limit;
39236c6e
A		 245uint32_t c_free_segno_head = (uint32_t)-1;
246
247uint32_t vm_compressor_minorcompact_threshold_divisor = 10;
248uint32_t vm_compressor_majorcompact_threshold_divisor = 10;
249uint32_t vm_compressor_unthrottle_threshold_divisor = 10;
250uint32_t vm_compressor_catchup_threshold_divisor = 10;
251
252#define C_SEGMENTS_PER_PAGE (PAGE_SIZE / sizeof(union c_segu))
253
254
255lck_grp_attr_t vm_compressor_lck_grp_attr;
256lck_attr_t vm_compressor_lck_attr;
257lck_grp_t vm_compressor_lck_grp;
39236c6e 258lck_mtx_t *c_list_lock;
39236c6e 259lck_rw_t c_master_lock;
fe8ab488 260boolean_t decompressions_blocked = FALSE;
39236c6e
A		 261
262zone_t compressor_segment_zone;
263int c_compressor_swap_trigger = 0;
264
265uint32_t compressor_cpus;
266char *compressor_scratch_bufs;
3e170ce0
A		 267char *kdp_compressor_scratch_buf;
268char *kdp_compressor_decompressed_page;
269addr64_t kdp_compressor_decompressed_page_paddr;
270ppnum_t kdp_compressor_decompressed_page_ppnum;
39236c6e
A		 271
272clock_sec_t start_of_sample_period_sec = 0;
273clock_nsec_t start_of_sample_period_nsec = 0;
274clock_sec_t start_of_eval_period_sec = 0;
275clock_nsec_t start_of_eval_period_nsec = 0;
276uint32_t sample_period_decompression_count = 0;
277uint32_t sample_period_compression_count = 0;
278uint32_t last_eval_decompression_count = 0;
279uint32_t last_eval_compression_count = 0;
280
281#define DECOMPRESSION_SAMPLE_MAX_AGE (60 * 30)
282
3e170ce0
A		 283boolean_t vm_swapout_ripe_segments = FALSE;
284uint32_t vm_ripe_target_age = (60 * 60 * 48);
285
39236c6e
A		 286uint32_t swapout_target_age = 0;
287uint32_t age_of_decompressions_during_sample_period[DECOMPRESSION_SAMPLE_MAX_AGE];
288uint32_t overage_decompressions_during_sample_period = 0;
5ba3f43e
A		 289uint32_t vm_compressor_pages_grabbed = 0;
290
39236c6e 291
d190cdc3 292void do_fastwake_warmup(queue_head_t *, boolean_t);
39236c6e
A		 293boolean_t fastwake_warmup = FALSE;
294boolean_t fastwake_recording_in_progress = FALSE;
295clock_sec_t dont_trim_until_ts = 0;
296
297uint64_t c_segment_warmup_count;
298uint64_t first_c_segment_to_warm_generation_id = 0;
299uint64_t last_c_segment_to_warm_generation_id = 0;
300boolean_t hibernate_flushing = FALSE;
301
fe8ab488
A		 302int64_t c_segment_input_bytes __attribute__((aligned(8))) = 0;
303int64_t c_segment_compressed_bytes __attribute__((aligned(8))) = 0;
304int64_t compressor_bytes_used __attribute__((aligned(8))) = 0;
3e170ce0
A		 305
306
307struct c_sv_hash_entry c_segment_sv_hash_table[C_SV_HASH_SIZE] __attribute__ ((aligned (8)));
308
39236c6e
A		 309static boolean_t compressor_needs_to_swap(void);
310static void vm_compressor_swap_trigger_thread(void);
311static void vm_compressor_do_delayed_compactions(boolean_t);
312static void vm_compressor_compact_and_swap(boolean_t);
313static void vm_compressor_age_swapped_in_segments(boolean_t);
39236c6e 314
5ba3f43e 315#if !CONFIG_EMBEDDED
3e170ce0 316static void vm_compressor_take_paging_space_action(void);
5ba3f43e 317#endif
39236c6e
A		 318
319void compute_swapout_target_age(void);
320
321boolean_t c_seg_major_compact(c_segment_t, c_segment_t);
322boolean_t c_seg_major_compact_ok(c_segment_t, c_segment_t);
323
324int c_seg_minor_compaction_and_unlock(c_segment_t, boolean_t);
325int c_seg_do_minor_compaction_and_unlock(c_segment_t, boolean_t, boolean_t, boolean_t);
326void c_seg_try_minor_compaction_and_unlock(c_segment_t c_seg);
39236c6e
A		 327
328void c_seg_move_to_sparse_list(c_segment_t);
329void c_seg_insert_into_q(queue_head_t *, c_segment_t);
330
39236c6e 331uint64_t vm_available_memory(void);
fe8ab488 332uint64_t vm_compressor_pages_compressed(void);
39236c6e 333
39037602
A		 334/*
335 * indicate the need to do a major compaction if
336 * the overall set of in-use compression segments
337 * becomes sparse... on systems that support pressure
338 * driven swapping, this will also cause swapouts to
339 * be initiated.
340 */
341static inline boolean_t vm_compressor_needs_to_major_compact()
342{
343 uint32_t incore_seg_count;
344
345 incore_seg_count = c_segment_count - c_swappedout_count - c_swappedout_sparse_count;
346
347 if ((c_segment_count >= (c_segments_nearing_limit / 8)) &&
348 ((incore_seg_count * C_SEG_MAX_PAGES) - VM_PAGE_COMPRESSOR_COUNT) >
349 ((incore_seg_count / 8) * C_SEG_MAX_PAGES))
350 return (1);
351 return (0);
352}
353
39236c6e
A		 354
355uint64_t
356vm_available_memory(void)
357{
358 return (((uint64_t)AVAILABLE_NON_COMPRESSED_MEMORY) * PAGE_SIZE_64);
359}
360
361
fe8ab488
A		 362uint64_t
363vm_compressor_pages_compressed(void)
364{
365 return (c_segment_pages_compressed * PAGE_SIZE_64);
366}
367
368
39236c6e
A		 369boolean_t
370vm_compressor_low_on_space(void)
371{
fe8ab488
A		 372 if ((c_segment_pages_compressed > c_segment_pages_compressed_nearing_limit) ||
373 (c_segment_count > c_segments_nearing_limit))
39236c6e
A		 374 return (TRUE);
375
376 return (FALSE);
377}
5ba3f43e
A		 378
379
380boolean_t
381vm_compressor_out_of_space(void)
382{
383 if ((c_segment_pages_compressed >= c_segment_pages_compressed_limit) ||
384 (c_segment_count >= c_segments_limit))
385 return (TRUE);
386
387 return (FALSE);
388}
39236c6e
A		 389
390
391int
fe8ab488 392vm_wants_task_throttled(task_t task)
39236c6e 393{
fe8ab488
A		 394 if (task == kernel_task)
395 return (0);
396
39037602 397 if (VM_CONFIG_SWAP_IS_ACTIVE) {
fe8ab488
A		 398 if ((vm_compressor_low_on_space() || HARD_THROTTLE_LIMIT_REACHED()) &&
399 (unsigned int)pmap_compressed(task->map->pmap) > (c_segment_pages_compressed / 4))
39236c6e 400 return (1);
39236c6e
A		 401 }
402 return (0);
403}
404
405
813fb2f6
A		 406#if DEVELOPMENT || DEBUG
407boolean_t kill_on_no_paging_space = FALSE; /* On compressor/swap exhaustion, kill the largest process regardless of
408 * its chosen process policy. Controlled by a boot-arg of the same name. */
409#endif /* DEVELOPMENT || DEBUG */
410
5ba3f43e 411#if !CONFIG_EMBEDDED
3e170ce0
A		 412
413static uint32_t no_paging_space_action_in_progress = 0;
414extern void memorystatus_send_low_swap_note(void);
415
416static void
417vm_compressor_take_paging_space_action(void)
418{
419 if (no_paging_space_action_in_progress == 0) {
420
421 if (OSCompareAndSwap(0, 1, (UInt32 *)&no_paging_space_action_in_progress)) {
422
423 if (no_paging_space_action()) {
813fb2f6
A		 424#if DEVELOPMENT || DEBUG
425 if (kill_on_no_paging_space == TRUE) {
426 /*
427 * Since we are choosing to always kill a process, we don't need the
428 * "out of application memory" dialog box in this mode. And, hence we won't
429 * send the knote.
430 */
431 no_paging_space_action_in_progress = 0;
432 return;
433 }
434#endif /* DEVELOPMENT || DEBUG */
3e170ce0
A		 435 memorystatus_send_low_swap_note();
436 }
437
438 no_paging_space_action_in_progress = 0;
439 }
440 }
441}
5ba3f43e 442#endif /* !CONFIG_EMBEDDED */
3e170ce0
A		 443
444
39236c6e
A		 445void
446vm_compressor_init_locks(void)
447{
448 lck_grp_attr_setdefault(&vm_compressor_lck_grp_attr);
449 lck_grp_init(&vm_compressor_lck_grp, "vm_compressor", &vm_compressor_lck_grp_attr);
450 lck_attr_setdefault(&vm_compressor_lck_attr);
451
452 lck_rw_init(&c_master_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
39236c6e
A		 453}
454
455
456void
457vm_decompressor_lock(void)
458{
fe8ab488
A		 459 PAGE_REPLACEMENT_ALLOWED(TRUE);
460
461 decompressions_blocked = TRUE;
462
463 PAGE_REPLACEMENT_ALLOWED(FALSE);
39236c6e
A		 464}
465
466void
467vm_decompressor_unlock(void)
468{
fe8ab488
A		 469 PAGE_REPLACEMENT_ALLOWED(TRUE);
470
471 decompressions_blocked = FALSE;
472
473 PAGE_REPLACEMENT_ALLOWED(FALSE);
39236c6e 474
fe8ab488 475 thread_wakeup((event_t)&decompressions_blocked);
39236c6e
A		 476}
477
39037602
A		 478static inline void cslot_copy(c_slot_t cdst, c_slot_t csrc) {
479#if CHECKSUM_THE_DATA
480 cdst->c_hash_data = csrc->c_hash_data;
481#endif
482#if CHECKSUM_THE_COMPRESSED_DATA
483 cdst->c_hash_compressed_data = csrc->c_hash_compressed_data;
5ba3f43e
A		 484#endif
485#if POPCOUNT_THE_COMPRESSED_DATA
486 cdst->c_pop_cdata = csrc->c_pop_cdata;
39037602
A		 487#endif
488 cdst->c_size = csrc->c_size;
489 cdst->c_packed_ptr = csrc->c_packed_ptr;
5ba3f43e
A		 490#if defined(__arm__) || defined(__arm64__)
491 cdst->c_codec = csrc->c_codec;
492#endif
39037602 493}
39236c6e 494
39037602 495vm_map_t compressor_map;
5ba3f43e
A		 496uint64_t compressor_pool_max_size;
497uint64_t compressor_pool_size;
498uint32_t compressor_pool_multiplier;
499
500#if DEVELOPMENT || DEBUG
501/*
502 * Compressor segments are write-protected in development/debug
503 * kernels to help debug memory corruption.
504 * In cases where performance is a concern, this can be disabled
505 * via the boot-arg "-disable_cseg_write_protection".
506 */
507boolean_t write_protect_c_segs = TRUE;
508int vm_compressor_test_seg_wp;
509uint32_t vm_ktrace_enabled;
510#endif /* DEVELOPMENT || DEBUG */
39236c6e
A		 511
512void
513vm_compressor_init(void)
514{
515 thread_t thread;
fe8ab488
A		 516 struct c_slot cs_dummy;
517 c_slot_t cs = &cs_dummy;
3e170ce0
A		 518 int c_segment_min_size;
519 int c_segment_padded_size;
5ba3f43e 520 int attempts = 1;
39037602
A		 521 kern_return_t retval = KERN_SUCCESS;
522 vm_offset_t start_addr = 0;
523 vm_size_t c_segments_arr_size = 0, compressor_submap_size = 0;
5ba3f43e 524 vm_map_kernel_flags_t vmk_flags;
39037602
A		 525#if RECORD_THE_COMPRESSED_DATA
526 vm_size_t c_compressed_record_sbuf_size = 0;
527#endif /* RECORD_THE_COMPRESSED_DATA */
fe8ab488 528
813fb2f6
A		 529#if DEVELOPMENT || DEBUG
530 char bootarg_name[32];
531 if (PE_parse_boot_argn("-kill_on_no_paging_space", bootarg_name, sizeof (bootarg_name))) {
532 kill_on_no_paging_space = TRUE;
533 }
5ba3f43e
A		 534 if (PE_parse_boot_argn("-disable_cseg_write_protection", bootarg_name, sizeof (bootarg_name))) {
535 write_protect_c_segs = FALSE;
536 }
537 int vmcval = 1;
538 PE_parse_boot_argn("vm_compressor_validation", &vmcval, sizeof(vmcval));
539
540 if (kern_feature_override(KF_COMPRSV_OVRD)) {
541 vmcval = 0;
542 }
543 if (vmcval == 0) {
544#if POPCOUNT_THE_COMPRESSED_DATA
545 popcount_c_segs = FALSE;
546#endif
547#if CHECKSUM_THE_DATA || CHECKSUM_THE_COMPRESSED_DATA
548 checksum_c_segs = FALSE;
549#endif
550 write_protect_c_segs = FALSE;
551 }
813fb2f6
A		 552#endif /* DEVELOPMENT || DEBUG */
553
fe8ab488
A		 554 /*
555 * ensure that any pointer that gets created from
556 * the vm_page zone can be packed properly
557 */
558 cs->c_packed_ptr = C_SLOT_PACK_PTR(zone_map_min_address);
559
560 if (C_SLOT_UNPACK_PTR(cs) != (uintptr_t)zone_map_min_address)
561 panic("C_SLOT_UNPACK_PTR failed on zone_map_min_address - %p", (void *)zone_map_min_address);
562
563 cs->c_packed_ptr = C_SLOT_PACK_PTR(zone_map_max_address);
564
565 if (C_SLOT_UNPACK_PTR(cs) != (uintptr_t)zone_map_max_address)
566 panic("C_SLOT_UNPACK_PTR failed on zone_map_max_address - %p", (void *)zone_map_max_address);
567
39236c6e
A		 568
569 assert((C_SEGMENTS_PER_PAGE * sizeof(union c_segu)) == PAGE_SIZE);
570
571 PE_parse_boot_argn("vm_compression_limit", &vm_compression_limit, sizeof (vm_compression_limit));
572
5ba3f43e
A		 573#ifdef CONFIG_EMBEDDED
574 vm_compressor_minorcompact_threshold_divisor = 20;
575 vm_compressor_majorcompact_threshold_divisor = 30;
576 vm_compressor_unthrottle_threshold_divisor = 40;
577 vm_compressor_catchup_threshold_divisor = 60;
578#else
39236c6e
A		 579 if (max_mem <= (3ULL * 1024ULL * 1024ULL * 1024ULL)) {
580 vm_compressor_minorcompact_threshold_divisor = 11;
581 vm_compressor_majorcompact_threshold_divisor = 13;
582 vm_compressor_unthrottle_threshold_divisor = 20;
583 vm_compressor_catchup_threshold_divisor = 35;
584 } else {
585 vm_compressor_minorcompact_threshold_divisor = 20;
586 vm_compressor_majorcompact_threshold_divisor = 25;
587 vm_compressor_unthrottle_threshold_divisor = 35;
588 vm_compressor_catchup_threshold_divisor = 50;
589 }
5ba3f43e 590#endif
39236c6e
A		 591 /*
592 * vm_page_init_lck_grp is now responsible for calling vm_compressor_init_locks
593 * c_master_lock needs to be available early so that "vm_page_find_contiguous" can
594 * use PAGE_REPLACEMENT_ALLOWED to coordinate with the compressor.
595 */
596
39236c6e 597 c_list_lock = lck_mtx_alloc_init(&vm_compressor_lck_grp, &vm_compressor_lck_attr);
3e170ce0 598
39236c6e 599 queue_init(&c_bad_list_head);
39236c6e
A		 600 queue_init(&c_age_list_head);
601 queue_init(&c_minor_list_head);
3e170ce0
A		 602 queue_init(&c_major_list_head);
603 queue_init(&c_filling_list_head);
39236c6e
A		 604 queue_init(&c_swapout_list_head);
605 queue_init(&c_swappedin_list_head);
606 queue_init(&c_swappedout_list_head);
607 queue_init(&c_swappedout_sparse_list_head);
608
39236c6e
A		 609 c_free_segno_head = -1;
610 c_segments_available = 0;
611
5ba3f43e
A		 612 if (vm_compression_limit)
613 compressor_pool_size = (uint64_t)vm_compression_limit * PAGE_SIZE_64;
39236c6e 614
5ba3f43e
A		 615 compressor_pool_max_size = C_SEG_MAX_LIMIT;
616 compressor_pool_max_size *= C_SEG_BUFSIZE;
39236c6e 617
5ba3f43e
A		 618#if defined(__x86_64__)
619
620 if (vm_compression_limit == 0) {
621
622 if (max_mem <= (4ULL * 1024ULL * 1024ULL * 1024ULL))
623 compressor_pool_size = 16ULL * max_mem;
624 else if (max_mem <= (8ULL * 1024ULL * 1024ULL * 1024ULL))
625 compressor_pool_size = 8ULL * max_mem;
626 else if (max_mem <= (32ULL * 1024ULL * 1024ULL * 1024ULL))
627 compressor_pool_size = 4ULL * max_mem;
39236c6e 628 else
5ba3f43e 629 compressor_pool_size = 2ULL * max_mem;
39236c6e 630 }
5ba3f43e
A		 631 if (max_mem <= (8ULL * 1024ULL * 1024ULL * 1024ULL))
632 compressor_pool_multiplier = 1;
633 else if (max_mem <= (32ULL * 1024ULL * 1024ULL * 1024ULL))
634 compressor_pool_multiplier = 2;
635 else
636 compressor_pool_multiplier = 4;
39236c6e 637
5ba3f43e 638#elif defined(__arm__)
fe8ab488 639
5ba3f43e
A		 640#define VM_RESERVE_SIZE (1024 * 1024 * 256)
641#define MAX_COMPRESSOR_POOL_SIZE (1024 * 1024 * 450)
642
643 if (compressor_pool_max_size > MAX_COMPRESSOR_POOL_SIZE)
644 compressor_pool_max_size = MAX_COMPRESSOR_POOL_SIZE;
645
646 if (vm_compression_limit == 0)
647 compressor_pool_size = ((kernel_map->max_offset - kernel_map->min_offset) - kernel_map->size) - VM_RESERVE_SIZE;
648 compressor_pool_multiplier = 1;
649#else
650 if (compressor_pool_max_size > max_mem)
651 compressor_pool_max_size = max_mem;
652
653 if (vm_compression_limit == 0)
654 compressor_pool_size = max_mem;
655 compressor_pool_multiplier = 1;
656#endif
657 if (compressor_pool_size > compressor_pool_max_size)
658 compressor_pool_size = compressor_pool_max_size;
659
660try_again:
661 c_segments_limit = (uint32_t)(compressor_pool_size / (vm_size_t)(C_SEG_ALLOCSIZE));
662 c_segments_nearing_limit = (uint32_t)(((uint64_t)c_segments_limit * 98ULL) / 100ULL);
663
664 c_segment_pages_compressed_limit = (c_segments_limit * (C_SEG_BUFSIZE / PAGE_SIZE) * compressor_pool_multiplier);
665
666 if (c_segment_pages_compressed_limit < (uint32_t)(max_mem / PAGE_SIZE))
667 c_segment_pages_compressed_limit = (uint32_t)(max_mem / PAGE_SIZE);
668
669 c_segment_pages_compressed_nearing_limit = (uint32_t)(((uint64_t)c_segment_pages_compressed_limit * 98ULL) / 100ULL);
39236c6e 670
39037602
A		 671 /*
672 * Submap needs space for:
673 * - c_segments
674 * - c_buffers
675 * - swap reclaimations -- C_SEG_BUFSIZE
676 */
5ba3f43e 677 c_segments_arr_size = vm_map_round_page((sizeof(union c_segu) * c_segments_limit), VM_MAP_PAGE_MASK(kernel_map));
39037602
A		 678 c_buffers_size = vm_map_round_page(((vm_size_t)C_SEG_ALLOCSIZE * (vm_size_t)c_segments_limit), VM_MAP_PAGE_MASK(kernel_map));
679
680 compressor_submap_size = c_segments_arr_size + c_buffers_size + C_SEG_BUFSIZE;
681
682#if RECORD_THE_COMPRESSED_DATA
683 c_compressed_record_sbuf_size = (vm_size_t)C_SEG_ALLOCSIZE + (PAGE_SIZE * 2);
684 compressor_submap_size += c_compressed_record_sbuf_size;
685#endif /* RECORD_THE_COMPRESSED_DATA */
686
5ba3f43e
A		 687 vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
688 vmk_flags.vmkf_permanent = TRUE;
39037602 689 retval = kmem_suballoc(kernel_map, &start_addr, compressor_submap_size,
5ba3f43e 690 FALSE, VM_FLAGS_ANYWHERE, vmk_flags, VM_KERN_MEMORY_COMPRESSOR,
39037602
A		 691 &compressor_map);
692
5ba3f43e
A		 693 if (retval != KERN_SUCCESS) {
694 if (++attempts > 3)
695 panic("vm_compressor_init: kmem_suballoc failed - 0x%llx", (uint64_t)compressor_submap_size);
39037602 696
5ba3f43e
A		 697 compressor_pool_size = compressor_pool_size / 2;
698
699 kprintf("retrying creation of the compressor submap at 0x%llx bytes\n", compressor_pool_size);
700 goto try_again;
701 }
39037602 702 if (kernel_memory_allocate(compressor_map, (vm_offset_t *)(&c_segments), (sizeof(union c_segu) * c_segments_limit), 0, KMA_KOBJECT | KMA_VAONLY | KMA_PERMANENT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS)
3e170ce0 703 panic("vm_compressor_init: kernel_memory_allocate failed - c_segments\n");
39037602 704 if (kernel_memory_allocate(compressor_map, &c_buffers, c_buffers_size, 0, KMA_COMPRESSOR | KMA_VAONLY | KMA_PERMANENT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS)
3e170ce0 705 panic("vm_compressor_init: kernel_memory_allocate failed - c_buffers\n");
39236c6e 706
5ba3f43e
A		 707
708 c_segment_min_size = sizeof(struct c_segment) + (C_SEG_SLOT_VAR_ARRAY_MIN_LEN * sizeof(struct c_slot));
709
710 for (c_segment_padded_size = 128; c_segment_padded_size < c_segment_min_size; c_segment_padded_size = c_segment_padded_size << 1);
711
712 compressor_segment_zone = zinit(c_segment_padded_size, c_segments_limit * c_segment_padded_size, PAGE_SIZE, "compressor_segment");
713 zone_change(compressor_segment_zone, Z_CALLERACCT, FALSE);
714 zone_change(compressor_segment_zone, Z_NOENCRYPT, TRUE);
715
716 c_seg_fixed_array_len = (c_segment_padded_size - sizeof(struct c_segment)) / sizeof(struct c_slot);
717
718 c_segments_busy = FALSE;
719
39236c6e 720 c_segments_next_page = (caddr_t)c_segments;
39037602 721 vm_compressor_algorithm_init();
39236c6e
A		 722
723 {
724 host_basic_info_data_t hinfo;
725 mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
726
727#define BSD_HOST 1
728 host_info((host_t)BSD_HOST, HOST_BASIC_INFO, (host_info_t)&hinfo, &count);
729
730 compressor_cpus = hinfo.max_cpus;
39037602 731 compressor_scratch_bufs = kalloc_tag(compressor_cpus * vm_compressor_get_decode_scratch_size(), VM_KERN_MEMORY_COMPRESSOR);
39236c6e 732
39037602 733 kdp_compressor_scratch_buf = kalloc_tag(vm_compressor_get_decode_scratch_size(), VM_KERN_MEMORY_COMPRESSOR);
3e170ce0
A		 734 kdp_compressor_decompressed_page = kalloc_tag(PAGE_SIZE, VM_KERN_MEMORY_COMPRESSOR);
735 kdp_compressor_decompressed_page_paddr = kvtophys((vm_offset_t)kdp_compressor_decompressed_page);
736 kdp_compressor_decompressed_page_ppnum = (ppnum_t) atop(kdp_compressor_decompressed_page_paddr);
39236c6e 737 }
39037602
A		 738#if CONFIG_FREEZE
739 freezer_compressor_scratch_buf = kalloc_tag(vm_compressor_get_encode_scratch_size(), VM_KERN_MEMORY_COMPRESSOR);
3e170ce0
A		 740#endif
741
742#if RECORD_THE_COMPRESSED_DATA
39037602 743 if (kernel_memory_allocate(compressor_map, (vm_offset_t *)&c_compressed_record_sbuf, c_compressed_record_sbuf_size, 0, KMA_KOBJECT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS)
3e170ce0
A		 744 panic("vm_compressor_init: kernel_memory_allocate failed - c_compressed_record_sbuf\n");
745
746 c_compressed_record_cptr = c_compressed_record_sbuf;
39037602 747 c_compressed_record_ebuf = c_compressed_record_sbuf + c_compressed_record_sbuf_size;
3e170ce0 748#endif
39236c6e
A		 749
750 if (kernel_thread_start_priority((thread_continue_t)vm_compressor_swap_trigger_thread, NULL,
5ba3f43e 751 BASEPRI_VM, &thread) != KERN_SUCCESS) {
39236c6e
A		 752 panic("vm_compressor_swap_trigger_thread: create failed");
753 }
39236c6e
A		 754 thread_deallocate(thread);
755
39236c6e
A		 756 if (vm_pageout_internal_start() != KERN_SUCCESS) {
757 panic("vm_compressor_init: Failed to start the internal pageout thread.\n");
758 }
39037602 759 if (VM_CONFIG_SWAP_IS_PRESENT)
fe8ab488 760 vm_compressor_swap_init();
fe8ab488 761
39037602 762 if (VM_CONFIG_COMPRESSOR_IS_ACTIVE)
04b8595b
A		 763 vm_compressor_is_active = 1;
764
39236c6e
A		 765#if CONFIG_FREEZE
766 memorystatus_freeze_enabled = TRUE;
767#endif /* CONFIG_FREEZE */
768
3e170ce0 769 vm_compressor_available = 1;
39236c6e
A		 770
771 vm_page_reactivate_all_throttled();
772}
773
774
775#if VALIDATE_C_SEGMENTS
776
777static void
778c_seg_validate(c_segment_t c_seg, boolean_t must_be_compact)
779{
780 int c_indx;
781 int32_t bytes_used;
39236c6e
A		 782 uint32_t c_rounded_size;
783 uint32_t c_size;
784 c_slot_t cs;
785
786 if (c_seg->c_firstemptyslot < c_seg->c_nextslot) {
787 c_indx = c_seg->c_firstemptyslot;
788 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
789
790 if (cs == NULL)
791 panic("c_seg_validate: no slot backing c_firstemptyslot");
792
793 if (cs->c_size)
794 panic("c_seg_validate: c_firstemptyslot has non-zero size (%d)\n", cs->c_size);
795 }
796 bytes_used = 0;
39236c6e
A		 797
798 for (c_indx = 0; c_indx < c_seg->c_nextslot; c_indx++) {
799
800 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
801
802 c_size = UNPACK_C_SIZE(cs);
803
804 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
805
806 bytes_used += c_rounded_size;
807
808#if CHECKSUM_THE_COMPRESSED_DATA
5ba3f43e
A		 809 unsigned csvhash;
810 if (c_size && cs->c_hash_compressed_data != (csvhash = vmc_hash((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size))) {
811 addr64_t csvphys = kvtophys((vm_offset_t)&c_seg->c_store.c_buffer[cs->c_offset]);
812 panic("Compressed data doesn't match original %p phys: 0x%llx %d %p %d %d 0x%x 0x%x", c_seg, csvphys, cs->c_offset, cs, c_indx, c_size, cs->c_hash_compressed_data, csvhash);
813 }
39236c6e
A		 814#endif
815 }
816
817 if (bytes_used != c_seg->c_bytes_used)
818 panic("c_seg_validate: bytes_used mismatch - found %d, segment has %d\n", bytes_used, c_seg->c_bytes_used);
819
820 if (c_seg->c_bytes_used > C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset))
821 panic("c_seg_validate: c_bytes_used > c_nextoffset - c_nextoffset = %d, c_bytes_used = %d\n",
822 (int32_t)C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset), c_seg->c_bytes_used);
823
824 if (must_be_compact) {
825 if (c_seg->c_bytes_used != C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset))
826 panic("c_seg_validate: c_bytes_used doesn't match c_nextoffset - c_nextoffset = %d, c_bytes_used = %d\n",
827 (int32_t)C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset), c_seg->c_bytes_used);
828 }
829}
830
831#endif
832
833
834void
39037602 835c_seg_need_delayed_compaction(c_segment_t c_seg, boolean_t c_list_lock_held)
39236c6e
A		 836{
837 boolean_t clear_busy = FALSE;
838
39037602
A		 839 if (c_list_lock_held == FALSE) {
840 if ( !lck_mtx_try_lock_spin_always(c_list_lock)) {
841 C_SEG_BUSY(c_seg);
39236c6e 842
39037602
A		 843 lck_mtx_unlock_always(&c_seg->c_lock);
844 lck_mtx_lock_spin_always(c_list_lock);
845 lck_mtx_lock_spin_always(&c_seg->c_lock);
39236c6e 846
39037602
A		 847 clear_busy = TRUE;
848 }
39236c6e 849 }
3e170ce0
A		 850 assert(c_seg->c_state != C_IS_FILLING);
851
a39ff7e2 852 if (!c_seg->c_on_minorcompact_q && !(C_SEG_IS_ON_DISK_OR_SOQ(c_seg))) {
39236c6e
A		 853 queue_enter(&c_minor_list_head, c_seg, c_segment_t, c_list);
854 c_seg->c_on_minorcompact_q = 1;
855 c_minor_count++;
856 }
39037602
A		 857 if (c_list_lock_held == FALSE)
858 lck_mtx_unlock_always(c_list_lock);
39236c6e
A		 859
860 if (clear_busy == TRUE)
861 C_SEG_WAKEUP_DONE(c_seg);
862}
863
864
865unsigned int c_seg_moved_to_sparse_list = 0;
866
867void
868c_seg_move_to_sparse_list(c_segment_t c_seg)
869{
870 boolean_t clear_busy = FALSE;
871
872 if ( !lck_mtx_try_lock_spin_always(c_list_lock)) {
fe8ab488 873 C_SEG_BUSY(c_seg);
39236c6e
A		 874
875 lck_mtx_unlock_always(&c_seg->c_lock);
876 lck_mtx_lock_spin_always(c_list_lock);
877 lck_mtx_lock_spin_always(&c_seg->c_lock);
878
879 clear_busy = TRUE;
880 }
3e170ce0 881 c_seg_switch_state(c_seg, C_ON_SWAPPEDOUTSPARSE_Q, FALSE);
39236c6e
A		 882
883 c_seg_moved_to_sparse_list++;
884
885 lck_mtx_unlock_always(c_list_lock);
886
887 if (clear_busy == TRUE)
888 C_SEG_WAKEUP_DONE(c_seg);
889}
890
891
892void
893c_seg_insert_into_q(queue_head_t *qhead, c_segment_t c_seg)
894{
895 c_segment_t c_seg_next;
896
897 if (queue_empty(qhead)) {
898 queue_enter(qhead, c_seg, c_segment_t, c_age_list);
899 } else {
900 c_seg_next = (c_segment_t)queue_first(qhead);
901
902 while (TRUE) {
903
904 if (c_seg->c_generation_id < c_seg_next->c_generation_id) {
905 queue_insert_before(qhead, c_seg, c_seg_next, c_segment_t, c_age_list);
906 break;
907 }
908 c_seg_next = (c_segment_t) queue_next(&c_seg_next->c_age_list);
909
910 if (queue_end(qhead, (queue_entry_t) c_seg_next)) {
911 queue_enter(qhead, c_seg, c_segment_t, c_age_list);
912 break;
913 }
914 }
915 }
916}
917
918
919int try_minor_compaction_failed = 0;
920int try_minor_compaction_succeeded = 0;
921
922void
923c_seg_try_minor_compaction_and_unlock(c_segment_t c_seg)
924{
925
926 assert(c_seg->c_on_minorcompact_q);
927 /*
928 * c_seg is currently on the delayed minor compaction
929 * queue and we have c_seg locked... if we can get the
930 * c_list_lock w/o blocking (if we blocked we could deadlock
931 * because the lock order is c_list_lock then c_seg's lock)
932 * we'll pull it from the delayed list and free it directly
933 */
934 if ( !lck_mtx_try_lock_spin_always(c_list_lock)) {
935 /*
936 * c_list_lock is held, we need to bail
937 */
938 try_minor_compaction_failed++;
939
940 lck_mtx_unlock_always(&c_seg->c_lock);
941 } else {
942 try_minor_compaction_succeeded++;
943
fe8ab488 944 C_SEG_BUSY(c_seg);
39236c6e
A		 945 c_seg_do_minor_compaction_and_unlock(c_seg, TRUE, FALSE, FALSE);
946 }
947}
948
949
950int
951c_seg_do_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy, boolean_t need_list_lock, boolean_t disallow_page_replacement)
952{
953 int c_seg_freed;
954
955 assert(c_seg->c_busy);
a39ff7e2 956 assert(!C_SEG_IS_ON_DISK_OR_SOQ(c_seg));
39236c6e 957
3e170ce0
A		 958 /*
959 * check for the case that can occur when we are not swapping
960 * and this segment has been major compacted in the past
961 * and moved to the majorcompact q to remove it from further
962 * consideration... if the occupancy falls too low we need
963 * to put it back on the age_q so that it will be considered
964 * in the next major compaction sweep... if we don't do this
965 * we will eventually run into the c_segments_limit
966 */
5ba3f43e 967 if (c_seg->c_state == C_ON_MAJORCOMPACT_Q && C_SEG_SHOULD_MAJORCOMPACT_NOW(c_seg)) {
3e170ce0
A		 968
969 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
970 }
39236c6e
A		 971 if (!c_seg->c_on_minorcompact_q) {
972 if (clear_busy == TRUE)
973 C_SEG_WAKEUP_DONE(c_seg);
974
975 lck_mtx_unlock_always(&c_seg->c_lock);
976
977 return (0);
978 }
979 queue_remove(&c_minor_list_head, c_seg, c_segment_t, c_list);
980 c_seg->c_on_minorcompact_q = 0;
981 c_minor_count--;
982
983 lck_mtx_unlock_always(c_list_lock);
984
985 if (disallow_page_replacement == TRUE) {
986 lck_mtx_unlock_always(&c_seg->c_lock);
987
988 PAGE_REPLACEMENT_DISALLOWED(TRUE);
989
990 lck_mtx_lock_spin_always(&c_seg->c_lock);
991 }
992 c_seg_freed = c_seg_minor_compaction_and_unlock(c_seg, clear_busy);
993
994 if (disallow_page_replacement == TRUE)
995 PAGE_REPLACEMENT_DISALLOWED(FALSE);
996
997 if (need_list_lock == TRUE)
998 lck_mtx_lock_spin_always(c_list_lock);
999
1000 return (c_seg_freed);
1001}
1002
1003
1004void
1005c_seg_wait_on_busy(c_segment_t c_seg)
1006{
1007 c_seg->c_wanted = 1;
1008 assert_wait((event_t) (c_seg), THREAD_UNINT);
1009
1010 lck_mtx_unlock_always(&c_seg->c_lock);
1011 thread_block(THREAD_CONTINUE_NULL);
1012}
1013
1014
3e170ce0
A		 1015void
1016c_seg_switch_state(c_segment_t c_seg, int new_state, boolean_t insert_head)
1017{
1018 int old_state = c_seg->c_state;
39236c6e 1019
3e170ce0
A		 1020#if __i386__ || __x86_64__
1021 if (new_state != C_IS_FILLING)
39037602
A		 1022 LCK_MTX_ASSERT(&c_seg->c_lock, LCK_MTX_ASSERT_OWNED);
1023 LCK_MTX_ASSERT(c_list_lock, LCK_MTX_ASSERT_OWNED);
3e170ce0
A		 1024#endif
1025 switch (old_state) {
39236c6e 1026
3e170ce0
A		 1027 case C_IS_EMPTY:
1028 assert(new_state == C_IS_FILLING || new_state == C_IS_FREE);
39236c6e 1029
3e170ce0
A		 1030 c_empty_count--;
1031 break;
1032
1033 case C_IS_FILLING:
1034 assert(new_state == C_ON_AGE_Q || new_state == C_ON_SWAPOUT_Q);
1035
1036 queue_remove(&c_filling_list_head, c_seg, c_segment_t, c_age_list);
1037 c_filling_count--;
1038 break;
1039
1040 case C_ON_AGE_Q:
1041 assert(new_state == C_ON_SWAPOUT_Q || new_state == C_ON_MAJORCOMPACT_Q ||
1042 new_state == C_IS_FREE);
1043
1044 queue_remove(&c_age_list_head, c_seg, c_segment_t, c_age_list);
1045 c_age_count--;
1046 break;
1047
1048 case C_ON_SWAPPEDIN_Q:
1049 assert(new_state == C_ON_AGE_Q || new_state == C_IS_FREE);
1050
1051 queue_remove(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
1052 c_swappedin_count--;
1053 break;
1054
1055 case C_ON_SWAPOUT_Q:
1056 assert(new_state == C_ON_SWAPPEDOUT_Q || new_state == C_ON_SWAPPEDOUTSPARSE_Q ||
1057 new_state == C_ON_AGE_Q || new_state == C_IS_FREE || new_state == C_IS_EMPTY);
1058
1059 queue_remove(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
1060 thread_wakeup((event_t)&compaction_swapper_running);
1061 c_swapout_count--;
1062 break;
1063
1064 case C_ON_SWAPPEDOUT_Q:
39037602
A		 1065 assert(new_state == C_ON_SWAPPEDIN_Q || new_state == C_ON_AGE_Q ||
1066 new_state == C_ON_SWAPPEDOUTSPARSE_Q ||
3e170ce0
A		 1067 new_state == C_ON_BAD_Q || new_state == C_IS_EMPTY || new_state == C_IS_FREE);
1068
1069 queue_remove(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
1070 c_swappedout_count--;
1071 break;
1072
1073 case C_ON_SWAPPEDOUTSPARSE_Q:
39037602 1074 assert(new_state == C_ON_SWAPPEDIN_Q || new_state == C_ON_AGE_Q ||
3e170ce0
A		 1075 new_state == C_ON_BAD_Q || new_state == C_IS_EMPTY || new_state == C_IS_FREE);
1076
1077 queue_remove(&c_swappedout_sparse_list_head, c_seg, c_segment_t, c_age_list);
1078 c_swappedout_sparse_count--;
1079 break;
1080
1081 case C_ON_MAJORCOMPACT_Q:
1082 assert(new_state == C_ON_AGE_Q || new_state == C_IS_FREE);
1083
1084 queue_remove(&c_major_list_head, c_seg, c_segment_t, c_age_list);
1085 c_major_count--;
1086 break;
1087
1088 case C_ON_BAD_Q:
1089 assert(new_state == C_IS_FREE);
1090
1091 queue_remove(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
1092 c_bad_count--;
1093 break;
1094
1095 default:
1096 panic("c_seg %p has bad c_state = %d\n", c_seg, old_state);
39236c6e 1097 }
39236c6e 1098
3e170ce0
A		 1099 switch(new_state) {
1100 case C_IS_FREE:
1101 assert(old_state != C_IS_FILLING);
1102
1103 break;
39236c6e 1104
3e170ce0
A		 1105 case C_IS_EMPTY:
1106 assert(old_state == C_ON_SWAPOUT_Q || old_state == C_ON_SWAPPEDOUT_Q || old_state == C_ON_SWAPPEDOUTSPARSE_Q);
39236c6e 1107
3e170ce0
A		 1108 c_empty_count++;
1109 break;
1110
1111 case C_IS_FILLING:
1112 assert(old_state == C_IS_EMPTY);
1113
1114 queue_enter(&c_filling_list_head, c_seg, c_segment_t, c_age_list);
1115 c_filling_count++;
1116 break;
1117
1118 case C_ON_AGE_Q:
39037602
A		 1119 assert(old_state == C_IS_FILLING || old_state == C_ON_SWAPPEDIN_Q || old_state == C_ON_SWAPOUT_Q ||
1120 old_state == C_ON_MAJORCOMPACT_Q || old_state == C_ON_SWAPPEDOUT_Q || old_state == C_ON_SWAPPEDOUTSPARSE_Q);
3e170ce0
A		 1121
1122 if (old_state == C_IS_FILLING)
1123 queue_enter(&c_age_list_head, c_seg, c_segment_t, c_age_list);
39037602
A		 1124 else {
1125 if (!queue_empty(&c_age_list_head)) {
1126 c_segment_t c_first;
1127
1128 c_first = (c_segment_t)queue_first(&c_age_list_head);
1129 c_seg->c_creation_ts = c_first->c_creation_ts;
1130 }
1131 queue_enter_first(&c_age_list_head, c_seg, c_segment_t, c_age_list);
1132 }
3e170ce0
A		 1133 c_age_count++;
1134 break;
1135
1136 case C_ON_SWAPPEDIN_Q:
1137 assert(c_seg->c_state == C_ON_SWAPPEDOUT_Q || c_seg->c_state == C_ON_SWAPPEDOUTSPARSE_Q);
1138
1139 if (insert_head == TRUE)
1140 queue_enter_first(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
1141 else
1142 queue_enter(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
1143 c_swappedin_count++;
1144 break;
1145
1146 case C_ON_SWAPOUT_Q:
1147 assert(old_state == C_ON_AGE_Q || old_state == C_IS_FILLING);
1148
1149 if (insert_head == TRUE)
1150 queue_enter_first(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
1151 else
1152 queue_enter(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
1153 c_swapout_count++;
1154 break;
1155
1156 case C_ON_SWAPPEDOUT_Q:
1157 assert(c_seg->c_state == C_ON_SWAPOUT_Q);
1158
1159 if (insert_head == TRUE)
1160 queue_enter_first(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
1161 else
1162 queue_enter(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
1163 c_swappedout_count++;
1164 break;
1165
1166 case C_ON_SWAPPEDOUTSPARSE_Q:
1167 assert(c_seg->c_state == C_ON_SWAPOUT_Q || c_seg->c_state == C_ON_SWAPPEDOUT_Q);
1168
39037602
A		 1169 if (insert_head == TRUE)
1170 queue_enter_first(&c_swappedout_sparse_list_head, c_seg, c_segment_t, c_age_list);
1171 else
1172 queue_enter(&c_swappedout_sparse_list_head, c_seg, c_segment_t, c_age_list);
1173
3e170ce0
A		 1174 c_swappedout_sparse_count++;
1175 break;
1176
1177 case C_ON_MAJORCOMPACT_Q:
1178 assert(c_seg->c_state == C_ON_AGE_Q);
1179
1180 if (insert_head == TRUE)
1181 queue_enter_first(&c_major_list_head, c_seg, c_segment_t, c_age_list);
1182 else
1183 queue_enter(&c_major_list_head, c_seg, c_segment_t, c_age_list);
1184 c_major_count++;
1185 break;
1186
1187 case C_ON_BAD_Q:
1188 assert(c_seg->c_state == C_ON_SWAPPEDOUT_Q || c_seg->c_state == C_ON_SWAPPEDOUTSPARSE_Q);
1189
1190 if (insert_head == TRUE)
1191 queue_enter_first(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
1192 else
1193 queue_enter(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
1194 c_bad_count++;
1195 break;
1196
1197 default:
1198 panic("c_seg %p requesting bad c_state = %d\n", c_seg, new_state);
1199 }
1200 c_seg->c_state = new_state;
39236c6e
A		 1201}
1202
1203
3e170ce0 1204
39236c6e
A		 1205void
1206c_seg_free(c_segment_t c_seg)
1207{
fe8ab488 1208 assert(c_seg->c_busy);
39236c6e
A		 1209
1210 lck_mtx_unlock_always(&c_seg->c_lock);
1211 lck_mtx_lock_spin_always(c_list_lock);
1212 lck_mtx_lock_spin_always(&c_seg->c_lock);
1213
1214 c_seg_free_locked(c_seg);
1215}
1216
1217
1218void
1219c_seg_free_locked(c_segment_t c_seg)
1220{
3e170ce0 1221 int segno;
04b8595b 1222 int pages_populated = 0;
39236c6e 1223 int32_t *c_buffer = NULL;
04b8595b 1224 uint64_t c_swap_handle = 0;
39236c6e 1225
3e170ce0 1226 assert(c_seg->c_busy);
5ba3f43e 1227 assert(c_seg->c_slots_used == 0);
39236c6e 1228 assert(!c_seg->c_on_minorcompact_q);
3e170ce0 1229 assert(!c_seg->c_busy_swapping);
39236c6e 1230
3e170ce0
A		 1231 if (c_seg->c_overage_swap == TRUE) {
1232 c_overage_swapped_count--;
1233 c_seg->c_overage_swap = FALSE;
1234 }
1235 if ( !(C_SEG_IS_ONDISK(c_seg)))
1236 c_buffer = c_seg->c_store.c_buffer;
1237 else
1238 c_swap_handle = c_seg->c_store.c_swap_handle;
39236c6e 1239
3e170ce0 1240 c_seg_switch_state(c_seg, C_IS_FREE, FALSE);
39236c6e 1241
3e170ce0 1242 lck_mtx_unlock_always(c_list_lock);
39236c6e 1243
3e170ce0 1244 if (c_buffer) {
39236c6e 1245 pages_populated = (round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE;
39236c6e 1246 c_seg->c_store.c_buffer = NULL;
3e170ce0 1247 } else
39236c6e 1248 c_seg->c_store.c_swap_handle = (uint64_t)-1;
3e170ce0 1249
39236c6e
A		 1250 lck_mtx_unlock_always(&c_seg->c_lock);
1251
1252 if (c_buffer) {
8a3053a0 1253 if (pages_populated)
39037602 1254 kernel_memory_depopulate(compressor_map, (vm_offset_t) c_buffer, pages_populated * PAGE_SIZE, KMA_COMPRESSOR);
39236c6e 1255
3e170ce0
A		 1256 } else if (c_swap_handle) {
1257 /*
1258 * Free swap space on disk.
1259 */
39236c6e 1260 vm_swap_free(c_swap_handle);
3e170ce0
A		 1261 }
1262 lck_mtx_lock_spin_always(&c_seg->c_lock);
5ba3f43e
A		 1263 /*
1264 * c_seg must remain busy until
1265 * after the call to vm_swap_free
1266 */
3e170ce0
A		 1267 C_SEG_WAKEUP_DONE(c_seg);
1268 lck_mtx_unlock_always(&c_seg->c_lock);
1269
1270 segno = c_seg->c_mysegno;
39236c6e 1271
3e170ce0
A		 1272 lck_mtx_lock_spin_always(c_list_lock);
1273 /*
1274 * because the c_buffer is now associated with the segno,
1275 * we can't put the segno back on the free list until
1276 * after we have depopulated the c_buffer range, or
1277 * we run the risk of depopulating a range that is
1278 * now being used in one of the compressor heads
1279 */
1280 c_segments[segno].c_segno = c_free_segno_head;
1281 c_free_segno_head = segno;
1282 c_segment_count--;
1283
1284 lck_mtx_unlock_always(c_list_lock);
39236c6e 1285
39236c6e 1286 lck_mtx_destroy(&c_seg->c_lock, &vm_compressor_lck_grp);
39236c6e 1287
3e170ce0
A		 1288 if (c_seg->c_slot_var_array_len)
1289 kfree(c_seg->c_slot_var_array, sizeof(struct c_slot) * c_seg->c_slot_var_array_len);
39236c6e 1290
39236c6e
A		 1291 zfree(compressor_segment_zone, c_seg);
1292}
1293
5ba3f43e 1294#if DEVELOPMENT || DEBUG
39236c6e 1295int c_seg_trim_page_count = 0;
5ba3f43e 1296#endif
39236c6e
A		 1297
1298void
1299c_seg_trim_tail(c_segment_t c_seg)
1300{
1301 c_slot_t cs;
1302 uint32_t c_size;
1303 uint32_t c_offset;
1304 uint32_t c_rounded_size;
1305 uint16_t current_nextslot;
1306 uint32_t current_populated_offset;
1307
1308 if (c_seg->c_bytes_used == 0)
1309 return;
1310 current_nextslot = c_seg->c_nextslot;
1311 current_populated_offset = c_seg->c_populated_offset;
1312
1313 while (c_seg->c_nextslot) {
1314
1315 cs = C_SEG_SLOT_FROM_INDEX(c_seg, (c_seg->c_nextslot - 1));
1316
1317 c_size = UNPACK_C_SIZE(cs);
1318
1319 if (c_size) {
1320 if (current_nextslot != c_seg->c_nextslot) {
1321 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
1322 c_offset = cs->c_offset + C_SEG_BYTES_TO_OFFSET(c_rounded_size);
1323
1324 c_seg->c_nextoffset = c_offset;
5ba3f43e
A		 1325 c_seg->c_populated_offset = (c_offset + (C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1)) &
1326 ~(C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1);
39236c6e
A		 1327
1328 if (c_seg->c_firstemptyslot > c_seg->c_nextslot)
1329 c_seg->c_firstemptyslot = c_seg->c_nextslot;
5ba3f43e 1330#if DEVELOPMENT || DEBUG
39236c6e 1331 c_seg_trim_page_count += ((round_page_32(C_SEG_OFFSET_TO_BYTES(current_populated_offset)) -
5ba3f43e
A		 1332 round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) /
1333 PAGE_SIZE);
1334#endif
39236c6e
A		 1335 }
1336 break;
1337 }
1338 c_seg->c_nextslot--;
1339 }
1340 assert(c_seg->c_nextslot);
1341}
1342
1343
1344int
1345c_seg_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy)
1346{
1347 c_slot_mapping_t slot_ptr;
1348 uint32_t c_offset = 0;
1349 uint32_t old_populated_offset;
1350 uint32_t c_rounded_size;
1351 uint32_t c_size;
1352 int c_indx = 0;
1353 int i;
1354 c_slot_t c_dst;
1355 c_slot_t c_src;
39236c6e
A		 1356
1357 assert(c_seg->c_busy);
1358
1359#if VALIDATE_C_SEGMENTS
1360 c_seg_validate(c_seg, FALSE);
1361#endif
1362 if (c_seg->c_bytes_used == 0) {
1363 c_seg_free(c_seg);
1364 return (1);
1365 }
39037602
A		 1366 lck_mtx_unlock_always(&c_seg->c_lock);
1367
39236c6e
A		 1368 if (c_seg->c_firstemptyslot >= c_seg->c_nextslot || C_SEG_UNUSED_BYTES(c_seg) < PAGE_SIZE)
1369 goto done;
5ba3f43e
A		 1370
1371/* TODO: assert first emptyslot's c_size is actually 0 */
1372
39037602
A		 1373#if DEVELOPMENT || DEBUG
1374 C_SEG_MAKE_WRITEABLE(c_seg);
1375#endif
1376
39236c6e
A		 1377#if VALIDATE_C_SEGMENTS
1378 c_seg->c_was_minor_compacted++;
1379#endif
1380 c_indx = c_seg->c_firstemptyslot;
1381 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
1382
1383 old_populated_offset = c_seg->c_populated_offset;
1384 c_offset = c_dst->c_offset;
1385
1386 for (i = c_indx + 1; i < c_seg->c_nextslot && c_offset < c_seg->c_nextoffset; i++) {
1387
1388 c_src = C_SEG_SLOT_FROM_INDEX(c_seg, i);
1389
1390 c_size = UNPACK_C_SIZE(c_src);
1391
1392 if (c_size == 0)
1393 continue;
1394
39037602 1395 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
5ba3f43e 1396/* N.B.: This memcpy may be an overlapping copy */
39037602
A		 1397 memcpy(&c_seg->c_store.c_buffer[c_offset], &c_seg->c_store.c_buffer[c_src->c_offset], c_rounded_size);
1398
1399 cslot_copy(c_dst, c_src);
39236c6e
A		 1400 c_dst->c_offset = c_offset;
1401
1402 slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
1403 slot_ptr->s_cindx = c_indx;
1404
39236c6e
A		 1405 c_offset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
1406 PACK_C_SIZE(c_src, 0);
1407 c_indx++;
1408
1409 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
1410 }
1411 c_seg->c_firstemptyslot = c_indx;
1412 c_seg->c_nextslot = c_indx;
1413 c_seg->c_nextoffset = c_offset;
1414 c_seg->c_populated_offset = (c_offset + (C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1)) & ~(C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1);
1415 c_seg->c_bytes_unused = 0;
1416
1417#if VALIDATE_C_SEGMENTS
1418 c_seg_validate(c_seg, TRUE);
1419#endif
39236c6e
A		 1420 if (old_populated_offset > c_seg->c_populated_offset) {
1421 uint32_t gc_size;
1422 int32_t *gc_ptr;
1423
1424 gc_size = C_SEG_OFFSET_TO_BYTES(old_populated_offset - c_seg->c_populated_offset);
1425 gc_ptr = &c_seg->c_store.c_buffer[c_seg->c_populated_offset];
1426
39037602
A		 1427 kernel_memory_depopulate(compressor_map, (vm_offset_t)gc_ptr, gc_size, KMA_COMPRESSOR);
1428 }
39236c6e 1429
39037602
A		 1430#if DEVELOPMENT || DEBUG
1431 C_SEG_WRITE_PROTECT(c_seg);
1432#endif
39236c6e 1433
39236c6e 1434done:
39037602
A		 1435 if (clear_busy == TRUE) {
1436 lck_mtx_lock_spin_always(&c_seg->c_lock);
1437 C_SEG_WAKEUP_DONE(c_seg);
39236c6e
A		 1438 lck_mtx_unlock_always(&c_seg->c_lock);
1439 }
1440 return (0);
1441}
1442
1443
3e170ce0
A		 1444static void
1445c_seg_alloc_nextslot(c_segment_t c_seg)
1446{
1447 struct c_slot *old_slot_array = NULL;
1448 struct c_slot *new_slot_array = NULL;
1449 int newlen;
1450 int oldlen;
1451
1452 if (c_seg->c_nextslot < c_seg_fixed_array_len)
1453 return;
1454
1455 if ((c_seg->c_nextslot - c_seg_fixed_array_len) >= c_seg->c_slot_var_array_len) {
1456
1457 oldlen = c_seg->c_slot_var_array_len;
1458 old_slot_array = c_seg->c_slot_var_array;
1459
1460 if (oldlen == 0)
1461 newlen = C_SEG_SLOT_VAR_ARRAY_MIN_LEN;
1462 else
1463 newlen = oldlen * 2;
1464
1465 new_slot_array = (struct c_slot *)kalloc(sizeof(struct c_slot) * newlen);
1466
1467 lck_mtx_lock_spin_always(&c_seg->c_lock);
1468
1469 if (old_slot_array)
1470 memcpy((char *)new_slot_array, (char *)old_slot_array, sizeof(struct c_slot) * oldlen);
1471
1472 c_seg->c_slot_var_array_len = newlen;
1473 c_seg->c_slot_var_array = new_slot_array;
1474
1475 lck_mtx_unlock_always(&c_seg->c_lock);
1476
1477 if (old_slot_array)
1478 kfree(old_slot_array, sizeof(struct c_slot) * oldlen);
1479 }
1480}
1481
1482
39236c6e
A		 1483
1484struct {
1485 uint64_t asked_permission;
1486 uint64_t compactions;
1487 uint64_t moved_slots;
1488 uint64_t moved_bytes;
1489 uint64_t wasted_space_in_swapouts;
1490 uint64_t count_of_swapouts;
3e170ce0 1491 uint64_t count_of_freed_segs;
39236c6e
A		 1492} c_seg_major_compact_stats;
1493
1494
04b8595b 1495#define C_MAJOR_COMPACTION_SIZE_APPROPRIATE ((C_SEG_BUFSIZE * 90) / 100)
39236c6e
A		 1496
1497
1498boolean_t
1499c_seg_major_compact_ok(
1500 c_segment_t c_seg_dst,
1501 c_segment_t c_seg_src)
1502{
1503
1504 c_seg_major_compact_stats.asked_permission++;
1505
39236c6e
A		 1506 if (c_seg_src->c_bytes_used >= C_MAJOR_COMPACTION_SIZE_APPROPRIATE &&
1507 c_seg_dst->c_bytes_used >= C_MAJOR_COMPACTION_SIZE_APPROPRIATE)
1508 return (FALSE);
1509
3e170ce0 1510 if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX_INDEX) {
39236c6e
A		 1511 /*
1512 * destination segment is full... can't compact
1513 */
1514 return (FALSE);
1515 }
1516
1517 return (TRUE);
1518}
1519
1520
1521boolean_t
1522c_seg_major_compact(
1523 c_segment_t c_seg_dst,
1524 c_segment_t c_seg_src)
1525{
1526 c_slot_mapping_t slot_ptr;
1527 uint32_t c_rounded_size;
1528 uint32_t c_size;
1529 uint16_t dst_slot;
1530 int i;
1531 c_slot_t c_dst;
1532 c_slot_t c_src;
39236c6e
A		 1533 boolean_t keep_compacting = TRUE;
1534
1535 /*
1536 * segments are not locked but they are both marked c_busy
1537 * which keeps c_decompress from working on them...
1538 * we can safely allocate new pages, move compressed data
1539 * from c_seg_src to c_seg_dst and update both c_segment's
1540 * state w/o holding the master lock
1541 */
39037602
A		 1542#if DEVELOPMENT || DEBUG
1543 C_SEG_MAKE_WRITEABLE(c_seg_dst);
1544#endif
39236c6e
A		 1545
1546#if VALIDATE_C_SEGMENTS
1547 c_seg_dst->c_was_major_compacted++;
1548 c_seg_src->c_was_major_donor++;
1549#endif
1550 c_seg_major_compact_stats.compactions++;
1551
1552 dst_slot = c_seg_dst->c_nextslot;
1553
1554 for (i = 0; i < c_seg_src->c_nextslot; i++) {
1555
1556 c_src = C_SEG_SLOT_FROM_INDEX(c_seg_src, i);
1557
1558 c_size = UNPACK_C_SIZE(c_src);
1559
1560 if (c_size == 0) {
1561 /* BATCH: move what we have so far; */
1562 continue;
1563 }
1564
1565 if (C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset - c_seg_dst->c_nextoffset) < (unsigned) c_size) {
3e170ce0
A		 1566 int size_to_populate;
1567
39236c6e 1568 /* doesn't fit */
3e170ce0
A		 1569 size_to_populate = C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset);
1570
1571 if (size_to_populate == 0) {
39236c6e
A		 1572 /* can't fit */
1573 keep_compacting = FALSE;
1574 break;
1575 }
3e170ce0
A		 1576 if (size_to_populate > C_SEG_MAX_POPULATE_SIZE)
1577 size_to_populate = C_SEG_MAX_POPULATE_SIZE;
1578
39037602 1579 kernel_memory_populate(compressor_map,
39236c6e 1580 (vm_offset_t) &c_seg_dst->c_store.c_buffer[c_seg_dst->c_populated_offset],
3e170ce0
A		 1581 size_to_populate,
1582 KMA_COMPRESSOR,
1583 VM_KERN_MEMORY_COMPRESSOR);
39236c6e 1584
3e170ce0 1585 c_seg_dst->c_populated_offset += C_SEG_BYTES_TO_OFFSET(size_to_populate);
39236c6e
A		 1586 assert(C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset) <= C_SEG_BUFSIZE);
1587 }
3e170ce0 1588 c_seg_alloc_nextslot(c_seg_dst);
39236c6e 1589
39236c6e
A		 1590 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, c_seg_dst->c_nextslot);
1591
1592 memcpy(&c_seg_dst->c_store.c_buffer[c_seg_dst->c_nextoffset], &c_seg_src->c_store.c_buffer[c_src->c_offset], c_size);
1593
1594 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
1595
1596 c_seg_major_compact_stats.moved_slots++;
1597 c_seg_major_compact_stats.moved_bytes += c_size;
1598
39037602 1599 cslot_copy(c_dst, c_src);
39236c6e
A		 1600 c_dst->c_offset = c_seg_dst->c_nextoffset;
1601
1602 if (c_seg_dst->c_firstemptyslot == c_seg_dst->c_nextslot)
1603 c_seg_dst->c_firstemptyslot++;
5ba3f43e 1604 c_seg_dst->c_slots_used++;
39236c6e
A		 1605 c_seg_dst->c_nextslot++;
1606 c_seg_dst->c_bytes_used += c_rounded_size;
1607 c_seg_dst->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
1608
1609 PACK_C_SIZE(c_src, 0);
1610
1611 c_seg_src->c_bytes_used -= c_rounded_size;
1612 c_seg_src->c_bytes_unused += c_rounded_size;
1613 c_seg_src->c_firstemptyslot = 0;
1614
5ba3f43e
A		 1615 assert(c_seg_src->c_slots_used);
1616 c_seg_src->c_slots_used--;
1617
3e170ce0 1618 if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX_INDEX) {
39236c6e
A		 1619 /* dest segment is now full */
1620 keep_compacting = FALSE;
1621 break;
1622 }
1623 }
39037602
A		 1624#if DEVELOPMENT || DEBUG
1625 C_SEG_WRITE_PROTECT(c_seg_dst);
1626#endif
39236c6e
A		 1627 if (dst_slot < c_seg_dst->c_nextslot) {
1628
1629 PAGE_REPLACEMENT_ALLOWED(TRUE);
1630 /*
1631 * we've now locked out c_decompress from
1632 * converting the slot passed into it into
1633 * a c_segment_t which allows us to use
1634 * the backptr to change which c_segment and
1635 * index the slot points to
1636 */
1637 while (dst_slot < c_seg_dst->c_nextslot) {
1638
1639 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, dst_slot);
1640
1641 slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
1642 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
1643 slot_ptr->s_cseg = c_seg_dst->c_mysegno + 1;
1644 slot_ptr->s_cindx = dst_slot++;
1645 }
1646 PAGE_REPLACEMENT_ALLOWED(FALSE);
1647 }
1648 return (keep_compacting);
1649}
1650
1651
fe8ab488
A		 1652uint64_t
1653vm_compressor_compute_elapsed_msecs(clock_sec_t end_sec, clock_nsec_t end_nsec, clock_sec_t start_sec, clock_nsec_t start_nsec)
39236c6e
A		 1654{
1655 uint64_t end_msecs;
1656 uint64_t start_msecs;
1657
1658 end_msecs = (end_sec * 1000) + end_nsec / 1000000;
1659 start_msecs = (start_sec * 1000) + start_nsec / 1000000;
1660
1661 return (end_msecs - start_msecs);
1662}
1663
1664
1665
1666uint32_t compressor_eval_period_in_msecs = 250;
1667uint32_t compressor_sample_min_in_msecs = 500;
1668uint32_t compressor_sample_max_in_msecs = 10000;
1669uint32_t compressor_thrashing_threshold_per_10msecs = 50;
1670uint32_t compressor_thrashing_min_per_10msecs = 20;
1671
fe8ab488
A		 1672/* When true, reset sample data next chance we get. */
1673static boolean_t compressor_need_sample_reset = FALSE;
1674
39236c6e
A		 1675extern uint32_t vm_page_filecache_min;
1676
1677
1678void
1679compute_swapout_target_age(void)
1680{
1681 clock_sec_t cur_ts_sec;
1682 clock_nsec_t cur_ts_nsec;
1683 uint32_t min_operations_needed_in_this_sample;
1684 uint64_t elapsed_msecs_in_eval;
1685 uint64_t elapsed_msecs_in_sample;
39236c6e
A		 1686 boolean_t need_eval_reset = FALSE;
1687
1688 clock_get_system_nanotime(&cur_ts_sec, &cur_ts_nsec);
1689
fe8ab488 1690 elapsed_msecs_in_sample = vm_compressor_compute_elapsed_msecs(cur_ts_sec, cur_ts_nsec, start_of_sample_period_sec, start_of_sample_period_nsec);
39236c6e 1691
fe8ab488
A		 1692 if (compressor_need_sample_reset ||
1693 elapsed_msecs_in_sample >= compressor_sample_max_in_msecs) {
1694 compressor_need_sample_reset = TRUE;
39236c6e
A		 1695 need_eval_reset = TRUE;
1696 goto done;
1697 }
fe8ab488 1698 elapsed_msecs_in_eval = vm_compressor_compute_elapsed_msecs(cur_ts_sec, cur_ts_nsec, start_of_eval_period_sec, start_of_eval_period_nsec);
39236c6e
A		 1699
1700 if (elapsed_msecs_in_eval < compressor_eval_period_in_msecs)
1701 goto done;
1702 need_eval_reset = TRUE;
1703
1704 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_START, elapsed_msecs_in_eval, sample_period_compression_count, sample_period_decompression_count, 0, 0);
1705
1706 min_operations_needed_in_this_sample = (compressor_thrashing_min_per_10msecs * (uint32_t)elapsed_msecs_in_eval) / 10;
1707
1708 if ((sample_period_compression_count - last_eval_compression_count) < min_operations_needed_in_this_sample ||
1709 (sample_period_decompression_count - last_eval_decompression_count) < min_operations_needed_in_this_sample) {
1710
1711 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, sample_period_compression_count - last_eval_compression_count,
1712 sample_period_decompression_count - last_eval_decompression_count, 0, 1, 0);
1713
1714 swapout_target_age = 0;
1715
fe8ab488 1716 compressor_need_sample_reset = TRUE;
39236c6e
A		 1717 need_eval_reset = TRUE;
1718 goto done;
1719 }
1720 last_eval_compression_count = sample_period_compression_count;
1721 last_eval_decompression_count = sample_period_decompression_count;
1722
1723 if (elapsed_msecs_in_sample < compressor_sample_min_in_msecs) {
1724
1725 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, swapout_target_age, 0, 0, 5, 0);
1726 goto done;
1727 }
1728 if (sample_period_decompression_count > ((compressor_thrashing_threshold_per_10msecs * elapsed_msecs_in_sample) / 10)) {
1729
1730 uint64_t running_total;
1731 uint64_t working_target;
1732 uint64_t aging_target;
1733 uint32_t oldest_age_of_csegs_sampled = 0;
1734 uint64_t working_set_approximation = 0;
1735
1736 swapout_target_age = 0;
1737
1738 working_target = (sample_period_decompression_count / 100) * 95; /* 95 percent */
1739 aging_target = (sample_period_decompression_count / 100) * 1; /* 1 percent */
1740 running_total = 0;
1741
1742 for (oldest_age_of_csegs_sampled = 0; oldest_age_of_csegs_sampled < DECOMPRESSION_SAMPLE_MAX_AGE; oldest_age_of_csegs_sampled++) {
1743
1744 running_total += age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
1745
1746 working_set_approximation += oldest_age_of_csegs_sampled * age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
1747
1748 if (running_total >= working_target)
1749 break;
1750 }
1751 if (oldest_age_of_csegs_sampled < DECOMPRESSION_SAMPLE_MAX_AGE) {
1752
1753 working_set_approximation = (working_set_approximation * 1000) / elapsed_msecs_in_sample;
1754
1755 if (working_set_approximation < VM_PAGE_COMPRESSOR_COUNT) {
1756
1757 running_total = overage_decompressions_during_sample_period;
1758
1759 for (oldest_age_of_csegs_sampled = DECOMPRESSION_SAMPLE_MAX_AGE - 1; oldest_age_of_csegs_sampled; oldest_age_of_csegs_sampled--) {
1760 running_total += age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
1761
1762 if (running_total >= aging_target)
1763 break;
1764 }
1765 swapout_target_age = (uint32_t)cur_ts_sec - oldest_age_of_csegs_sampled;
1766
1767 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, swapout_target_age, working_set_approximation, VM_PAGE_COMPRESSOR_COUNT, 2, 0);
1768 } else {
1769 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, working_set_approximation, VM_PAGE_COMPRESSOR_COUNT, 0, 3, 0);
1770 }
1771 } else
1772 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, working_target, running_total, 0, 4, 0);
1773
fe8ab488 1774 compressor_need_sample_reset = TRUE;
39236c6e
A		 1775 need_eval_reset = TRUE;
1776 } else
1777 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, sample_period_decompression_count, (compressor_thrashing_threshold_per_10msecs * elapsed_msecs_in_sample) / 10, 0, 6, 0);
1778done:
fe8ab488 1779 if (compressor_need_sample_reset == TRUE) {
39236c6e
A		 1780 bzero(age_of_decompressions_during_sample_period, sizeof(age_of_decompressions_during_sample_period));
1781 overage_decompressions_during_sample_period = 0;
1782
1783 start_of_sample_period_sec = cur_ts_sec;
1784 start_of_sample_period_nsec = cur_ts_nsec;
1785 sample_period_decompression_count = 0;
1786 sample_period_compression_count = 0;
1787 last_eval_decompression_count = 0;
1788 last_eval_compression_count = 0;
fe8ab488 1789 compressor_need_sample_reset = FALSE;
39236c6e
A		 1790 }
1791 if (need_eval_reset == TRUE) {
1792 start_of_eval_period_sec = cur_ts_sec;
1793 start_of_eval_period_nsec = cur_ts_nsec;
1794 }
1795}
1796
1797
fe8ab488 1798int compaction_swapper_init_now = 0;
39236c6e 1799int compaction_swapper_running = 0;
39037602 1800int compaction_swapper_awakened = 0;
39236c6e
A		 1801int compaction_swapper_abort = 0;
1802
1803
1804#if CONFIG_JETSAM
1805boolean_t memorystatus_kill_on_VM_thrashing(boolean_t);
fe8ab488 1806boolean_t memorystatus_kill_on_FC_thrashing(boolean_t);
39236c6e 1807int compressor_thrashing_induced_jetsam = 0;
fe8ab488
A		 1808int filecache_thrashing_induced_jetsam = 0;
1809static boolean_t vm_compressor_thrashing_detected = FALSE;
39236c6e
A		 1810#endif /* CONFIG_JETSAM */
1811
1812static boolean_t
1813compressor_needs_to_swap(void)
1814{
1815 boolean_t should_swap = FALSE;
1816
3e170ce0
A		 1817 if (vm_swapout_ripe_segments == TRUE && c_overage_swapped_count < c_overage_swapped_limit) {
1818 c_segment_t c_seg;
1819 clock_sec_t now;
1820 clock_sec_t age;
1821 clock_nsec_t nsec;
1822
1823 clock_get_system_nanotime(&now, &nsec);
1824 age = 0;
1825
1826 lck_mtx_lock_spin_always(c_list_lock);
1827
1828 if ( !queue_empty(&c_age_list_head)) {
1829 c_seg = (c_segment_t) queue_first(&c_age_list_head);
1830
1831 age = now - c_seg->c_creation_ts;
1832 }
1833 lck_mtx_unlock_always(c_list_lock);
1834
1835 if (age >= vm_ripe_target_age)
1836 return (TRUE);
1837 }
39037602 1838 if (VM_CONFIG_SWAP_IS_ACTIVE) {
39236c6e
A		 1839 if (COMPRESSOR_NEEDS_TO_SWAP()) {
1840 return (TRUE);
1841 }
1842 if (VM_PAGE_Q_THROTTLED(&vm_pageout_queue_external) && vm_page_anonymous_count < (vm_page_inactive_count / 20)) {
1843 return (TRUE);
1844 }
3e170ce0 1845 if (vm_page_free_count < (vm_page_free_reserved - (COMPRESSOR_FREE_RESERVED_LIMIT * 2)))
39236c6e
A		 1846 return (TRUE);
1847 }
1848 compute_swapout_target_age();
1849
1850 if (swapout_target_age) {
1851 c_segment_t c_seg;
1852
1853 lck_mtx_lock_spin_always(c_list_lock);
1854
1855 if (!queue_empty(&c_age_list_head)) {
1856
1857 c_seg = (c_segment_t) queue_first(&c_age_list_head);
1858
fe8ab488 1859 if (c_seg->c_creation_ts > swapout_target_age)
39236c6e
A		 1860 swapout_target_age = 0;
1861 }
1862 lck_mtx_unlock_always(c_list_lock);
1863 }
fe8ab488
A		 1864#if CONFIG_PHANTOM_CACHE
1865 if (vm_phantom_cache_check_pressure())
1866 should_swap = TRUE;
1867#endif
1868 if (swapout_target_age)
1869 should_swap = TRUE;
39236c6e 1870
fe8ab488 1871#if CONFIG_JETSAM
5ba3f43e 1872 if (should_swap || vm_compressor_low_on_space() == TRUE) {
3e170ce0
A		 1873
1874 if (vm_compressor_thrashing_detected == FALSE) {
1875 vm_compressor_thrashing_detected = TRUE;
fe8ab488 1876
5ba3f43e 1877 if (swapout_target_age || vm_compressor_low_on_space() == TRUE) {
3e170ce0
A		 1878 memorystatus_kill_on_VM_thrashing(TRUE /* async */);
1879 compressor_thrashing_induced_jetsam++;
1880 } else {
1881 memorystatus_kill_on_FC_thrashing(TRUE /* async */);
1882 filecache_thrashing_induced_jetsam++;
39236c6e 1883 }
3e170ce0
A		 1884 }
1885 /*
1886 * let the jetsam take precedence over
1887 * any major compactions we might have
1888 * been able to do... otherwise we run
1889 * the risk of doing major compactions
1890 * on segments we're about to free up
1891 * due to the jetsam activity.
1892 */
1893 should_swap = FALSE;
39236c6e 1894 }
fe8ab488 1895
3e170ce0
A		 1896#endif /* CONFIG_JETSAM */
1897
1898 if (should_swap == FALSE) {
1899 /*
39037602 1900 * vm_compressor_needs_to_major_compact returns true only if we're
3e170ce0
A		 1901 * about to run out of available compressor segments... in this
1902 * case, we absolutely need to run a major compaction even if
1903 * we've just kicked off a jetsam or we don't otherwise need to
1904 * swap... terminating objects releases
1905 * pages back to the uncompressed cache, but does not guarantee
1906 * that we will free up even a single compression segment
1907 */
39037602 1908 should_swap = vm_compressor_needs_to_major_compact();
3e170ce0
A		 1909 }
1910
1911 /*
1912 * returning TRUE when swap_supported == FALSE
39236c6e
A		 1913 * will cause the major compaction engine to
1914 * run, but will not trigger any swapping...
1915 * segments that have been major compacted
3e170ce0 1916 * will be moved to the majorcompact queue
39236c6e
A		 1917 */
1918 return (should_swap);
1919}
1920
fe8ab488
A		 1921#if CONFIG_JETSAM
1922/*
1923 * This function is called from the jetsam thread after killing something to
1924 * mitigate thrashing.
1925 *
1926 * We need to restart our thrashing detection heuristics since memory pressure
1927 * has potentially changed significantly, and we don't want to detect on old
1928 * data from before the jetsam.
1929 */
1930void
1931vm_thrashing_jetsam_done(void)
39236c6e 1932{
fe8ab488 1933 vm_compressor_thrashing_detected = FALSE;
39236c6e 1934
fe8ab488
A		 1935 /* Were we compressor-thrashing or filecache-thrashing? */
1936 if (swapout_target_age) {
1937 swapout_target_age = 0;
1938 compressor_need_sample_reset = TRUE;
39236c6e 1939 }
fe8ab488
A		 1940#if CONFIG_PHANTOM_CACHE
1941 else {
1942 vm_phantom_cache_restart_sample();
1943 }
1944#endif
39236c6e 1945}
fe8ab488 1946#endif /* CONFIG_JETSAM */
39236c6e
A		 1947
1948uint32_t vm_wake_compactor_swapper_calls = 0;
39037602
A		 1949uint32_t vm_run_compactor_already_running = 0;
1950uint32_t vm_run_compactor_empty_minor_q = 0;
1951uint32_t vm_run_compactor_did_compact = 0;
1952uint32_t vm_run_compactor_waited = 0;
1953
1954void
1955vm_run_compactor(void)
1956{
1957 if (c_segment_count == 0)
1958 return;
1959
1960 lck_mtx_lock_spin_always(c_list_lock);
1961
1962 if (c_minor_count == 0) {
1963 vm_run_compactor_empty_minor_q++;
1964
1965 lck_mtx_unlock_always(c_list_lock);
1966 return;
1967 }
1968 if (compaction_swapper_running) {
1969
1970 if (vm_restricted_to_single_processor == FALSE) {
1971 vm_run_compactor_already_running++;
1972
1973 lck_mtx_unlock_always(c_list_lock);
1974 return;
1975 }
1976 vm_run_compactor_waited++;
1977
1978 assert_wait((event_t)&compaction_swapper_running, THREAD_UNINT);
1979
1980 lck_mtx_unlock_always(c_list_lock);
1981
1982 thread_block(THREAD_CONTINUE_NULL);
1983
1984 return;
1985 }
1986 vm_run_compactor_did_compact++;
1987
1988 fastwake_warmup = FALSE;
1989 compaction_swapper_running = 1;
1990
1991 vm_compressor_do_delayed_compactions(FALSE);
1992
1993 compaction_swapper_running = 0;
1994
1995 lck_mtx_unlock_always(c_list_lock);
1996
1997 thread_wakeup((event_t)&compaction_swapper_running);
1998}
1999
39236c6e
A		 2000
2001void
2002vm_wake_compactor_swapper(void)
2003{
39037602 2004 if (compaction_swapper_running || compaction_swapper_awakened || c_segment_count == 0)
39236c6e
A		 2005 return;
2006
39037602 2007 if (c_minor_count || vm_compressor_needs_to_major_compact()) {
3e170ce0
A		 2008
2009 lck_mtx_lock_spin_always(c_list_lock);
2010
2011 fastwake_warmup = FALSE;
2012
39037602 2013 if (compaction_swapper_running == 0 && compaction_swapper_awakened == 0) {
3e170ce0
A		 2014
2015 vm_wake_compactor_swapper_calls++;
2016
39037602 2017 compaction_swapper_awakened = 1;
3e170ce0 2018 thread_wakeup((event_t)&c_compressor_swap_trigger);
3e170ce0
A		 2019 }
2020 lck_mtx_unlock_always(c_list_lock);
2021 }
2022}
2023
2024
2025void
2026vm_consider_swapping()
2027{
2028 c_segment_t c_seg, c_seg_next;
2029 clock_sec_t now;
2030 clock_nsec_t nsec;
2031
39037602 2032 assert(VM_CONFIG_SWAP_IS_PRESENT);
39236c6e
A		 2033
2034 lck_mtx_lock_spin_always(c_list_lock);
2035
3e170ce0 2036 compaction_swapper_abort = 1;
39236c6e 2037
3e170ce0
A		 2038 while (compaction_swapper_running) {
2039 assert_wait((event_t)&compaction_swapper_running, THREAD_UNINT);
39236c6e 2040
3e170ce0 2041 lck_mtx_unlock_always(c_list_lock);
39236c6e 2042
3e170ce0
A		 2043 thread_block(THREAD_CONTINUE_NULL);
2044
2045 lck_mtx_lock_spin_always(c_list_lock);
2046 }
2047 compaction_swapper_abort = 0;
2048 compaction_swapper_running = 1;
2049
2050 vm_swapout_ripe_segments = TRUE;
2051
2052 if (!queue_empty(&c_major_list_head)) {
2053
2054 clock_get_system_nanotime(&now, &nsec);
2055
2056 c_seg = (c_segment_t)queue_first(&c_major_list_head);
2057
2058 while (!queue_end(&c_major_list_head, (queue_entry_t)c_seg)) {
2059
2060 if (c_overage_swapped_count >= c_overage_swapped_limit)
2061 break;
2062
2063 c_seg_next = (c_segment_t) queue_next(&c_seg->c_age_list);
2064
2065 if ((now - c_seg->c_creation_ts) >= vm_ripe_target_age) {
2066
2067 lck_mtx_lock_spin_always(&c_seg->c_lock);
2068
2069 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
2070
2071 lck_mtx_unlock_always(&c_seg->c_lock);
2072 }
2073 c_seg = c_seg_next;
2074 }
39236c6e 2075 }
3e170ce0
A		 2076 vm_compressor_compact_and_swap(FALSE);
2077
2078 compaction_swapper_running = 0;
2079
2080 vm_swapout_ripe_segments = FALSE;
2081
39236c6e 2082 lck_mtx_unlock_always(c_list_lock);
39037602
A		 2083
2084 thread_wakeup((event_t)&compaction_swapper_running);
39236c6e
A		 2085}
2086
fe8ab488 2087
39236c6e
A		 2088void
2089vm_consider_waking_compactor_swapper(void)
2090{
2091 boolean_t need_wakeup = FALSE;
2092
39037602 2093 if (c_segment_count == 0)
39236c6e 2094 return;
fe8ab488 2095
39037602 2096 if (compaction_swapper_running || compaction_swapper_awakened)
3e170ce0
A		 2097 return;
2098
fe8ab488
A		 2099 if (!compaction_swapper_inited && !compaction_swapper_init_now) {
2100 compaction_swapper_init_now = 1;
2101 need_wakeup = TRUE;
2102 }
39236c6e
A		 2103
2104 if (c_minor_count && (COMPRESSOR_NEEDS_TO_MINOR_COMPACT())) {
2105
2106 need_wakeup = TRUE;
2107
2108 } else if (compressor_needs_to_swap()) {
2109
2110 need_wakeup = TRUE;
2111
2112 } else if (c_minor_count) {
2113 uint64_t total_bytes;
2114
2115 total_bytes = compressor_object->resident_page_count * PAGE_SIZE_64;
2116
2117 if ((total_bytes - compressor_bytes_used) > total_bytes / 10)
2118 need_wakeup = TRUE;
2119 }
2120 if (need_wakeup == TRUE) {
2121
2122 lck_mtx_lock_spin_always(c_list_lock);
2123
2124 fastwake_warmup = FALSE;
2125
39037602 2126 if (compaction_swapper_running == 0 && compaction_swapper_awakened == 0) {
39236c6e
A		 2127 memoryshot(VM_WAKEUP_COMPACTOR_SWAPPER, DBG_FUNC_NONE);
2128
39037602 2129 compaction_swapper_awakened = 1;
39236c6e 2130 thread_wakeup((event_t)&c_compressor_swap_trigger);
39236c6e
A		 2131 }
2132 lck_mtx_unlock_always(c_list_lock);
2133 }
2134}
2135
2136
2137#define C_SWAPOUT_LIMIT 4
2138#define DELAYED_COMPACTIONS_PER_PASS 30
2139
2140void
2141vm_compressor_do_delayed_compactions(boolean_t flush_all)
2142{
2143 c_segment_t c_seg;
2144 int number_compacted = 0;
2145 boolean_t needs_to_swap = FALSE;
2146
2147
5ba3f43e 2148#if !CONFIG_EMBEDDED
39037602 2149 LCK_MTX_ASSERT(c_list_lock, LCK_MTX_ASSERT_OWNED);
5ba3f43e 2150#endif /* !CONFIG_EMBEDDED */
39236c6e
A		 2151
2152 while (!queue_empty(&c_minor_list_head) && needs_to_swap == FALSE) {
2153
2154 c_seg = (c_segment_t)queue_first(&c_minor_list_head);
2155
2156 lck_mtx_lock_spin_always(&c_seg->c_lock);
8a3053a0 2157
fe8ab488 2158 if (c_seg->c_busy) {
8a3053a0 2159
fe8ab488
A		 2160 lck_mtx_unlock_always(c_list_lock);
2161 c_seg_wait_on_busy(c_seg);
2162 lck_mtx_lock_spin_always(c_list_lock);
8a3053a0 2163
fe8ab488
A		 2164 continue;
2165 }
2166 C_SEG_BUSY(c_seg);
39236c6e
A		 2167
2168 c_seg_do_minor_compaction_and_unlock(c_seg, TRUE, FALSE, TRUE);
2169
39037602 2170 if (VM_CONFIG_SWAP_IS_ACTIVE && (number_compacted++ > DELAYED_COMPACTIONS_PER_PASS)) {
39236c6e
A		 2171
2172 if ((flush_all == TRUE || compressor_needs_to_swap() == TRUE) && c_swapout_count < C_SWAPOUT_LIMIT)
2173 needs_to_swap = TRUE;
2174
2175 number_compacted = 0;
2176 }
2177 lck_mtx_lock_spin_always(c_list_lock);
2178 }
2179}
2180
2181
2182#define C_SEGMENT_SWAPPEDIN_AGE_LIMIT 10
2183
2184static void
2185vm_compressor_age_swapped_in_segments(boolean_t flush_all)
2186{
2187 c_segment_t c_seg;
2188 clock_sec_t now;
2189 clock_nsec_t nsec;
2190
2191 clock_get_system_nanotime(&now, &nsec);
2192
2193 while (!queue_empty(&c_swappedin_list_head)) {
2194
2195 c_seg = (c_segment_t)queue_first(&c_swappedin_list_head);
2196
2197 if (flush_all == FALSE && (now - c_seg->c_swappedin_ts) < C_SEGMENT_SWAPPEDIN_AGE_LIMIT)
2198 break;
2199
2200 lck_mtx_lock_spin_always(&c_seg->c_lock);
2201
3e170ce0 2202 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
39236c6e
A		 2203
2204 lck_mtx_unlock_always(&c_seg->c_lock);
2205 }
2206}
2207
2208
39037602
A		 2209extern int vm_num_swap_files;
2210extern int vm_num_pinned_swap_files;
2211extern int vm_swappin_enabled;
2212
2213extern unsigned int vm_swapfile_total_segs_used;
2214extern unsigned int vm_swapfile_total_segs_alloced;
2215
2216
39236c6e
A		 2217void
2218vm_compressor_flush(void)
2219{
2220 uint64_t vm_swap_put_failures_at_start;
2221 wait_result_t wait_result = 0;
2222 AbsoluteTime startTime, endTime;
2223 clock_sec_t now_sec;
2224 clock_nsec_t now_nsec;
2225 uint64_t nsec;
2226
2227 HIBLOG("vm_compressor_flush - starting\n");
2228
2229 clock_get_uptime(&startTime);
2230
2231 lck_mtx_lock_spin_always(c_list_lock);
2232
2233 fastwake_warmup = FALSE;
2234 compaction_swapper_abort = 1;
2235
2236 while (compaction_swapper_running) {
2237 assert_wait((event_t)&compaction_swapper_running, THREAD_UNINT);
2238
2239 lck_mtx_unlock_always(c_list_lock);
2240
2241 thread_block(THREAD_CONTINUE_NULL);
2242
2243 lck_mtx_lock_spin_always(c_list_lock);
2244 }
2245 compaction_swapper_abort = 0;
2246 compaction_swapper_running = 1;
2247
2248 hibernate_flushing = TRUE;
2249 hibernate_no_swapspace = FALSE;
2250 c_generation_id_flush_barrier = c_generation_id + 1000;
2251
2252 clock_get_system_nanotime(&now_sec, &now_nsec);
2253 hibernate_flushing_deadline = now_sec + HIBERNATE_FLUSHING_SECS_TO_COMPLETE;
2254
2255 vm_swap_put_failures_at_start = vm_swap_put_failures;
2256
2257 vm_compressor_compact_and_swap(TRUE);
2258
2259 while (!queue_empty(&c_swapout_list_head)) {
2260
2261 assert_wait_timeout((event_t) &compaction_swapper_running, THREAD_INTERRUPTIBLE, 5000, 1000*NSEC_PER_USEC);
2262
2263 lck_mtx_unlock_always(c_list_lock);
2264
2265 wait_result = thread_block(THREAD_CONTINUE_NULL);
2266
2267 lck_mtx_lock_spin_always(c_list_lock);
2268
2269 if (wait_result == THREAD_TIMED_OUT)
2270 break;
2271 }
2272 hibernate_flushing = FALSE;
2273 compaction_swapper_running = 0;
2274
2275 if (vm_swap_put_failures > vm_swap_put_failures_at_start)
2276 HIBLOG("vm_compressor_flush failed to clean %llu segments - vm_page_compressor_count(%d)\n",
2277 vm_swap_put_failures - vm_swap_put_failures_at_start, VM_PAGE_COMPRESSOR_COUNT);
2278
2279 lck_mtx_unlock_always(c_list_lock);
2280
39037602
A		 2281 thread_wakeup((event_t)&compaction_swapper_running);
2282
39236c6e
A		 2283 clock_get_uptime(&endTime);
2284 SUB_ABSOLUTETIME(&endTime, &startTime);
2285 absolutetime_to_nanoseconds(endTime, &nsec);
2286
39037602
A		 2287 HIBLOG("vm_compressor_flush completed - took %qd msecs - vm_num_swap_files = %d, vm_num_pinned_swap_files = %d, vm_swappin_enabled = %d\n",
2288 nsec / 1000000ULL, vm_num_swap_files, vm_num_pinned_swap_files, vm_swappin_enabled);
39236c6e
A		 2289}
2290
2291
39236c6e
A		 2292int compaction_swap_trigger_thread_awakened = 0;
2293
2294static void
2295vm_compressor_swap_trigger_thread(void)
2296{
3e170ce0
A		 2297 current_thread()->options |= TH_OPT_VMPRIV;
2298
fe8ab488
A		 2299 /*
2300 * compaction_swapper_init_now is set when the first call to
2301 * vm_consider_waking_compactor_swapper is made from
2302 * vm_pageout_scan... since this function is called upon
2303 * thread creation, we want to make sure to delay adjusting
2304 * the tuneables until we are awakened via vm_pageout_scan
2305 * so that we are at a point where the vm_swapfile_open will
2306 * be operating on the correct directory (in case the default
2307 * of /var/vm/ is overridden by the dymanic_pager
2308 */
39037602
A		 2309 if (compaction_swapper_init_now) {
2310 vm_compaction_swapper_do_init();
39236c6e 2311
3e170ce0
A		 2312 if (vm_restricted_to_single_processor == TRUE)
2313 thread_vm_bind_group_add();
5ba3f43e 2314 thread_set_thread_name(current_thread(), "VM_cswap_trigger");
39037602 2315 compaction_swapper_init_now = 0;
fe8ab488 2316 }
39236c6e
A		 2317 lck_mtx_lock_spin_always(c_list_lock);
2318
2319 compaction_swap_trigger_thread_awakened++;
39037602 2320 compaction_swapper_awakened = 0;
39236c6e 2321
39037602 2322 if (compaction_swapper_running == 0) {
39236c6e 2323
39037602
A		 2324 compaction_swapper_running = 1;
2325
2326 vm_compressor_compact_and_swap(FALSE);
2327
2328 compaction_swapper_running = 0;
2329 }
39236c6e
A		 2330 assert_wait((event_t)&c_compressor_swap_trigger, THREAD_UNINT);
2331
39037602
A		 2332 if (compaction_swapper_running == 0)
2333 thread_wakeup((event_t)&compaction_swapper_running);
39236c6e
A		 2334
2335 lck_mtx_unlock_always(c_list_lock);
2336
2337 thread_block((thread_continue_t)vm_compressor_swap_trigger_thread);
2338
2339 /* NOTREACHED */
2340}
2341
2342
2343void
2344vm_compressor_record_warmup_start(void)
2345{
2346 c_segment_t c_seg;
2347
2348 lck_mtx_lock_spin_always(c_list_lock);
2349
8a3053a0
A		 2350 if (first_c_segment_to_warm_generation_id == 0) {
2351 if (!queue_empty(&c_age_list_head)) {
39236c6e 2352
8a3053a0 2353 c_seg = (c_segment_t)queue_last(&c_age_list_head);
39236c6e 2354
8a3053a0
A		 2355 first_c_segment_to_warm_generation_id = c_seg->c_generation_id;
2356 } else
2357 first_c_segment_to_warm_generation_id = 0;
39236c6e 2358
8a3053a0
A		 2359 fastwake_recording_in_progress = TRUE;
2360 }
39236c6e
A		 2361 lck_mtx_unlock_always(c_list_lock);
2362}
2363
2364
2365void
2366vm_compressor_record_warmup_end(void)
2367{
2368 c_segment_t c_seg;
2369
2370 lck_mtx_lock_spin_always(c_list_lock);
2371
8a3053a0 2372 if (fastwake_recording_in_progress == TRUE) {
39236c6e 2373
8a3053a0 2374 if (!queue_empty(&c_age_list_head)) {
39236c6e 2375
8a3053a0
A		 2376 c_seg = (c_segment_t)queue_last(&c_age_list_head);
2377
2378 last_c_segment_to_warm_generation_id = c_seg->c_generation_id;
2379 } else
2380 last_c_segment_to_warm_generation_id = first_c_segment_to_warm_generation_id;
39236c6e 2381
8a3053a0 2382 fastwake_recording_in_progress = FALSE;
39236c6e 2383
8a3053a0
A		 2384 HIBLOG("vm_compressor_record_warmup (%qd - %qd)\n", first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id);
2385 }
39236c6e
A		 2386 lck_mtx_unlock_always(c_list_lock);
2387}
2388
2389
39037602 2390#define DELAY_TRIM_ON_WAKE_SECS 25
39236c6e
A		 2391
2392void
8a3053a0 2393vm_compressor_delay_trim(void)
39236c6e 2394{
8a3053a0 2395 clock_sec_t sec;
39236c6e
A		 2396 clock_nsec_t nsec;
2397
2398 clock_get_system_nanotime(&sec, &nsec);
2399 dont_trim_until_ts = sec + DELAY_TRIM_ON_WAKE_SECS;
8a3053a0 2400}
39236c6e 2401
39236c6e 2402
8a3053a0
A		 2403void
2404vm_compressor_do_warmup(void)
2405{
39236c6e
A		 2406 lck_mtx_lock_spin_always(c_list_lock);
2407
8a3053a0
A		 2408 if (first_c_segment_to_warm_generation_id == last_c_segment_to_warm_generation_id) {
2409 first_c_segment_to_warm_generation_id = last_c_segment_to_warm_generation_id = 0;
2410
2411 lck_mtx_unlock_always(c_list_lock);
2412 return;
2413 }
2414
39037602 2415 if (compaction_swapper_running == 0 && compaction_swapper_awakened == 0) {
39236c6e
A		 2416
2417 fastwake_warmup = TRUE;
39037602
A		 2418
2419 compaction_swapper_awakened = 1;
39236c6e
A		 2420 thread_wakeup((event_t)&c_compressor_swap_trigger);
2421 }
2422 lck_mtx_unlock_always(c_list_lock);
2423}
2424
d190cdc3
A		 2425void
2426do_fastwake_warmup_all(void)
2427{
2428
2429 lck_mtx_lock_spin_always(c_list_lock);
2430
2431 if (queue_empty(&c_swappedout_list_head) && queue_empty(&c_swappedout_sparse_list_head)) {
2432
2433 lck_mtx_unlock_always(c_list_lock);
2434 return;
2435 }
2436
2437 fastwake_warmup = TRUE;
2438
2439 do_fastwake_warmup(&c_swappedout_list_head, TRUE);
2440
2441 do_fastwake_warmup(&c_swappedout_sparse_list_head, TRUE);
2442
2443 fastwake_warmup = FALSE;
2444
2445 lck_mtx_unlock_always(c_list_lock);
2446
2447}
39236c6e
A		 2448
2449void
d190cdc3 2450do_fastwake_warmup(queue_head_t *c_queue, boolean_t consider_all_cseg)
39236c6e 2451{
39236c6e 2452 c_segment_t c_seg = NULL;
8a3053a0
A		 2453 AbsoluteTime startTime, endTime;
2454 uint64_t nsec;
2455
2456
2457 HIBLOG("vm_compressor_fastwake_warmup (%qd - %qd) - starting\n", first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id);
2458
2459 clock_get_uptime(&startTime);
39236c6e
A		 2460
2461 lck_mtx_unlock_always(c_list_lock);
2462
39037602
A		 2463 proc_set_thread_policy(current_thread(),
2464 TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER2);
39236c6e
A		 2465
2466 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2467
2468 lck_mtx_lock_spin_always(c_list_lock);
2469
d190cdc3 2470 while (!queue_empty(c_queue) && fastwake_warmup == TRUE) {
39236c6e 2471
d190cdc3 2472 c_seg = (c_segment_t) queue_first(c_queue);
39236c6e 2473
d190cdc3
A		 2474 if (consider_all_cseg == FALSE) {
2475 if (c_seg->c_generation_id < first_c_segment_to_warm_generation_id ||
2476 c_seg->c_generation_id > last_c_segment_to_warm_generation_id)
2477 break;
39236c6e 2478
d190cdc3
A		 2479 if (vm_page_free_count < (AVAILABLE_MEMORY / 4))
2480 break;
2481 }
4bd07ac2 2482
39236c6e
A		 2483 lck_mtx_lock_spin_always(&c_seg->c_lock);
2484 lck_mtx_unlock_always(c_list_lock);
2485
8a3053a0
A		 2486 if (c_seg->c_busy) {
2487 PAGE_REPLACEMENT_DISALLOWED(FALSE);
39236c6e 2488 c_seg_wait_on_busy(c_seg);
8a3053a0
A		 2489 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2490 } else {
39037602
A		 2491 if (c_seg_swapin(c_seg, TRUE, FALSE) == 0)
2492 lck_mtx_unlock_always(&c_seg->c_lock);
39236c6e 2493 c_segment_warmup_count++;
8a3053a0
A		 2494
2495 PAGE_REPLACEMENT_DISALLOWED(FALSE);
39236c6e 2496 vm_pageout_io_throttle();
8a3053a0 2497 PAGE_REPLACEMENT_DISALLOWED(TRUE);
39236c6e
A		 2498 }
2499 lck_mtx_lock_spin_always(c_list_lock);
2500 }
2501 lck_mtx_unlock_always(c_list_lock);
2502
2503 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2504
39037602
A		 2505 proc_set_thread_policy(current_thread(),
2506 TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER0);
39236c6e 2507
8a3053a0
A		 2508 clock_get_uptime(&endTime);
2509 SUB_ABSOLUTETIME(&endTime, &startTime);
2510 absolutetime_to_nanoseconds(endTime, &nsec);
2511
2512 HIBLOG("vm_compressor_fastwake_warmup completed - took %qd msecs\n", nsec / 1000000ULL);
2513
39236c6e 2514 lck_mtx_lock_spin_always(c_list_lock);
8a3053a0 2515
d190cdc3
A		 2516 if (consider_all_cseg == FALSE) {
2517 first_c_segment_to_warm_generation_id = last_c_segment_to_warm_generation_id = 0;
2518 }
39236c6e
A		 2519}
2520
2521
2522void
2523vm_compressor_compact_and_swap(boolean_t flush_all)
2524{
2525 c_segment_t c_seg, c_seg_next;
2526 boolean_t keep_compacting;
3e170ce0
A		 2527 clock_sec_t now;
2528 clock_nsec_t nsec;
39236c6e
A		 2529
2530
2531 if (fastwake_warmup == TRUE) {
2532 uint64_t starting_warmup_count;
2533
2534 starting_warmup_count = c_segment_warmup_count;
2535
2536 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 11) | DBG_FUNC_START, c_segment_warmup_count,
2537 first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id, 0, 0);
d190cdc3 2538 do_fastwake_warmup(&c_swappedout_list_head, FALSE);
39236c6e
A		 2539 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 11) | DBG_FUNC_END, c_segment_warmup_count, c_segment_warmup_count - starting_warmup_count, 0, 0, 0);
2540
2541 fastwake_warmup = FALSE;
2542 }
2543
8a3053a0
A		 2544 /*
2545 * it's possible for the c_age_list_head to be empty if we
2546 * hit our limits for growing the compressor pool and we subsequently
2547 * hibernated... on the next hibernation we could see the queue as
2548 * empty and not proceeed even though we have a bunch of segments on
2549 * the swapped in queue that need to be dealt with.
2550 */
2551 vm_compressor_do_delayed_compactions(flush_all);
2552
2553 vm_compressor_age_swapped_in_segments(flush_all);
2554
3e170ce0
A		 2555 /*
2556 * we only need to grab the timestamp once per
2557 * invocation of this function since the
2558 * timescale we're interested in is measured
2559 * in days
2560 */
2561 clock_get_system_nanotime(&now, &nsec);
8a3053a0 2562
39236c6e
A		 2563 while (!queue_empty(&c_age_list_head) && compaction_swapper_abort == 0) {
2564
2565 if (hibernate_flushing == TRUE) {
2566 clock_sec_t sec;
39236c6e
A		 2567
2568 if (hibernate_should_abort()) {
2569 HIBLOG("vm_compressor_flush - hibernate_should_abort returned TRUE\n");
2570 break;
2571 }
2572 if (hibernate_no_swapspace == TRUE) {
2573 HIBLOG("vm_compressor_flush - out of swap space\n");
2574 break;
2575 }
39037602
A		 2576 if (vm_swap_files_pinned() == FALSE) {
2577 HIBLOG("vm_compressor_flush - unpinned swap files\n");
2578 break;
2579 }
2580 if (hibernate_in_progress_with_pinned_swap == TRUE &&
2581 (vm_swapfile_total_segs_alloced == vm_swapfile_total_segs_used)) {
2582 HIBLOG("vm_compressor_flush - out of pinned swap space\n");
2583 break;
2584 }
39236c6e
A		 2585 clock_get_system_nanotime(&sec, &nsec);
2586
2587 if (sec > hibernate_flushing_deadline) {
2588 HIBLOG("vm_compressor_flush - failed to finish before deadline\n");
2589 break;
2590 }
2591 }
2592 if (c_swapout_count >= C_SWAPOUT_LIMIT) {
2593
2594 assert_wait_timeout((event_t) &compaction_swapper_running, THREAD_INTERRUPTIBLE, 100, 1000*NSEC_PER_USEC);
2595
2596 lck_mtx_unlock_always(c_list_lock);
2597
2598 thread_block(THREAD_CONTINUE_NULL);
2599
2600 lck_mtx_lock_spin_always(c_list_lock);
2601 }
2602 /*
2603 * Minor compactions
2604 */
2605 vm_compressor_do_delayed_compactions(flush_all);
2606
2607 vm_compressor_age_swapped_in_segments(flush_all);
2608
2609 if (c_swapout_count >= C_SWAPOUT_LIMIT) {
2610 /*
2611 * we timed out on the above thread_block
2612 * let's loop around and try again
2613 * the timeout allows us to continue
2614 * to do minor compactions to make
2615 * more memory available
2616 */
2617 continue;
2618 }
2619
2620 /*
2621 * Swap out segments?
2622 */
2623 if (flush_all == FALSE) {
2624 boolean_t needs_to_swap;
2625
2626 lck_mtx_unlock_always(c_list_lock);
2627
2628 needs_to_swap = compressor_needs_to_swap();
2629
5ba3f43e 2630#if !CONFIG_EMBEDDED
3e170ce0
A		 2631 if (needs_to_swap == TRUE && vm_swap_low_on_space())
2632 vm_compressor_take_paging_space_action();
5ba3f43e 2633#endif /* !CONFIG_EMBEDDED */
3e170ce0 2634
39236c6e
A		 2635 lck_mtx_lock_spin_always(c_list_lock);
2636
2637 if (needs_to_swap == FALSE)
2638 break;
2639 }
2640 if (queue_empty(&c_age_list_head))
2641 break;
2642 c_seg = (c_segment_t) queue_first(&c_age_list_head);
2643
3e170ce0 2644 assert(c_seg->c_state == C_ON_AGE_Q);
39236c6e 2645
3e170ce0 2646 if (flush_all == TRUE && c_seg->c_generation_id > c_generation_id_flush_barrier)
39236c6e 2647 break;
3e170ce0 2648
39236c6e
A		 2649 lck_mtx_lock_spin_always(&c_seg->c_lock);
2650
2651 if (c_seg->c_busy) {
2652
2653 lck_mtx_unlock_always(c_list_lock);
2654 c_seg_wait_on_busy(c_seg);
2655 lck_mtx_lock_spin_always(c_list_lock);
2656
2657 continue;
2658 }
fe8ab488 2659 C_SEG_BUSY(c_seg);
39236c6e
A		 2660
2661 if (c_seg_do_minor_compaction_and_unlock(c_seg, FALSE, TRUE, TRUE)) {
2662 /*
2663 * found an empty c_segment and freed it
2664 * so go grab the next guy in the queue
2665 */
3e170ce0 2666 c_seg_major_compact_stats.count_of_freed_segs++;
39236c6e
A		 2667 continue;
2668 }
2669 /*
2670 * Major compaction
2671 */
2672 keep_compacting = TRUE;
2673
2674 while (keep_compacting == TRUE) {
2675
2676 assert(c_seg->c_busy);
2677
2678 /* look for another segment to consolidate */
2679
2680 c_seg_next = (c_segment_t) queue_next(&c_seg->c_age_list);
2681
2682 if (queue_end(&c_age_list_head, (queue_entry_t)c_seg_next))
2683 break;
2684
3e170ce0
A		 2685 assert(c_seg_next->c_state == C_ON_AGE_Q);
2686
39236c6e
A		 2687 if (c_seg_major_compact_ok(c_seg, c_seg_next) == FALSE)
2688 break;
2689
2690 lck_mtx_lock_spin_always(&c_seg_next->c_lock);
2691
2692 if (c_seg_next->c_busy) {
2693
2694 lck_mtx_unlock_always(c_list_lock);
2695 c_seg_wait_on_busy(c_seg_next);
2696 lck_mtx_lock_spin_always(c_list_lock);
2697
2698 continue;
2699 }
2700 /* grab that segment */
fe8ab488 2701 C_SEG_BUSY(c_seg_next);
39236c6e
A		 2702
2703 if (c_seg_do_minor_compaction_and_unlock(c_seg_next, FALSE, TRUE, TRUE)) {
2704 /*
2705 * found an empty c_segment and freed it
2706 * so we can't continue to use c_seg_next
2707 */
3e170ce0 2708 c_seg_major_compact_stats.count_of_freed_segs++;
39236c6e
A		 2709 continue;
2710 }
2711
2712 /* unlock the list ... */
2713 lck_mtx_unlock_always(c_list_lock);
2714
2715 /* do the major compaction */
2716
2717 keep_compacting = c_seg_major_compact(c_seg, c_seg_next);
2718
2719 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2720
2721 lck_mtx_lock_spin_always(&c_seg_next->c_lock);
2722 /*
2723 * run a minor compaction on the donor segment
2724 * since we pulled at least some of it's
2725 * data into our target... if we've emptied
2726 * it, now is a good time to free it which
2727 * c_seg_minor_compaction_and_unlock also takes care of
2728 *
2729 * by passing TRUE, we ask for c_busy to be cleared
2730 * and c_wanted to be taken care of
2731 */
3e170ce0
A		 2732 if (c_seg_minor_compaction_and_unlock(c_seg_next, TRUE))
2733 c_seg_major_compact_stats.count_of_freed_segs++;
39236c6e
A		 2734
2735 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2736
2737 /* relock the list */
2738 lck_mtx_lock_spin_always(c_list_lock);
2739
2740 } /* major compaction */
2741
39236c6e
A		 2742 lck_mtx_lock_spin_always(&c_seg->c_lock);
2743
2744 assert(c_seg->c_busy);
39236c6e
A		 2745 assert(!c_seg->c_on_minorcompact_q);
2746
39037602 2747 if (VM_CONFIG_SWAP_IS_ACTIVE) {
3e170ce0
A		 2748 /*
2749 * This mode of putting a generic c_seg on the swapout list is
39037602 2750 * only supported when we have general swapping enabled
3e170ce0 2751 */
39037602
A		 2752 c_seg_switch_state(c_seg, C_ON_SWAPOUT_Q, FALSE);
2753 } else {
2754 if ((vm_swapout_ripe_segments == TRUE && c_overage_swapped_count < c_overage_swapped_limit)) {
2755
2756 assert(VM_CONFIG_SWAP_IS_PRESENT);
2757 /*
2758 * we are running compressor sweeps with swap-behind
2759 * make sure the c_seg has aged enough before swapping it
2760 * out...
2761 */
2762 if ((now - c_seg->c_creation_ts) >= vm_ripe_target_age) {
2763 c_seg->c_overage_swap = TRUE;
2764 c_overage_swapped_count++;
2765 c_seg_switch_state(c_seg, C_ON_SWAPOUT_Q, FALSE);
3e170ce0
A		 2766 }
2767 }
2768 }
2769 if (c_seg->c_state == C_ON_AGE_Q) {
2770 /*
2771 * this c_seg didn't get moved to the swapout queue
2772 * so we need to move it out of the way...
2773 * we just did a major compaction on it so put it
2774 * on that queue
2775 */
2776 c_seg_switch_state(c_seg, C_ON_MAJORCOMPACT_Q, FALSE);
39236c6e 2777 } else {
3e170ce0
A		 2778 c_seg_major_compact_stats.wasted_space_in_swapouts += C_SEG_BUFSIZE - c_seg->c_bytes_used;
2779 c_seg_major_compact_stats.count_of_swapouts++;
39236c6e
A		 2780 }
2781 C_SEG_WAKEUP_DONE(c_seg);
2782
2783 lck_mtx_unlock_always(&c_seg->c_lock);
2784
2785 if (c_swapout_count) {
2786 lck_mtx_unlock_always(c_list_lock);
2787
2788 thread_wakeup((event_t)&c_swapout_list_head);
2789
2790 lck_mtx_lock_spin_always(c_list_lock);
2791 }
2792 }
2793}
2794
2795
2796static c_segment_t
2797c_seg_allocate(c_segment_t *current_chead)
2798{
39236c6e 2799 c_segment_t c_seg;
3e170ce0
A		 2800 int min_needed;
2801 int size_to_populate;
2802
5ba3f43e 2803#if !CONFIG_EMBEDDED
3e170ce0
A		 2804 if (vm_compressor_low_on_space())
2805 vm_compressor_take_paging_space_action();
5ba3f43e 2806#endif /* !CONFIG_EMBEDDED */
39236c6e
A		 2807
2808 if ( (c_seg = *current_chead) == NULL ) {
2809 uint32_t c_segno;
2810
39236c6e
A		 2811 lck_mtx_lock_spin_always(c_list_lock);
2812
2813 while (c_segments_busy == TRUE) {
2814 assert_wait((event_t) (&c_segments_busy), THREAD_UNINT);
2815
2816 lck_mtx_unlock_always(c_list_lock);
2817
2818 thread_block(THREAD_CONTINUE_NULL);
2819
2820 lck_mtx_lock_spin_always(c_list_lock);
2821 }
2822 if (c_free_segno_head == (uint32_t)-1) {
3e170ce0 2823 uint32_t c_segments_available_new;
39236c6e
A		 2824
2825 if (c_segments_available >= c_segments_limit || c_segment_pages_compressed >= c_segment_pages_compressed_limit) {
2826 lck_mtx_unlock_always(c_list_lock);
2827
39236c6e
A		 2828 return (NULL);
2829 }
2830 c_segments_busy = TRUE;
2831 lck_mtx_unlock_always(c_list_lock);
2832
39037602 2833 kernel_memory_populate(compressor_map, (vm_offset_t)c_segments_next_page,
3e170ce0 2834 PAGE_SIZE, KMA_KOBJECT, VM_KERN_MEMORY_COMPRESSOR);
39236c6e
A		 2835 c_segments_next_page += PAGE_SIZE;
2836
3e170ce0
A		 2837 c_segments_available_new = c_segments_available + C_SEGMENTS_PER_PAGE;
2838
2839 if (c_segments_available_new > c_segments_limit)
2840 c_segments_available_new = c_segments_limit;
2841
2842 for (c_segno = c_segments_available + 1; c_segno < c_segments_available_new; c_segno++)
39236c6e
A		 2843 c_segments[c_segno - 1].c_segno = c_segno;
2844
2845 lck_mtx_lock_spin_always(c_list_lock);
2846
2847 c_segments[c_segno - 1].c_segno = c_free_segno_head;
2848 c_free_segno_head = c_segments_available;
3e170ce0 2849 c_segments_available = c_segments_available_new;
39236c6e
A		 2850
2851 c_segments_busy = FALSE;
2852 thread_wakeup((event_t) (&c_segments_busy));
2853 }
2854 c_segno = c_free_segno_head;
3e170ce0
A		 2855 assert(c_segno >= 0 && c_segno < c_segments_limit);
2856
39037602 2857 c_free_segno_head = (uint32_t)c_segments[c_segno].c_segno;
39236c6e 2858
3e170ce0
A		 2859 /*
2860 * do the rest of the bookkeeping now while we're still behind
2861 * the list lock and grab our generation id now into a local
2862 * so that we can install it once we have the c_seg allocated
2863 */
2864 c_segment_count++;
2865 if (c_segment_count > c_segment_count_max)
2866 c_segment_count_max = c_segment_count;
2867
39236c6e
A		 2868 lck_mtx_unlock_always(c_list_lock);
2869
2870 c_seg = (c_segment_t)zalloc(compressor_segment_zone);
2871 bzero((char *)c_seg, sizeof(struct c_segment));
2872
3e170ce0 2873 c_seg->c_store.c_buffer = (int32_t *)C_SEG_BUFFER_ADDRESS(c_segno);
39236c6e 2874
39236c6e 2875 lck_mtx_init(&c_seg->c_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
39236c6e 2876
3e170ce0
A		 2877 c_seg->c_state = C_IS_EMPTY;
2878 c_seg->c_firstemptyslot = C_SLOT_MAX_INDEX;
39236c6e 2879 c_seg->c_mysegno = c_segno;
39236c6e
A		 2880
2881 lck_mtx_lock_spin_always(c_list_lock);
3e170ce0
A		 2882 c_empty_count++;
2883 c_seg_switch_state(c_seg, C_IS_FILLING, FALSE);
39236c6e 2884 c_segments[c_segno].c_seg = c_seg;
39037602 2885 assert(c_segments[c_segno].c_segno > c_segments_available);
39236c6e
A		 2886 lck_mtx_unlock_always(c_list_lock);
2887
39236c6e 2888 *current_chead = c_seg;
39037602
A		 2889
2890#if DEVELOPMENT || DEBUG
2891 C_SEG_MAKE_WRITEABLE(c_seg);
2892#endif
2893
39236c6e 2894 }
3e170ce0
A		 2895 c_seg_alloc_nextslot(c_seg);
2896
2897 size_to_populate = C_SEG_ALLOCSIZE - C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset);
2898
2899 if (size_to_populate) {
2900
2901 min_needed = PAGE_SIZE + (C_SEG_ALLOCSIZE - C_SEG_BUFSIZE);
39236c6e 2902
3e170ce0 2903 if (C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset - c_seg->c_nextoffset) < (unsigned) min_needed) {
39236c6e 2904
3e170ce0
A		 2905 if (size_to_populate > C_SEG_MAX_POPULATE_SIZE)
2906 size_to_populate = C_SEG_MAX_POPULATE_SIZE;
5ba3f43e 2907 vm_compressor_pages_grabbed += size_to_populate / PAGE_SIZE;
39236c6e 2908
39037602 2909 kernel_memory_populate(compressor_map,
3e170ce0
A		 2910 (vm_offset_t) &c_seg->c_store.c_buffer[c_seg->c_populated_offset],
2911 size_to_populate,
2912 KMA_COMPRESSOR,
2913 VM_KERN_MEMORY_COMPRESSOR);
2914 } else
2915 size_to_populate = 0;
39236c6e 2916 }
39236c6e
A		 2917 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2918
2919 lck_mtx_lock_spin_always(&c_seg->c_lock);
2920
3e170ce0
A		 2921 if (size_to_populate)
2922 c_seg->c_populated_offset += C_SEG_BYTES_TO_OFFSET(size_to_populate);
2923
39236c6e
A		 2924 return (c_seg);
2925}
2926
2927
39236c6e
A		 2928static void
2929c_current_seg_filled(c_segment_t c_seg, c_segment_t *current_chead)
2930{
2931 uint32_t unused_bytes;
2932 uint32_t offset_to_depopulate;
3e170ce0
A		 2933 int new_state = C_ON_AGE_Q;
2934 clock_sec_t sec;
2935 clock_nsec_t nsec;
39236c6e
A		 2936
2937 unused_bytes = trunc_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset - c_seg->c_nextoffset));
2938
5ba3f43e
A		 2939#ifndef _OPEN_SOURCE
2940 /* TODO: The HW codec can generate, lazily, a '2nd page not mapped'
2941 * exception. So on such a platform, or platforms where we're confident
2942 * the codec does not require a buffer page to absorb trailing writes,
2943 * we can create an unmapped hole at the tail of the segment, rather
2944 * than a populated mapping. This will also guarantee that the codec
2945 * does not overwrite valid data past the edge of the segment and
2946 * thus eliminate the depopulation overhead.
2947 */
2948#endif
39236c6e 2949 if (unused_bytes) {
39236c6e
A		 2950 offset_to_depopulate = C_SEG_BYTES_TO_OFFSET(round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_nextoffset)));
2951
2952 /*
2953 * release the extra physical page(s) at the end of the segment
2954 */
2955 lck_mtx_unlock_always(&c_seg->c_lock);
2956
2957 kernel_memory_depopulate(
39037602 2958 compressor_map,
39236c6e
A		 2959 (vm_offset_t) &c_seg->c_store.c_buffer[offset_to_depopulate],
2960 unused_bytes,
2961 KMA_COMPRESSOR);
2962
2963 lck_mtx_lock_spin_always(&c_seg->c_lock);
2964
2965 c_seg->c_populated_offset = offset_to_depopulate;
2966 }
3e170ce0 2967 assert(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset) <= C_SEG_BUFSIZE);
39236c6e 2968
39037602
A		 2969#if DEVELOPMENT || DEBUG
2970 {
2971 boolean_t c_seg_was_busy = FALSE;
2972
2973 if ( !c_seg->c_busy)
2974 C_SEG_BUSY(c_seg);
2975 else
2976 c_seg_was_busy = TRUE;
2977
2978 lck_mtx_unlock_always(&c_seg->c_lock);
2979
2980 C_SEG_WRITE_PROTECT(c_seg);
2981
2982 lck_mtx_lock_spin_always(&c_seg->c_lock);
2983
2984 if (c_seg_was_busy == FALSE)
2985 C_SEG_WAKEUP_DONE(c_seg);
2986 }
2987#endif
2988
3e170ce0 2989#if CONFIG_FREEZE
39037602
A		 2990 if (current_chead == (c_segment_t*)&freezer_chead &&
2991 VM_CONFIG_SWAP_IS_PRESENT &&
2992 VM_CONFIG_FREEZER_SWAP_IS_ACTIVE &&
3e170ce0
A		 2993 c_freezer_swapout_count < VM_MAX_FREEZER_CSEG_SWAP_COUNT) {
2994 new_state = C_ON_SWAPOUT_Q;
2995 }
2996#endif /* CONFIG_FREEZE */
39236c6e 2997
3e170ce0
A		 2998 clock_get_system_nanotime(&sec, &nsec);
2999 c_seg->c_creation_ts = (uint32_t)sec;
3000
3001 lck_mtx_lock_spin_always(c_list_lock);
3002
a39ff7e2
A		 3003 c_seg->c_generation_id = c_generation_id++;
3004 c_seg_switch_state(c_seg, new_state, FALSE);
3005
3e170ce0
A		 3006#if CONFIG_FREEZE
3007 if (c_seg->c_state == C_ON_SWAPOUT_Q)
3008 c_freezer_swapout_count++;
3009#endif /* CONFIG_FREEZE */
3010
39037602
A		 3011 if (c_seg->c_state == C_ON_AGE_Q && C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE)
3012 c_seg_need_delayed_compaction(c_seg, TRUE);
3013
3e170ce0
A		 3014 lck_mtx_unlock_always(c_list_lock);
3015
3016#if CONFIG_FREEZE
3017 if (c_seg->c_state == C_ON_SWAPOUT_Q)
3018 thread_wakeup((event_t)&c_swapout_list_head);
3019#endif /* CONFIG_FREEZE */
3020
39236c6e
A		 3021 *current_chead = NULL;
3022}
3023
39037602 3024
39236c6e
A		 3025/*
3026 * returns with c_seg locked
3027 */
3028void
39037602 3029c_seg_swapin_requeue(c_segment_t c_seg, boolean_t has_data, boolean_t minor_compact_ok, boolean_t age_on_swapin_q)
39236c6e
A		 3030{
3031 clock_sec_t sec;
3032 clock_nsec_t nsec;
3033
3034 clock_get_system_nanotime(&sec, &nsec);
3035
3036 lck_mtx_lock_spin_always(c_list_lock);
3037 lck_mtx_lock_spin_always(&c_seg->c_lock);
3038
39037602
A		 3039 assert(c_seg->c_busy_swapping);
3040 assert(c_seg->c_busy);
3041
3e170ce0
A		 3042 c_seg->c_busy_swapping = 0;
3043
3044 if (c_seg->c_overage_swap == TRUE) {
3045 c_overage_swapped_count--;
3046 c_seg->c_overage_swap = FALSE;
3047 }
3048 if (has_data == TRUE) {
39037602
A		 3049 if (age_on_swapin_q == TRUE)
3050 c_seg_switch_state(c_seg, C_ON_SWAPPEDIN_Q, FALSE);
3051 else
3052 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
3053
3054 if (minor_compact_ok == TRUE && !c_seg->c_on_minorcompact_q && C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE)
3055 c_seg_need_delayed_compaction(c_seg, TRUE);
39236c6e 3056 } else {
3e170ce0
A		 3057 c_seg->c_store.c_buffer = (int32_t*) NULL;
3058 c_seg->c_populated_offset = C_SEG_BYTES_TO_OFFSET(0);
39236c6e 3059
3e170ce0 3060 c_seg_switch_state(c_seg, C_ON_BAD_Q, FALSE);
39236c6e 3061 }
39236c6e 3062 c_seg->c_swappedin_ts = (uint32_t)sec;
39236c6e
A		 3063
3064 lck_mtx_unlock_always(c_list_lock);
3065}
3066
3067
3068
3069/*
39037602 3070 * c_seg has to be locked and is returned locked if the c_seg isn't freed
39236c6e 3071 * PAGE_REPLACMENT_DISALLOWED has to be TRUE on entry and is returned TRUE
39037602 3072 * c_seg_swapin returns 1 if the c_seg was freed, 0 otherwise
39236c6e
A		 3073 */
3074
39037602
A		 3075int
3076c_seg_swapin(c_segment_t c_seg, boolean_t force_minor_compaction, boolean_t age_on_swapin_q)
39236c6e
A		 3077{
3078 vm_offset_t addr = 0;
3079 uint32_t io_size = 0;
3080 uint64_t f_offset;
3081
3e170ce0
A		 3082 assert(C_SEG_IS_ONDISK(c_seg));
3083
39236c6e 3084#if !CHECKSUM_THE_SWAP
3e170ce0 3085 c_seg_trim_tail(c_seg);
39236c6e
A		 3086#endif
3087 io_size = round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset));
3088 f_offset = c_seg->c_store.c_swap_handle;
fe8ab488
A		 3089
3090 C_SEG_BUSY(c_seg);
3e170ce0 3091 c_seg->c_busy_swapping = 1;
ecc0ceb4
A		 3092
3093 /*
3094 * This thread is likely going to block for I/O.
3095 * Make sure it is ready to run when the I/O completes because
3096 * it needs to clear the busy bit on the c_seg so that other
3097 * waiting threads can make progress too. To do that, boost
3098 * the rwlock_count so that the priority is boosted.
3099 */
3100 set_thread_rwlock_boost();
39236c6e 3101 lck_mtx_unlock_always(&c_seg->c_lock);
39236c6e 3102
3e170ce0 3103 PAGE_REPLACEMENT_DISALLOWED(FALSE);
39236c6e 3104
3e170ce0 3105 addr = (vm_offset_t)C_SEG_BUFFER_ADDRESS(c_seg->c_mysegno);
39037602 3106 c_seg->c_store.c_buffer = (int32_t*) addr;
39236c6e 3107
39037602 3108 kernel_memory_populate(compressor_map, addr, io_size, KMA_COMPRESSOR, VM_KERN_MEMORY_COMPRESSOR);
39236c6e 3109
39037602 3110 if (vm_swap_get(c_seg, f_offset, io_size) != KERN_SUCCESS) {
3e170ce0 3111 PAGE_REPLACEMENT_DISALLOWED(TRUE);
39236c6e 3112
39037602
A		 3113 kernel_memory_depopulate(compressor_map, addr, io_size, KMA_COMPRESSOR);
3114
3115 c_seg_swapin_requeue(c_seg, FALSE, TRUE, age_on_swapin_q);
3e170ce0 3116 } else {
fe8ab488 3117#if ENCRYPTED_SWAP
3e170ce0 3118 vm_swap_decrypt(c_seg);
fe8ab488 3119#endif /* ENCRYPTED_SWAP */
39236c6e
A		 3120
3121#if CHECKSUM_THE_SWAP
3e170ce0
A		 3122 if (c_seg->cseg_swap_size != io_size)
3123 panic("swapin size doesn't match swapout size");
39236c6e 3124
5ba3f43e 3125 if (c_seg->cseg_hash != vmc_hash((char*) c_seg->c_store.c_buffer, (int)io_size)) {
3e170ce0
A		 3126 panic("c_seg_swapin - Swap hash mismatch\n");
3127 }
39236c6e
A		 3128#endif /* CHECKSUM_THE_SWAP */
3129
3e170ce0 3130 PAGE_REPLACEMENT_DISALLOWED(TRUE);
39236c6e 3131
39037602
A		 3132 c_seg_swapin_requeue(c_seg, TRUE, force_minor_compaction == TRUE ? FALSE : TRUE, age_on_swapin_q);
3133
3134 OSAddAtomic64(c_seg->c_bytes_used, &compressor_bytes_used);
3135
3e170ce0 3136 if (force_minor_compaction == TRUE) {
39037602
A		 3137 if (c_seg_minor_compaction_and_unlock(c_seg, FALSE)) {
3138 /*
5ba3f43e
A		 3139 * c_seg was completely empty so it was freed,
3140 * so be careful not to reference it again
3141 *
39037602
A		 3142 * Drop the rwlock_count so that the thread priority
3143 * is returned back to where it is supposed to be.
3144 */
3145 clear_thread_rwlock_boost();
3146 return (1);
3147 }
3148
3e170ce0 3149 lck_mtx_lock_spin_always(&c_seg->c_lock);
3e170ce0 3150 }
3e170ce0
A		 3151 }
3152 C_SEG_WAKEUP_DONE(c_seg);
ecc0ceb4
A		 3153
3154 /*
3155 * Drop the rwlock_count so that the thread priority
3156 * is returned back to where it is supposed to be.
3157 */
3158 clear_thread_rwlock_boost();
39037602
A		 3159
3160 return (0);
3e170ce0
A		 3161}
3162
3163
3164static void
3165c_segment_sv_hash_drop_ref(int hash_indx)
3166{
3167 struct c_sv_hash_entry o_sv_he, n_sv_he;
3168
3169 while (1) {
3170
3171 o_sv_he.he_record = c_segment_sv_hash_table[hash_indx].he_record;
3172
3173 n_sv_he.he_ref = o_sv_he.he_ref - 1;
3174 n_sv_he.he_data = o_sv_he.he_data;
3175
3176 if (OSCompareAndSwap64((UInt64)o_sv_he.he_record, (UInt64)n_sv_he.he_record, (UInt64 *) &c_segment_sv_hash_table[hash_indx].he_record) == TRUE) {
3177 if (n_sv_he.he_ref == 0)
3178 OSAddAtomic(-1, &c_segment_svp_in_hash);
3179 break;
3180 }
3181 }
3182}
3183
3184
3185static int
3186c_segment_sv_hash_insert(uint32_t data)
3187{
3188 int hash_sindx;
3189 int misses;
3190 struct c_sv_hash_entry o_sv_he, n_sv_he;
3191 boolean_t got_ref = FALSE;
3192
3193 if (data == 0)
3194 OSAddAtomic(1, &c_segment_svp_zero_compressions);
3195 else
3196 OSAddAtomic(1, &c_segment_svp_nonzero_compressions);
3197
3198 hash_sindx = data & C_SV_HASH_MASK;
3199
3200 for (misses = 0; misses < C_SV_HASH_MAX_MISS; misses++)
3201 {
3202 o_sv_he.he_record = c_segment_sv_hash_table[hash_sindx].he_record;
3203
3204 while (o_sv_he.he_data == data || o_sv_he.he_ref == 0) {
3205 n_sv_he.he_ref = o_sv_he.he_ref + 1;
3206 n_sv_he.he_data = data;
3207
3208 if (OSCompareAndSwap64((UInt64)o_sv_he.he_record, (UInt64)n_sv_he.he_record, (UInt64 *) &c_segment_sv_hash_table[hash_sindx].he_record) == TRUE) {
3209 if (n_sv_he.he_ref == 1)
3210 OSAddAtomic(1, &c_segment_svp_in_hash);
3211 got_ref = TRUE;
3212 break;
39236c6e 3213 }
3e170ce0 3214 o_sv_he.he_record = c_segment_sv_hash_table[hash_sindx].he_record;
39236c6e 3215 }
3e170ce0
A		 3216 if (got_ref == TRUE)
3217 break;
3218 hash_sindx++;
3219
3220 if (hash_sindx == C_SV_HASH_SIZE)
3221 hash_sindx = 0;
39236c6e 3222 }
3e170ce0
A		 3223 if (got_ref == FALSE)
3224 return(-1);
39236c6e 3225
3e170ce0
A		 3226 return (hash_sindx);
3227}
3228
3229
3230#if RECORD_THE_COMPRESSED_DATA
3231
3232static void
3233c_compressed_record_data(char *src, int c_size)
3234{
3235 if ((c_compressed_record_cptr + c_size + 4) >= c_compressed_record_ebuf)
3236 panic("c_compressed_record_cptr >= c_compressed_record_ebuf");
3237
3238 *(int *)((void *)c_compressed_record_cptr) = c_size;
3239
3240 c_compressed_record_cptr += 4;
3241
3242 memcpy(c_compressed_record_cptr, src, c_size);
3243 c_compressed_record_cptr += c_size;
39236c6e 3244}
3e170ce0 3245#endif
39236c6e
A		 3246
3247
3248static int
3249c_compress_page(char *src, c_slot_mapping_t slot_ptr, c_segment_t *current_chead, char *scratch_buf)
3250{
3251 int c_size;
04b8595b 3252 int c_rounded_size = 0;
39236c6e
A		 3253 int max_csize;
3254 c_slot_t cs;
3255 c_segment_t c_seg;
3256
3257 KERNEL_DEBUG(0xe0400000 | DBG_FUNC_START, *current_chead, 0, 0, 0, 0);
3258retry:
5ba3f43e 3259 if ((c_seg = c_seg_allocate(current_chead)) == NULL) {
39236c6e 3260 return (1);
5ba3f43e 3261 }
39236c6e
A		 3262 /*
3263 * returns with c_seg lock held
3e170ce0
A		 3264 * and PAGE_REPLACEMENT_DISALLOWED(TRUE)...
3265 * c_nextslot has been allocated and
3266 * c_store.c_buffer populated
39236c6e 3267 */
3e170ce0
A		 3268 assert(c_seg->c_state == C_IS_FILLING);
3269
39236c6e
A		 3270 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_seg->c_nextslot);
3271
3272 cs->c_packed_ptr = C_SLOT_PACK_PTR(slot_ptr);
fe8ab488
A		 3273 assert(slot_ptr == (c_slot_mapping_t)C_SLOT_UNPACK_PTR(cs));
3274
39236c6e
A		 3275 cs->c_offset = c_seg->c_nextoffset;
3276
3277 max_csize = C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES((int32_t)cs->c_offset);
3278
3279 if (max_csize > PAGE_SIZE)
3280 max_csize = PAGE_SIZE;
3281
39236c6e 3282#if CHECKSUM_THE_DATA
5ba3f43e 3283 cs->c_hash_data = vmc_hash(src, PAGE_SIZE);
39236c6e 3284#endif
5ba3f43e
A		 3285 boolean_t incomp_copy = FALSE;
3286 int max_csize_adj = (max_csize - 4);
39236c6e 3287
39037602 3288 if (vm_compressor_algorithm() != VM_COMPRESSOR_DEFAULT_CODEC) {
5ba3f43e
A		 3289#if defined(__arm__) || defined(__arm64__)
3290 uint16_t ccodec = CINVALID;
3291
3292 if (max_csize >= C_SEG_OFFSET_ALIGNMENT_BOUNDARY) {
3293 c_size = metacompressor((const uint8_t *) src,
3294 (uint8_t *) &c_seg->c_store.c_buffer[cs->c_offset],
3295 max_csize_adj, &ccodec,
3296 scratch_buf, &incomp_copy);
3297#if C_SEG_OFFSET_ALIGNMENT_BOUNDARY > 4
3298 if (c_size > max_csize_adj) {
3299 c_size = -1;
3300 }
3301#endif
3302 } else {
3303 c_size = -1;
3304 }
3305 assert(ccodec == CCWK || ccodec == CCLZ4);
3306 cs->c_codec = ccodec;
3307#endif
39037602 3308 } else {
5ba3f43e
A		 3309#if defined(__arm__) || defined(__arm64__)
3310 cs->c_codec = CCWK;
3311#endif
3312#if defined(__arm64__)
3313 __unreachable_ok_push
3314 if (PAGE_SIZE == 4096)
3315 c_size = WKdm_compress_4k((WK_word *)(uintptr_t)src, (WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
3316 (WK_word *)(uintptr_t)scratch_buf, max_csize_adj);
3317 else {
3318 c_size = WKdm_compress_16k((WK_word *)(uintptr_t)src, (WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
3319 (WK_word *)(uintptr_t)scratch_buf, max_csize_adj);
3320 }
3321 __unreachable_ok_pop
3322#else
3e170ce0 3323 c_size = WKdm_compress_new((const WK_word *)(uintptr_t)src, (WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
5ba3f43e
A		 3324 (WK_word *)(uintptr_t)scratch_buf, max_csize_adj);
3325#endif
39037602 3326 }
5ba3f43e
A		 3327 assertf(((c_size <= max_csize_adj) && (c_size >= -1)),
3328 "c_size invalid (%d, %d), cur compressions: %d", c_size, max_csize_adj, c_segment_pages_compressed);
39236c6e
A		 3329
3330 if (c_size == -1) {
39236c6e
A		 3331 if (max_csize < PAGE_SIZE) {
3332 c_current_seg_filled(c_seg, current_chead);
3e170ce0 3333 assert(*current_chead == NULL);
39236c6e 3334
3e170ce0 3335 lck_mtx_unlock_always(&c_seg->c_lock);
5ba3f43e
A		 3336 /* TODO: it may be worth requiring codecs to distinguish
3337 * between incompressible inputs and failures due to
3338 * budget exhaustion.
3339 */
3e170ce0 3340 PAGE_REPLACEMENT_DISALLOWED(FALSE);
39236c6e
A		 3341 goto retry;
3342 }
3343 c_size = PAGE_SIZE;
3344
5ba3f43e
A		 3345 if (incomp_copy == FALSE) {
3346 memcpy(&c_seg->c_store.c_buffer[cs->c_offset], src, c_size);
3347 }
3e170ce0
A		 3348
3349 OSAddAtomic(1, &c_segment_noncompressible_pages);
3350
3351 } else if (c_size == 0) {
3352 int hash_index;
3353
3354 /*
3355 * special case - this is a page completely full of a single 32 bit value
3356 */
3357 hash_index = c_segment_sv_hash_insert(*(uint32_t *)(uintptr_t)src);
3358
3359 if (hash_index != -1) {
3360 slot_ptr->s_cindx = hash_index;
3361 slot_ptr->s_cseg = C_SV_CSEG_ID;
3362
3363 OSAddAtomic(1, &c_segment_svp_hash_succeeded);
3364#if RECORD_THE_COMPRESSED_DATA
3365 c_compressed_record_data(src, 4);
3366#endif
3367 goto sv_compression;
3368 }
3369 c_size = 4;
3370
3371 memcpy(&c_seg->c_store.c_buffer[cs->c_offset], src, c_size);
3372
3373 OSAddAtomic(1, &c_segment_svp_hash_failed);
39236c6e 3374 }
3e170ce0
A		 3375
3376#if RECORD_THE_COMPRESSED_DATA
3377 c_compressed_record_data((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size);
3378#endif
39236c6e 3379#if CHECKSUM_THE_COMPRESSED_DATA
5ba3f43e
A		 3380 cs->c_hash_compressed_data = vmc_hash((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size);
3381#endif
3382#if POPCOUNT_THE_COMPRESSED_DATA
3383 cs->c_pop_cdata = vmc_pop((uintptr_t) &c_seg->c_store.c_buffer[cs->c_offset], c_size);
39236c6e
A		 3384#endif
3385 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
3386
3387 PACK_C_SIZE(cs, c_size);
3388 c_seg->c_bytes_used += c_rounded_size;
3389 c_seg->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
5ba3f43e 3390 c_seg->c_slots_used++;
39236c6e
A		 3391
3392 slot_ptr->s_cindx = c_seg->c_nextslot++;
3393 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
3394 slot_ptr->s_cseg = c_seg->c_mysegno + 1;
3395
3e170ce0
A		 3396sv_compression:
3397 if (c_seg->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg->c_nextslot >= C_SLOT_MAX_INDEX) {
39236c6e 3398 c_current_seg_filled(c_seg, current_chead);
3e170ce0
A		 3399 assert(*current_chead == NULL);
3400 }
3401 lck_mtx_unlock_always(&c_seg->c_lock);
39236c6e
A		 3402
3403 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3404
3e170ce0
A		 3405#if RECORD_THE_COMPRESSED_DATA
3406 if ((c_compressed_record_cptr - c_compressed_record_sbuf) >= C_SEG_ALLOCSIZE) {
3407 c_compressed_record_write(c_compressed_record_sbuf, (int)(c_compressed_record_cptr - c_compressed_record_sbuf));
3408 c_compressed_record_cptr = c_compressed_record_sbuf;
3409 }
3410#endif
3411 if (c_size) {
3412 OSAddAtomic64(c_size, &c_segment_compressed_bytes);
3413 OSAddAtomic64(c_rounded_size, &compressor_bytes_used);
3414 }
39236c6e 3415 OSAddAtomic64(PAGE_SIZE, &c_segment_input_bytes);
39236c6e
A		 3416
3417 OSAddAtomic(1, &c_segment_pages_compressed);
3418 OSAddAtomic(1, &sample_period_compression_count);
3419
3420 KERNEL_DEBUG(0xe0400000 | DBG_FUNC_END, *current_chead, c_size, c_segment_input_bytes, c_segment_compressed_bytes, 0);
3421
39236c6e
A		 3422 return (0);
3423}
3424
39037602
A		 3425static inline void sv_decompress(int32_t *ddst, int32_t pattern) {
3426#if __x86_64__
3427 memset_word(ddst, pattern, PAGE_SIZE / sizeof(int32_t));
3428#else
3429 size_t i;
3430
3431 /* Unroll the pattern fill loop 4x to encourage the
3432 * compiler to emit NEON stores, cf.
3433 * <rdar://problem/25839866> Loop autovectorization
3434 * anomalies.
3435 * We use separate loops for each PAGE_SIZE
3436 * to allow the autovectorizer to engage, as PAGE_SIZE
3437 * is currently not a constant.
3438 */
3439
5ba3f43e 3440 __unreachable_ok_push
39037602
A		 3441 if (PAGE_SIZE == 4096) {
3442 for (i = 0; i < (4096U / sizeof(int32_t)); i += 4) {
3443 *ddst++ = pattern;
3444 *ddst++ = pattern;
3445 *ddst++ = pattern;
3446 *ddst++ = pattern;
3447 }
3448 } else {
3449 assert(PAGE_SIZE == 16384);
3450 for (i = 0; i < (int)(16384U / sizeof(int32_t)); i += 4) {
3451 *ddst++ = pattern;
3452 *ddst++ = pattern;
3453 *ddst++ = pattern;
3454 *ddst++ = pattern;
3455 }
3456 }
5ba3f43e 3457 __unreachable_ok_pop
39037602
A		 3458#endif
3459}
39236c6e
A		 3460
3461static int
3462c_decompress_page(char *dst, volatile c_slot_mapping_t slot_ptr, int flags, int *zeroslot)
3463{
3464 c_slot_t cs;
3465 c_segment_t c_seg;
39037602 3466 uint32_t c_segno;
39236c6e
A		 3467 int c_indx;
3468 int c_rounded_size;
3469 uint32_t c_size;
3470 int retval = 0;
39236c6e
A		 3471 boolean_t need_unlock = TRUE;
3472 boolean_t consider_defragmenting = FALSE;
3e170ce0
A		 3473 boolean_t kdp_mode = FALSE;
3474
39037602 3475 if (__improbable(flags & C_KDP)) {
3e170ce0
A		 3476 if (not_in_kdp) {
3477 panic("C_KDP passed to decompress page from outside of debugger context");
3478 }
3479
5ba3f43e 3480 assert((flags & C_KEEP) == C_KEEP);
3e170ce0
A		 3481 assert((flags & C_DONT_BLOCK) == C_DONT_BLOCK);
3482
3483 if ((flags & (C_DONT_BLOCK | C_KEEP)) != (C_DONT_BLOCK | C_KEEP)) {
3484 return (-2);
3485 }
3486
3487 kdp_mode = TRUE;
39037602 3488 *zeroslot = 0;
3e170ce0 3489 }
39236c6e
A		 3490
3491ReTry:
39037602 3492 if (__probable(!kdp_mode)) {
3e170ce0
A		 3493 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3494 } else {
3495 if (kdp_lck_rw_lock_is_acquired_exclusive(&c_master_lock)) {
3496 return (-2);
3497 }
3498 }
fe8ab488 3499
39236c6e 3500#if HIBERNATION
fe8ab488
A		 3501 /*
3502 * if hibernation is enabled, it indicates (via a call
3503 * to 'vm_decompressor_lock' that no further
3504 * decompressions are allowed once it reaches
3505 * the point of flushing all of the currently dirty
3506 * anonymous memory through the compressor and out
3507 * to disk... in this state we allow freeing of compressed
3508 * pages and must honor the C_DONT_BLOCK case
3509 */
5ba3f43e 3510 if (__improbable(dst && decompressions_blocked == TRUE)) {
fe8ab488
A		 3511 if (flags & C_DONT_BLOCK) {
3512
39037602 3513 if (__probable(!kdp_mode)) {
3e170ce0
A		 3514 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3515 }
fe8ab488
A		 3516
3517 *zeroslot = 0;
3518 return (-2);
39236c6e 3519 }
fe8ab488
A		 3520 /*
3521 * it's safe to atomically assert and block behind the
3522 * lock held in shared mode because "decompressions_blocked" is
3523 * only set and cleared and the thread_wakeup done when the lock
3524 * is held exclusively
3525 */
3526 assert_wait((event_t)&decompressions_blocked, THREAD_UNINT);
3527
3528 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3529
3530 thread_block(THREAD_CONTINUE_NULL);
3531
3532 goto ReTry;
39236c6e
A		 3533 }
3534#endif
39236c6e 3535 /* s_cseg is actually "segno+1" */
39037602
A		 3536 c_segno = slot_ptr->s_cseg - 1;
3537
3538 if (__improbable(c_segno >= c_segments_available))
3539 panic("c_decompress_page: c_segno %d >= c_segments_available %d, slot_ptr(%p), slot_data(%x)",
3540 c_segno, c_segments_available, slot_ptr, *(int *)((void *)slot_ptr));
3541
3542 if (__improbable(c_segments[c_segno].c_segno < c_segments_available))
3543 panic("c_decompress_page: c_segno %d is free, slot_ptr(%p), slot_data(%x)",
3544 c_segno, slot_ptr, *(int *)((void *)slot_ptr));
39236c6e 3545
39037602
A		 3546 c_seg = c_segments[c_segno].c_seg;
3547
3548 if (__probable(!kdp_mode)) {
3e170ce0
A		 3549 lck_mtx_lock_spin_always(&c_seg->c_lock);
3550 } else {
3551 if (kdp_lck_mtx_lock_spin_is_acquired(&c_seg->c_lock)) {
3552 return (-2);
3553 }
3554 }
39236c6e 3555
3e170ce0 3556 assert(c_seg->c_state != C_IS_EMPTY && c_seg->c_state != C_IS_FREE);
39236c6e 3557
39037602
A		 3558 if (dst == NULL && c_seg->c_busy_swapping) {
3559 assert(c_seg->c_busy);
3560
3561 goto bypass_busy_check;
3562 }
3e170ce0
A		 3563 if (flags & C_DONT_BLOCK) {
3564 if (c_seg->c_busy || (C_SEG_IS_ONDISK(c_seg) && dst)) {
39236c6e
A		 3565 *zeroslot = 0;
3566
3e170ce0 3567 retval = -2;
39236c6e
A		 3568 goto done;
3569 }
3570 }
3571 if (c_seg->c_busy) {
3572
3573 PAGE_REPLACEMENT_DISALLOWED(FALSE);
fe8ab488 3574
39236c6e
A		 3575 c_seg_wait_on_busy(c_seg);
3576
3577 goto ReTry;
3578 }
39037602
A		 3579bypass_busy_check:
3580
39236c6e
A		 3581 c_indx = slot_ptr->s_cindx;
3582
39037602
A		 3583 if (__improbable(c_indx >= c_seg->c_nextslot))
3584 panic("c_decompress_page: c_indx %d >= c_nextslot %d, c_seg(%p), slot_ptr(%p), slot_data(%x)",
3585 c_indx, c_seg->c_nextslot, c_seg, slot_ptr, *(int *)((void *)slot_ptr));
3586
39236c6e
A		 3587 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
3588
3589 c_size = UNPACK_C_SIZE(cs);
3590
39037602
A		 3591 if (__improbable(c_size == 0))
3592 panic("c_decompress_page: c_size == 0, c_seg(%p), slot_ptr(%p), slot_data(%x)",
3593 c_seg, slot_ptr, *(int *)((void *)slot_ptr));
3594
39236c6e
A		 3595 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
3596
3597 if (dst) {
3598 uint32_t age_of_cseg;
3599 clock_sec_t cur_ts_sec;
3600 clock_nsec_t cur_ts_nsec;
3601
3e170ce0
A		 3602 if (C_SEG_IS_ONDISK(c_seg)) {
3603 assert(kdp_mode == FALSE);
39037602
A		 3604 retval = c_seg_swapin(c_seg, FALSE, TRUE);
3605 assert(retval == 0);
3e170ce0
A		 3606
3607 retval = 1;
39236c6e 3608 }
3e170ce0
A		 3609 if (c_seg->c_state == C_ON_BAD_Q) {
3610 assert(c_seg->c_store.c_buffer == NULL);
5ba3f43e 3611 *zeroslot = 0;
3e170ce0
A		 3612
3613 retval = -1;
5ba3f43e
A		 3614 goto done;
3615 }
3616
3617#if POPCOUNT_THE_COMPRESSED_DATA
3618 unsigned csvpop;
3619 uintptr_t csvaddr = (uintptr_t) &c_seg->c_store.c_buffer[cs->c_offset];
3620 if (cs->c_pop_cdata != (csvpop = vmc_pop(csvaddr, c_size))) {
3621 panic("Compressed data popcount doesn't match original, bit distance: %d %p (phys: %p) %p %p 0x%llx 0x%x 0x%x 0x%x", (csvpop - cs->c_pop_cdata), (void *)csvaddr, (void *) kvtophys(csvaddr), c_seg, cs, cs->c_offset, c_size, csvpop, cs->c_pop_cdata);
39236c6e 3622 }
5ba3f43e
A		 3623#endif
3624
39236c6e 3625#if CHECKSUM_THE_COMPRESSED_DATA
5ba3f43e
A		 3626 unsigned csvhash;
3627 if (cs->c_hash_compressed_data != (csvhash = vmc_hash((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size))) {
3628 panic("Compressed data doesn't match original %p %p %u %u %u", c_seg, cs, c_size, cs->c_hash_compressed_data, csvhash);
3629 }
39236c6e
A		 3630#endif
3631 if (c_rounded_size == PAGE_SIZE) {
3632 /*
3633 * page wasn't compressible... just copy it out
3634 */
3635 memcpy(dst, &c_seg->c_store.c_buffer[cs->c_offset], PAGE_SIZE);
3e170ce0
A		 3636 } else if (c_size == 4) {
3637 int32_t data;
3638 int32_t *dptr;
3639
3640 /*
3641 * page was populated with a single value
3642 * that didn't fit into our fast hash
3643 * so we packed it in as a single non-compressed value
3644 * that we need to populate the page with
3645 */
3646 dptr = (int32_t *)(uintptr_t)dst;
3647 data = *(int32_t *)(&c_seg->c_store.c_buffer[cs->c_offset]);
39037602 3648 sv_decompress(dptr, data);
39236c6e
A		 3649 } else {
3650 uint32_t my_cpu_no;
3651 char *scratch_buf;
3652
39037602 3653 if (__probable(!kdp_mode)) {
3e170ce0
A		 3654 /*
3655 * we're behind the c_seg lock held in spin mode
3656 * which means pre-emption is disabled... therefore
3657 * the following sequence is atomic and safe
3658 */
3659 my_cpu_no = cpu_number();
39236c6e 3660
3e170ce0 3661 assert(my_cpu_no < compressor_cpus);
39236c6e 3662
39037602 3663 scratch_buf = &compressor_scratch_bufs[my_cpu_no * vm_compressor_get_decode_scratch_size()];
3e170ce0
A		 3664 } else {
3665 scratch_buf = kdp_compressor_scratch_buf;
3666 }
39037602
A		 3667
3668 if (vm_compressor_algorithm() != VM_COMPRESSOR_DEFAULT_CODEC) {
5ba3f43e
A		 3669#if defined(__arm__) || defined(__arm64__)
3670 uint16_t c_codec = cs->c_codec;
3671 metadecompressor((const uint8_t *) &c_seg->c_store.c_buffer[cs->c_offset],
3672 (uint8_t *)dst, c_size, c_codec, (void *)scratch_buf);
3673#endif
39037602 3674 } else {
5ba3f43e
A		 3675#if defined(__arm64__)
3676 __unreachable_ok_push
3677 if (PAGE_SIZE == 4096)
3678 WKdm_decompress_4k((WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
3679 (WK_word *)(uintptr_t)dst, (WK_word *)(uintptr_t)scratch_buf, c_size);
3680 else {
3681 WKdm_decompress_16k((WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
3682 (WK_word *)(uintptr_t)dst, (WK_word *)(uintptr_t)scratch_buf, c_size);
3683 }
3684 __unreachable_ok_pop
3685#else
39236c6e
A		 3686 WKdm_decompress_new((WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
3687 (WK_word *)(uintptr_t)dst, (WK_word *)(uintptr_t)scratch_buf, c_size);
5ba3f43e 3688#endif
39037602 3689 }
39236c6e
A		 3690 }
3691
3692#if CHECKSUM_THE_DATA
5ba3f43e
A		 3693 if (cs->c_hash_data != vmc_hash(dst, PAGE_SIZE)) {
3694#if defined(__arm__) || defined(__arm64__)
3695 int32_t *dinput = &c_seg->c_store.c_buffer[cs->c_offset];
3696 panic("decompressed data doesn't match original cs: %p, hash: 0x%x, offset: %d, c_size: %d, c_rounded_size: %d, codec: %d, header: 0x%x 0x%x 0x%x", cs, cs->c_hash_data, cs->c_offset, c_size, c_rounded_size, cs->c_codec, *dinput, *(dinput + 1), *(dinput + 2));
3697#else
3698 panic("decompressed data doesn't match original cs: %p, hash: %d, offset: 0x%x, c_size: %d", cs, cs->c_hash_data, cs->c_offset, c_size);
3699#endif
3700 }
39236c6e 3701#endif
3e170ce0 3702 if (c_seg->c_swappedin_ts == 0 && !kdp_mode) {
39236c6e
A		 3703
3704 clock_get_system_nanotime(&cur_ts_sec, &cur_ts_nsec);
3705
3706 age_of_cseg = (uint32_t)cur_ts_sec - c_seg->c_creation_ts;
39236c6e
A		 3707 if (age_of_cseg < DECOMPRESSION_SAMPLE_MAX_AGE)
3708 OSAddAtomic(1, &age_of_decompressions_during_sample_period[age_of_cseg]);
3709 else
3710 OSAddAtomic(1, &overage_decompressions_during_sample_period);
3711
3712 OSAddAtomic(1, &sample_period_decompression_count);
3713 }
39236c6e 3714 }
39236c6e
A		 3715 if (flags & C_KEEP) {
3716 *zeroslot = 0;
3717 goto done;
3718 }
3e170ce0 3719 assert(kdp_mode == FALSE);
39037602 3720
39236c6e
A		 3721 c_seg->c_bytes_unused += c_rounded_size;
3722 c_seg->c_bytes_used -= c_rounded_size;
5ba3f43e
A		 3723
3724 assert(c_seg->c_slots_used);
3725 c_seg->c_slots_used--;
3726
39236c6e
A		 3727 PACK_C_SIZE(cs, 0);
3728
3729 if (c_indx < c_seg->c_firstemptyslot)
3730 c_seg->c_firstemptyslot = c_indx;
3731
3732 OSAddAtomic(-1, &c_segment_pages_compressed);
3733
3e170ce0 3734 if (c_seg->c_state != C_ON_BAD_Q && !(C_SEG_IS_ONDISK(c_seg))) {
39236c6e 3735 /*
3e170ce0 3736 * C_SEG_IS_ONDISK == TRUE can occur when we're doing a
39236c6e
A		 3737 * free of a compressed page (i.e. dst == NULL)
3738 */
3739 OSAddAtomic64(-c_rounded_size, &compressor_bytes_used);
3740 }
39037602
A		 3741 if (c_seg->c_busy_swapping) {
3742 /*
3743 * bypass case for c_busy_swapping...
3744 * let the swapin/swapout paths deal with putting
3745 * the c_seg on the minor compaction queue if needed
3746 */
3747 assert(c_seg->c_busy);
3748 goto done;
3749 }
3750 assert(!c_seg->c_busy);
3751
3e170ce0 3752 if (c_seg->c_state != C_IS_FILLING) {
39236c6e 3753 if (c_seg->c_bytes_used == 0) {
3e170ce0 3754 if ( !(C_SEG_IS_ONDISK(c_seg))) {
8a3053a0
A		 3755 int pages_populated;
3756
3757 pages_populated = (round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE;
3758 c_seg->c_populated_offset = C_SEG_BYTES_TO_OFFSET(0);
3759
3760 if (pages_populated) {
3e170ce0
A		 3761
3762 assert(c_seg->c_state != C_ON_BAD_Q);
8a3053a0
A		 3763 assert(c_seg->c_store.c_buffer != NULL);
3764
fe8ab488 3765 C_SEG_BUSY(c_seg);
8a3053a0
A		 3766 lck_mtx_unlock_always(&c_seg->c_lock);
3767
39037602 3768 kernel_memory_depopulate(compressor_map, (vm_offset_t) c_seg->c_store.c_buffer, pages_populated * PAGE_SIZE, KMA_COMPRESSOR);
8a3053a0
A		 3769
3770 lck_mtx_lock_spin_always(&c_seg->c_lock);
3771 C_SEG_WAKEUP_DONE(c_seg);
3772 }
39037602
A		 3773 if (!c_seg->c_on_minorcompact_q && c_seg->c_state != C_ON_SWAPOUT_Q)
3774 c_seg_need_delayed_compaction(c_seg, FALSE);
3775 } else {
3776 if (c_seg->c_state != C_ON_SWAPPEDOUTSPARSE_Q) {
8a3053a0 3777
39037602
A		 3778 c_seg_move_to_sparse_list(c_seg);
3779 consider_defragmenting = TRUE;
3780 }
3781 }
39236c6e
A		 3782 } else if (c_seg->c_on_minorcompact_q) {
3783
3e170ce0 3784 assert(c_seg->c_state != C_ON_BAD_Q);
a39ff7e2 3785 assert(!C_SEG_IS_ON_DISK_OR_SOQ(c_seg));
3e170ce0 3786
5ba3f43e 3787 if (C_SEG_SHOULD_MINORCOMPACT_NOW(c_seg)) {
39236c6e
A		 3788 c_seg_try_minor_compaction_and_unlock(c_seg);
3789 need_unlock = FALSE;
3790 }
3e170ce0 3791 } else if ( !(C_SEG_IS_ONDISK(c_seg))) {
39236c6e 3792
3e170ce0 3793 if (c_seg->c_state != C_ON_BAD_Q && c_seg->c_state != C_ON_SWAPOUT_Q && C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE) {
39037602 3794 c_seg_need_delayed_compaction(c_seg, FALSE);
39236c6e 3795 }
3e170ce0 3796 } else if (c_seg->c_state != C_ON_SWAPPEDOUTSPARSE_Q && C_SEG_ONDISK_IS_SPARSE(c_seg)) {
39236c6e
A		 3797
3798 c_seg_move_to_sparse_list(c_seg);
3799 consider_defragmenting = TRUE;
3800 }
3801 }
3802done:
39037602 3803 if (__improbable(kdp_mode)) {
3e170ce0
A		 3804 return retval;
3805 }
3806
39236c6e
A		 3807 if (need_unlock == TRUE)
3808 lck_mtx_unlock_always(&c_seg->c_lock);
3809
3810 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3811
3812 if (consider_defragmenting == TRUE)
3813 vm_swap_consider_defragmenting();
fe8ab488 3814
5ba3f43e
A		 3815#if CONFIG_EMBEDDED
3816 if ((c_minor_count && COMPRESSOR_NEEDS_TO_MINOR_COMPACT()) || vm_compressor_needs_to_major_compact())
3817 vm_wake_compactor_swapper();
3818#endif
fe8ab488 3819
39236c6e
A		 3820 return (retval);
3821}
3822
3823
3824int
3825vm_compressor_get(ppnum_t pn, int *slot, int flags)
3826{
3e170ce0 3827 c_slot_mapping_t slot_ptr;
39236c6e
A		 3828 char *dst;
3829 int zeroslot = 1;
3830 int retval;
3831
3832#if __x86_64__
3833 dst = PHYSMAP_PTOV((uint64_t)pn << (uint64_t)PAGE_SHIFT);
5ba3f43e
A		 3834#elif __arm__ || __arm64__
3835 dst = (char *) phystokv((pmap_paddr_t)pn << PAGE_SHIFT);
39236c6e
A		 3836#else
3837#error "unsupported architecture"
3838#endif
3e170ce0
A		 3839 slot_ptr = (c_slot_mapping_t)slot;
3840
3841 if (slot_ptr->s_cseg == C_SV_CSEG_ID) {
3842 int32_t data;
3843 int32_t *dptr;
3844
3845 /*
3846 * page was populated with a single value
3847 * that found a home in our hash table
3848 * grab that value from the hash and populate the page
3849 * that we need to populate the page with
3850 */
3851 dptr = (int32_t *)(uintptr_t)dst;
3852 data = c_segment_sv_hash_table[slot_ptr->s_cindx].he_data;
3853#if __x86_64__
3854 memset_word(dptr, data, PAGE_SIZE / sizeof(int32_t));
3855#else
3856 {
3857 int i;
3858
3859 for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++)
3860 *dptr++ = data;
3861 }
3862#endif
3e170ce0 3863 if ( !(flags & C_KEEP)) {
743345f9
A		 3864 c_segment_sv_hash_drop_ref(slot_ptr->s_cindx);
3865
3e170ce0
A		 3866 OSAddAtomic(-1, &c_segment_pages_compressed);
3867 *slot = 0;
3868 }
3869 if (data)
3870 OSAddAtomic(1, &c_segment_svp_nonzero_decompressions);
3871 else
3872 OSAddAtomic(1, &c_segment_svp_zero_decompressions);
3873
3874 return (0);
3875 }
3876
3877 retval = c_decompress_page(dst, slot_ptr, flags, &zeroslot);
39236c6e
A		 3878
3879 /*
3880 * zeroslot will be set to 0 by c_decompress_page if (flags & C_KEEP)
3e170ce0 3881 * or (flags & C_DONT_BLOCK) and we found 'c_busy' or 'C_SEG_IS_ONDISK' to be TRUE
39236c6e
A		 3882 */
3883 if (zeroslot) {
39236c6e
A		 3884 *slot = 0;
3885 }
3886 /*
3887 * returns 0 if we successfully decompressed a page from a segment already in memory
3888 * returns 1 if we had to first swap in the segment, before successfully decompressing the page
3889 * returns -1 if we encountered an error swapping in the segment - decompression failed
3e170ce0 3890 * returns -2 if (flags & C_DONT_BLOCK) and we found 'c_busy' or 'C_SEG_IS_ONDISK' to be true
39236c6e
A		 3891 */
3892 return (retval);
3893}
3894
3895
fe8ab488
A		 3896int
3897vm_compressor_free(int *slot, int flags)
39236c6e 3898{
3e170ce0 3899 c_slot_mapping_t slot_ptr;
39236c6e 3900 int zeroslot = 1;
fe8ab488 3901 int retval;
39236c6e 3902
fe8ab488 3903 assert(flags == 0 || flags == C_DONT_BLOCK);
39236c6e 3904
3e170ce0
A		 3905 slot_ptr = (c_slot_mapping_t)slot;
3906
3907 if (slot_ptr->s_cseg == C_SV_CSEG_ID) {
3908
3909 c_segment_sv_hash_drop_ref(slot_ptr->s_cindx);
3910 OSAddAtomic(-1, &c_segment_pages_compressed);
3911
3912 *slot = 0;
3913 return (0);
3914 }
3915 retval = c_decompress_page(NULL, slot_ptr, flags, &zeroslot);
fe8ab488
A		 3916 /*
3917 * returns 0 if we successfully freed the specified compressed page
3918 * returns -2 if (flags & C_DONT_BLOCK) and we found 'c_busy' set
3919 */
3920
3921 if (retval == 0)
3922 *slot = 0;
3e170ce0
A		 3923 else
3924 assert(retval == -2);
fe8ab488
A		 3925
3926 return (retval);
39236c6e
A		 3927}
3928
3929
3930int
3931vm_compressor_put(ppnum_t pn, int *slot, void **current_chead, char *scratch_buf)
3932{
3933 char *src;
3934 int retval;
3935
39236c6e
A		 3936#if __x86_64__
3937 src = PHYSMAP_PTOV((uint64_t)pn << (uint64_t)PAGE_SHIFT);
5ba3f43e
A		 3938#elif __arm__ || __arm64__
3939 src = (char *) phystokv((pmap_paddr_t)pn << PAGE_SHIFT);
39236c6e
A		 3940#else
3941#error "unsupported architecture"
3942#endif
5ba3f43e 3943
39236c6e
A		 3944 retval = c_compress_page(src, (c_slot_mapping_t)slot, (c_segment_t *)current_chead, scratch_buf);
3945
3946 return (retval);
3947}
fe8ab488
A		 3948
3949void
3950vm_compressor_transfer(
3951 int *dst_slot_p,
3952 int *src_slot_p)
3953{
3954 c_slot_mapping_t dst_slot, src_slot;
3955 c_segment_t c_seg;
3956 int c_indx;
3957 c_slot_t cs;
3958
fe8ab488
A		 3959 src_slot = (c_slot_mapping_t) src_slot_p;
3960
3e170ce0
A		 3961 if (src_slot->s_cseg == C_SV_CSEG_ID) {
3962 *dst_slot_p = *src_slot_p;
3963 *src_slot_p = 0;
3964 return;
3965 }
3966 dst_slot = (c_slot_mapping_t) dst_slot_p;
fe8ab488
A		 3967Retry:
3968 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3969 /* get segment for src_slot */
3970 c_seg = c_segments[src_slot->s_cseg -1].c_seg;
3971 /* lock segment */
3972 lck_mtx_lock_spin_always(&c_seg->c_lock);
3973 /* wait if it's busy */
3e170ce0 3974 if (c_seg->c_busy && !c_seg->c_busy_swapping) {
fe8ab488
A		 3975 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3976 c_seg_wait_on_busy(c_seg);
3977 goto Retry;
3978 }
3979 /* find the c_slot */
3980 c_indx = src_slot->s_cindx;
3981 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
3982 /* point the c_slot back to dst_slot instead of src_slot */
3983 cs->c_packed_ptr = C_SLOT_PACK_PTR(dst_slot);
3984 /* transfer */
3985 *dst_slot_p = *src_slot_p;
3986 *src_slot_p = 0;
3987 lck_mtx_unlock_always(&c_seg->c_lock);
3988 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3989}
3e170ce0
A		 3990
3991#if CONFIG_FREEZE
3992
3993int freezer_finished_filling = 0;
3994
3995void
3996vm_compressor_finished_filling(
3997 void **current_chead)
3998{
3999 c_segment_t c_seg;
4000
4001 if ((c_seg = *(c_segment_t *)current_chead) == NULL)
4002 return;
4003
4004 assert(c_seg->c_state == C_IS_FILLING);
4005
4006 lck_mtx_lock_spin_always(&c_seg->c_lock);
4007
4008 c_current_seg_filled(c_seg, (c_segment_t *)current_chead);
4009
4010 lck_mtx_unlock_always(&c_seg->c_lock);
4011
4012 freezer_finished_filling++;
4013}
4014
4015
4016/*
4017 * This routine is used to transfer the compressed chunks from
4018 * the c_seg/cindx pointed to by slot_p into a new c_seg headed
4019 * by the current_chead and a new cindx within that c_seg.
4020 *
4021 * Currently, this routine is only used by the "freezer backed by
4022 * compressor with swap" mode to create a series of c_segs that
4023 * only contain compressed data belonging to one task. So, we
4024 * move a task's previously compressed data into a set of new
4025 * c_segs which will also hold the task's yet to be compressed data.
4026 */
4027
4028kern_return_t
4029vm_compressor_relocate(
4030 void **current_chead,
4031 int *slot_p)
4032{
4033 c_slot_mapping_t slot_ptr;
4034 c_slot_mapping_t src_slot;
4035 uint32_t c_rounded_size;
4036 uint32_t c_size;
4037 uint16_t dst_slot;
4038 c_slot_t c_dst;
4039 c_slot_t c_src;
4040 int c_indx;
4041 c_segment_t c_seg_dst = NULL;
4042 c_segment_t c_seg_src = NULL;
4043 kern_return_t kr = KERN_SUCCESS;
4044
4045
4046 src_slot = (c_slot_mapping_t) slot_p;
4047
4048 if (src_slot->s_cseg == C_SV_CSEG_ID) {
4049 /*
4050 * no need to relocate... this is a page full of a single
4051 * value which is hashed to a single entry not contained
4052 * in a c_segment_t
4053 */
4054 return (kr);
4055 }
4056
4057Relookup_dst:
4058 c_seg_dst = c_seg_allocate((c_segment_t *)current_chead);
4059 /*
4060 * returns with c_seg lock held
4061 * and PAGE_REPLACEMENT_DISALLOWED(TRUE)...
4062 * c_nextslot has been allocated and
4063 * c_store.c_buffer populated
4064 */
4065 if (c_seg_dst == NULL) {
4066 /*
4067 * Out of compression segments?
4068 */
4069 kr = KERN_RESOURCE_SHORTAGE;
4070 goto out;
4071 }
4072
4073 assert(c_seg_dst->c_busy == 0);
4074
4075 C_SEG_BUSY(c_seg_dst);
4076
4077 dst_slot = c_seg_dst->c_nextslot;
4078
4079 lck_mtx_unlock_always(&c_seg_dst->c_lock);
4080
4081Relookup_src:
4082 c_seg_src = c_segments[src_slot->s_cseg - 1].c_seg;
4083
4084 assert(c_seg_dst != c_seg_src);
4085
4086 lck_mtx_lock_spin_always(&c_seg_src->c_lock);
4087
4088 if (C_SEG_IS_ONDISK(c_seg_src)) {
4089
4090 /*
4091 * A "thaw" can mark a process as eligible for
4092 * another freeze cycle without bringing any of
4093 * its swapped out c_segs back from disk (because
4094 * that is done on-demand).
4095 *
4096 * If the src c_seg we find for our pre-compressed
4097 * data is already on-disk, then we are dealing
4098 * with an app's data that is already packed and
4099 * swapped out. Don't do anything.
4100 */
4101
4102 PAGE_REPLACEMENT_DISALLOWED(FALSE);
4103
4104 lck_mtx_unlock_always(&c_seg_src->c_lock);
4105
4106 c_seg_src = NULL;
4107
4108 goto out;
4109 }
4110
4111 if (c_seg_src->c_busy) {
4112
4113 PAGE_REPLACEMENT_DISALLOWED(FALSE);
4114 c_seg_wait_on_busy(c_seg_src);
4115
4116 c_seg_src = NULL;
4117
4118 PAGE_REPLACEMENT_DISALLOWED(TRUE);
4119
4120 goto Relookup_src;
4121 }
4122
4123 C_SEG_BUSY(c_seg_src);
4124
4125 lck_mtx_unlock_always(&c_seg_src->c_lock);
4126
4127 PAGE_REPLACEMENT_DISALLOWED(FALSE);
4128
4129 /* find the c_slot */
4130 c_indx = src_slot->s_cindx;
4131
4132 c_src = C_SEG_SLOT_FROM_INDEX(c_seg_src, c_indx);
4133
4134 c_size = UNPACK_C_SIZE(c_src);
4135
4136 assert(c_size);
4137
4138 if (c_size > (uint32_t)(C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES((int32_t)c_seg_dst->c_nextoffset))) {
4139 /*
4140 * This segment is full. We need a new one.
4141 */
4142
4143 PAGE_REPLACEMENT_DISALLOWED(TRUE);
4144
4145 lck_mtx_lock_spin_always(&c_seg_src->c_lock);
4146 C_SEG_WAKEUP_DONE(c_seg_src);
4147 lck_mtx_unlock_always(&c_seg_src->c_lock);
4148
4149 c_seg_src = NULL;
4150
4151 lck_mtx_lock_spin_always(&c_seg_dst->c_lock);
4152
4153 assert(c_seg_dst->c_busy);
4154 assert(c_seg_dst->c_state == C_IS_FILLING);
4155 assert(!c_seg_dst->c_on_minorcompact_q);
4156
4157 c_current_seg_filled(c_seg_dst, (c_segment_t *)current_chead);
4158 assert(*current_chead == NULL);
4159
4160 C_SEG_WAKEUP_DONE(c_seg_dst);
4161
4162 lck_mtx_unlock_always(&c_seg_dst->c_lock);
4163
4164 c_seg_dst = NULL;
4165
4166 PAGE_REPLACEMENT_DISALLOWED(FALSE);
4167
4168 goto Relookup_dst;
4169 }
4170
4171 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, c_seg_dst->c_nextslot);
4172
4173 memcpy(&c_seg_dst->c_store.c_buffer[c_seg_dst->c_nextoffset], &c_seg_src->c_store.c_buffer[c_src->c_offset], c_size);
5ba3f43e 4174//is platform alignment actually necessary since wkdm aligns its output?
3e170ce0
A		 4175 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
4176
39037602 4177 cslot_copy(c_dst, c_src);
3e170ce0
A		 4178 c_dst->c_offset = c_seg_dst->c_nextoffset;
4179
4180 if (c_seg_dst->c_firstemptyslot == c_seg_dst->c_nextslot)
4181 c_seg_dst->c_firstemptyslot++;
4182
5ba3f43e 4183 c_seg_dst->c_slots_used++;
3e170ce0
A		 4184 c_seg_dst->c_nextslot++;
4185 c_seg_dst->c_bytes_used += c_rounded_size;
4186 c_seg_dst->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
4187
4188
4189 PACK_C_SIZE(c_src, 0);
4190
4191 c_seg_src->c_bytes_used -= c_rounded_size;
4192 c_seg_src->c_bytes_unused += c_rounded_size;
5ba3f43e
A		 4193
4194 assert(c_seg_src->c_slots_used);
4195 c_seg_src->c_slots_used--;
3e170ce0
A		 4196
4197 if (c_indx < c_seg_src->c_firstemptyslot) {
4198 c_seg_src->c_firstemptyslot = c_indx;
4199 }
4200
4201 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, dst_slot);
4202
4203 PAGE_REPLACEMENT_ALLOWED(TRUE);
4204 slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
4205 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
4206 slot_ptr->s_cseg = c_seg_dst->c_mysegno + 1;
4207 slot_ptr->s_cindx = dst_slot;
4208
4209 PAGE_REPLACEMENT_ALLOWED(FALSE);
4210
4211out:
4212 if (c_seg_src) {
4213
4214 lck_mtx_lock_spin_always(&c_seg_src->c_lock);
4215
4216 C_SEG_WAKEUP_DONE(c_seg_src);
4217
4218 if (c_seg_src->c_bytes_used == 0 && c_seg_src->c_state != C_IS_FILLING) {
4219 if (!c_seg_src->c_on_minorcompact_q)
39037602 4220 c_seg_need_delayed_compaction(c_seg_src, FALSE);
3e170ce0
A		 4221 }
4222
4223 lck_mtx_unlock_always(&c_seg_src->c_lock);
4224 }
4225
4226 if (c_seg_dst) {
4227
4228 PAGE_REPLACEMENT_DISALLOWED(TRUE);
4229
4230 lck_mtx_lock_spin_always(&c_seg_dst->c_lock);
4231
4232 if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX_INDEX) {
4233 /*
4234 * Nearing or exceeded maximum slot and offset capacity.
4235 */
4236 assert(c_seg_dst->c_busy);
4237 assert(c_seg_dst->c_state == C_IS_FILLING);
4238 assert(!c_seg_dst->c_on_minorcompact_q);
4239
4240 c_current_seg_filled(c_seg_dst, (c_segment_t *)current_chead);
4241 assert(*current_chead == NULL);
4242 }
4243
4244 C_SEG_WAKEUP_DONE(c_seg_dst);
4245
4246 lck_mtx_unlock_always(&c_seg_dst->c_lock);
4247
4248 c_seg_dst = NULL;
4249
4250 PAGE_REPLACEMENT_DISALLOWED(FALSE);
4251 }
4252
4253 return kr;
4254}
4255#endif /* CONFIG_FREEZE */
mirror of XNU