]> git.saurik.com Git - apple/xnu.git/blob - osfmk/vm/vm_compressor.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-6153.61.1.tar.gz
[apple/xnu.git] / osfmk / vm / vm_compressor.c
1 /*
2 * Copyright (c) 2000-2013 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <vm/vm_compressor.h>
30
31 #if CONFIG_PHANTOM_CACHE
32 #include <vm/vm_phantom_cache.h>
33 #endif
34
35 #include <vm/vm_map.h>
36 #include <vm/vm_pageout.h>
37 #include <vm/memory_object.h>
38 #include <vm/vm_compressor_algorithms.h>
39 #include <vm/vm_fault.h>
40 #include <vm/vm_protos.h>
41 #include <mach/mach_host.h> /* for host_info() */
42 #include <kern/ledger.h>
43 #include <kern/policy_internal.h>
44 #include <kern/thread_group.h>
45 #include <san/kasan.h>
46
47 #if defined(__x86_64__)
48 #include <i386/misc_protos.h>
49 #endif
50 #if defined(__arm64__)
51 #include <arm/machine_routines.h>
52 #endif
53
54 #include <IOKit/IOHibernatePrivate.h>
55
56 extern boolean_t vm_darkwake_mode;
57
58 #if POPCOUNT_THE_COMPRESSED_DATA
59 boolean_t popcount_c_segs = TRUE;
60
61 static inline uint32_t
62 vmc_pop(uintptr_t ins, int sz)
63 {
64 uint32_t rv = 0;
65
66 if (__probable(popcount_c_segs == FALSE)) {
67 return 0xDEAD707C;
68 }
69
70 while (sz >= 16) {
71 uint32_t rv1, rv2;
72 uint64_t *ins64 = (uint64_t *) ins;
73 uint64_t *ins642 = (uint64_t *) (ins + 8);
74 rv1 = __builtin_popcountll(*ins64);
75 rv2 = __builtin_popcountll(*ins642);
76 rv += rv1 + rv2;
77 sz -= 16;
78 ins += 16;
79 }
80
81 while (sz >= 4) {
82 uint32_t *ins32 = (uint32_t *) ins;
83 rv += __builtin_popcount(*ins32);
84 sz -= 4;
85 ins += 4;
86 }
87
88 while (sz > 0) {
89 char *ins8 = (char *)ins;
90 rv += __builtin_popcount(*ins8);
91 sz--;
92 ins++;
93 }
94 return rv;
95 }
96 #endif
97
98 #if VALIDATE_C_SEGMENTS
99 boolean_t validate_c_segs = TRUE;
100 #endif
101 /*
102 * vm_compressor_mode has a heirarchy of control to set its value.
103 * boot-args are checked first, then device-tree, and finally
104 * the default value that is defined below. See vm_fault_init() for
105 * the boot-arg & device-tree code.
106 */
107
108 #if CONFIG_EMBEDDED
109
110 #if CONFIG_FREEZE
111 int vm_compressor_mode = VM_PAGER_FREEZER_DEFAULT;
112
113 void *freezer_chead; /* The chead used to track c_segs allocated for the exclusive use of holding just one task's compressed memory.*/
114 char *freezer_compressor_scratch_buf = NULL;
115
116 extern int c_freezer_swapout_page_count; /* This count keeps track of the # of compressed pages holding just one task's compressed memory on the swapout queue. This count is used during each freeze i.e. on a per-task basis.*/
117
118 #else /* CONFIG_FREEZE */
119 int vm_compressor_mode = VM_PAGER_NOT_CONFIGURED;
120 #endif /* CONFIG_FREEZE */
121
122 int vm_scale = 1;
123
124 #else /* CONFIG_EMBEDDED */
125 int vm_compressor_mode = VM_PAGER_COMPRESSOR_WITH_SWAP;
126 int vm_scale = 16;
127
128 #endif /* CONFIG_EMBEDDED */
129
130 int vm_compressor_is_active = 0;
131 int vm_compression_limit = 0;
132 int vm_compressor_available = 0;
133
134 extern void vm_pageout_io_throttle(void);
135
136 #if CHECKSUM_THE_DATA || CHECKSUM_THE_SWAP || CHECKSUM_THE_COMPRESSED_DATA
137 extern unsigned int hash_string(char *cp, int len);
138 static unsigned int vmc_hash(char *, int);
139 boolean_t checksum_c_segs = TRUE;
140
141 unsigned int
142 vmc_hash(char *cp, int len)
143 {
144 if (__probable(checksum_c_segs == FALSE)) {
145 return 0xDEAD7A37;
146 }
147 return hash_string(cp, len);
148 }
149 #endif
150
151 #define UNPACK_C_SIZE(cs) ((cs->c_size == (PAGE_SIZE-1)) ? PAGE_SIZE : cs->c_size)
152 #define PACK_C_SIZE(cs, size) (cs->c_size = ((size == PAGE_SIZE) ? PAGE_SIZE - 1 : size))
153
154
155 struct c_sv_hash_entry {
156 union {
157 struct {
158 uint32_t c_sv_he_ref;
159 uint32_t c_sv_he_data;
160 } c_sv_he;
161 uint64_t c_sv_he_record;
162 } c_sv_he_un;
163 };
164
165 #define he_ref c_sv_he_un.c_sv_he.c_sv_he_ref
166 #define he_data c_sv_he_un.c_sv_he.c_sv_he_data
167 #define he_record c_sv_he_un.c_sv_he_record
168
169 #define C_SV_HASH_MAX_MISS 32
170 #define C_SV_HASH_SIZE ((1 << 10))
171 #define C_SV_HASH_MASK ((1 << 10) - 1)
172 #define C_SV_CSEG_ID ((1 << 22) - 1)
173
174
175 union c_segu {
176 c_segment_t c_seg;
177 uintptr_t c_segno;
178 };
179
180
181
182 #define C_SLOT_PACK_PTR(ptr) (((uintptr_t)ptr - (uintptr_t) KERNEL_PMAP_HEAP_RANGE_START) >> 2)
183 #define C_SLOT_UNPACK_PTR(cslot) ((uintptr_t)(cslot->c_packed_ptr << 2) + (uintptr_t) KERNEL_PMAP_HEAP_RANGE_START)
184
185
186 uint32_t c_segment_count = 0;
187 uint32_t c_segment_count_max = 0;
188
189 uint64_t c_generation_id = 0;
190 uint64_t c_generation_id_flush_barrier;
191
192
193 #define HIBERNATE_FLUSHING_SECS_TO_COMPLETE 120
194
195 boolean_t hibernate_no_swapspace = FALSE;
196 clock_sec_t hibernate_flushing_deadline = 0;
197
198
199 #if RECORD_THE_COMPRESSED_DATA
200 char *c_compressed_record_sbuf;
201 char *c_compressed_record_ebuf;
202 char *c_compressed_record_cptr;
203 #endif
204
205
206 queue_head_t c_age_list_head;
207 queue_head_t c_swappedin_list_head;
208 queue_head_t c_swapout_list_head;
209 queue_head_t c_swapio_list_head;
210 queue_head_t c_swappedout_list_head;
211 queue_head_t c_swappedout_sparse_list_head;
212 queue_head_t c_major_list_head;
213 queue_head_t c_filling_list_head;
214 queue_head_t c_bad_list_head;
215
216 uint32_t c_age_count = 0;
217 uint32_t c_swappedin_count = 0;
218 uint32_t c_swapout_count = 0;
219 uint32_t c_swapio_count = 0;
220 uint32_t c_swappedout_count = 0;
221 uint32_t c_swappedout_sparse_count = 0;
222 uint32_t c_major_count = 0;
223 uint32_t c_filling_count = 0;
224 uint32_t c_empty_count = 0;
225 uint32_t c_bad_count = 0;
226
227
228 queue_head_t c_minor_list_head;
229 uint32_t c_minor_count = 0;
230
231 int c_overage_swapped_count = 0;
232 int c_overage_swapped_limit = 0;
233
234 int c_seg_fixed_array_len;
235 union c_segu *c_segments;
236 vm_offset_t c_buffers;
237 vm_size_t c_buffers_size;
238 caddr_t c_segments_next_page;
239 boolean_t c_segments_busy;
240 uint32_t c_segments_available;
241 uint32_t c_segments_limit;
242 uint32_t c_segments_nearing_limit;
243
244 uint32_t c_segment_svp_in_hash;
245 uint32_t c_segment_svp_hash_succeeded;
246 uint32_t c_segment_svp_hash_failed;
247 uint32_t c_segment_svp_zero_compressions;
248 uint32_t c_segment_svp_nonzero_compressions;
249 uint32_t c_segment_svp_zero_decompressions;
250 uint32_t c_segment_svp_nonzero_decompressions;
251
252 uint32_t c_segment_noncompressible_pages;
253
254 uint32_t c_segment_pages_compressed;
255 uint32_t c_segment_pages_compressed_limit;
256 uint32_t c_segment_pages_compressed_nearing_limit;
257 uint32_t c_free_segno_head = (uint32_t)-1;
258
259 uint32_t vm_compressor_minorcompact_threshold_divisor = 10;
260 uint32_t vm_compressor_majorcompact_threshold_divisor = 10;
261 uint32_t vm_compressor_unthrottle_threshold_divisor = 10;
262 uint32_t vm_compressor_catchup_threshold_divisor = 10;
263
264 uint32_t vm_compressor_minorcompact_threshold_divisor_overridden = 0;
265 uint32_t vm_compressor_majorcompact_threshold_divisor_overridden = 0;
266 uint32_t vm_compressor_unthrottle_threshold_divisor_overridden = 0;
267 uint32_t vm_compressor_catchup_threshold_divisor_overridden = 0;
268
269 #define C_SEGMENTS_PER_PAGE (PAGE_SIZE / sizeof(union c_segu))
270
271
272 lck_grp_attr_t vm_compressor_lck_grp_attr;
273 lck_attr_t vm_compressor_lck_attr;
274 lck_grp_t vm_compressor_lck_grp;
275 lck_mtx_t *c_list_lock;
276 lck_rw_t c_master_lock;
277 boolean_t decompressions_blocked = FALSE;
278
279 zone_t compressor_segment_zone;
280 int c_compressor_swap_trigger = 0;
281
282 uint32_t compressor_cpus;
283 char *compressor_scratch_bufs;
284 char *kdp_compressor_scratch_buf;
285 char *kdp_compressor_decompressed_page;
286 addr64_t kdp_compressor_decompressed_page_paddr;
287 ppnum_t kdp_compressor_decompressed_page_ppnum;
288
289 clock_sec_t start_of_sample_period_sec = 0;
290 clock_nsec_t start_of_sample_period_nsec = 0;
291 clock_sec_t start_of_eval_period_sec = 0;
292 clock_nsec_t start_of_eval_period_nsec = 0;
293 uint32_t sample_period_decompression_count = 0;
294 uint32_t sample_period_compression_count = 0;
295 uint32_t last_eval_decompression_count = 0;
296 uint32_t last_eval_compression_count = 0;
297
298 #define DECOMPRESSION_SAMPLE_MAX_AGE (60 * 30)
299
300 boolean_t vm_swapout_ripe_segments = FALSE;
301 uint32_t vm_ripe_target_age = (60 * 60 * 48);
302
303 uint32_t swapout_target_age = 0;
304 uint32_t age_of_decompressions_during_sample_period[DECOMPRESSION_SAMPLE_MAX_AGE];
305 uint32_t overage_decompressions_during_sample_period = 0;
306
307
308 void do_fastwake_warmup(queue_head_t *, boolean_t);
309 boolean_t fastwake_warmup = FALSE;
310 boolean_t fastwake_recording_in_progress = FALSE;
311 clock_sec_t dont_trim_until_ts = 0;
312
313 uint64_t c_segment_warmup_count;
314 uint64_t first_c_segment_to_warm_generation_id = 0;
315 uint64_t last_c_segment_to_warm_generation_id = 0;
316 boolean_t hibernate_flushing = FALSE;
317
318 int64_t c_segment_input_bytes __attribute__((aligned(8))) = 0;
319 int64_t c_segment_compressed_bytes __attribute__((aligned(8))) = 0;
320 int64_t compressor_bytes_used __attribute__((aligned(8))) = 0;
321
322
323 struct c_sv_hash_entry c_segment_sv_hash_table[C_SV_HASH_SIZE] __attribute__ ((aligned(8)));
324
325 static boolean_t compressor_needs_to_swap(void);
326 static void vm_compressor_swap_trigger_thread(void);
327 static void vm_compressor_do_delayed_compactions(boolean_t);
328 static void vm_compressor_compact_and_swap(boolean_t);
329 static void vm_compressor_age_swapped_in_segments(boolean_t);
330
331 #if !CONFIG_EMBEDDED
332 static void vm_compressor_take_paging_space_action(void);
333 #endif
334
335 void compute_swapout_target_age(void);
336
337 boolean_t c_seg_major_compact(c_segment_t, c_segment_t);
338 boolean_t c_seg_major_compact_ok(c_segment_t, c_segment_t);
339
340 int c_seg_minor_compaction_and_unlock(c_segment_t, boolean_t);
341 int c_seg_do_minor_compaction_and_unlock(c_segment_t, boolean_t, boolean_t, boolean_t);
342 void c_seg_try_minor_compaction_and_unlock(c_segment_t c_seg);
343
344 void c_seg_move_to_sparse_list(c_segment_t);
345 void c_seg_insert_into_q(queue_head_t *, c_segment_t);
346
347 uint64_t vm_available_memory(void);
348 uint64_t vm_compressor_pages_compressed(void);
349
350 /*
351 * indicate the need to do a major compaction if
352 * the overall set of in-use compression segments
353 * becomes sparse... on systems that support pressure
354 * driven swapping, this will also cause swapouts to
355 * be initiated.
356 */
357 static inline boolean_t
358 vm_compressor_needs_to_major_compact()
359 {
360 uint32_t incore_seg_count;
361
362 incore_seg_count = c_segment_count - c_swappedout_count - c_swappedout_sparse_count;
363
364 if ((c_segment_count >= (c_segments_nearing_limit / 8)) &&
365 ((incore_seg_count * C_SEG_MAX_PAGES) - VM_PAGE_COMPRESSOR_COUNT) >
366 ((incore_seg_count / 8) * C_SEG_MAX_PAGES)) {
367 return 1;
368 }
369 return 0;
370 }
371
372
373 uint64_t
374 vm_available_memory(void)
375 {
376 return ((uint64_t)AVAILABLE_NON_COMPRESSED_MEMORY) * PAGE_SIZE_64;
377 }
378
379
380 uint64_t
381 vm_compressor_pages_compressed(void)
382 {
383 return c_segment_pages_compressed * PAGE_SIZE_64;
384 }
385
386
387 boolean_t
388 vm_compressor_low_on_space(void)
389 {
390 if ((c_segment_pages_compressed > c_segment_pages_compressed_nearing_limit) ||
391 (c_segment_count > c_segments_nearing_limit)) {
392 return TRUE;
393 }
394
395 return FALSE;
396 }
397
398
399 boolean_t
400 vm_compressor_out_of_space(void)
401 {
402 if ((c_segment_pages_compressed >= c_segment_pages_compressed_limit) ||
403 (c_segment_count >= c_segments_limit)) {
404 return TRUE;
405 }
406
407 return FALSE;
408 }
409
410
411 int
412 vm_wants_task_throttled(task_t task)
413 {
414 if (task == kernel_task) {
415 return 0;
416 }
417
418 if (VM_CONFIG_SWAP_IS_ACTIVE) {
419 if ((vm_compressor_low_on_space() || HARD_THROTTLE_LIMIT_REACHED()) &&
420 (unsigned int)pmap_compressed(task->map->pmap) > (c_segment_pages_compressed / 4)) {
421 return 1;
422 }
423 }
424 return 0;
425 }
426
427
428 #if DEVELOPMENT || DEBUG
429 boolean_t kill_on_no_paging_space = FALSE; /* On compressor/swap exhaustion, kill the largest process regardless of
430 * its chosen process policy. Controlled by a boot-arg of the same name. */
431 #endif /* DEVELOPMENT || DEBUG */
432
433 #if !CONFIG_EMBEDDED
434
435 static uint32_t no_paging_space_action_in_progress = 0;
436 extern void memorystatus_send_low_swap_note(void);
437
438 static void
439 vm_compressor_take_paging_space_action(void)
440 {
441 if (no_paging_space_action_in_progress == 0) {
442 if (OSCompareAndSwap(0, 1, (UInt32 *)&no_paging_space_action_in_progress)) {
443 if (no_paging_space_action()) {
444 #if DEVELOPMENT || DEBUG
445 if (kill_on_no_paging_space == TRUE) {
446 /*
447 * Since we are choosing to always kill a process, we don't need the
448 * "out of application memory" dialog box in this mode. And, hence we won't
449 * send the knote.
450 */
451 no_paging_space_action_in_progress = 0;
452 return;
453 }
454 #endif /* DEVELOPMENT || DEBUG */
455 memorystatus_send_low_swap_note();
456 }
457
458 no_paging_space_action_in_progress = 0;
459 }
460 }
461 }
462 #endif /* !CONFIG_EMBEDDED */
463
464
465 void
466 vm_compressor_init_locks(void)
467 {
468 lck_grp_attr_setdefault(&vm_compressor_lck_grp_attr);
469 lck_grp_init(&vm_compressor_lck_grp, "vm_compressor", &vm_compressor_lck_grp_attr);
470 lck_attr_setdefault(&vm_compressor_lck_attr);
471
472 lck_rw_init(&c_master_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
473 }
474
475
476 void
477 vm_decompressor_lock(void)
478 {
479 PAGE_REPLACEMENT_ALLOWED(TRUE);
480
481 decompressions_blocked = TRUE;
482
483 PAGE_REPLACEMENT_ALLOWED(FALSE);
484 }
485
486 void
487 vm_decompressor_unlock(void)
488 {
489 PAGE_REPLACEMENT_ALLOWED(TRUE);
490
491 decompressions_blocked = FALSE;
492
493 PAGE_REPLACEMENT_ALLOWED(FALSE);
494
495 thread_wakeup((event_t)&decompressions_blocked);
496 }
497
498 static inline void
499 cslot_copy(c_slot_t cdst, c_slot_t csrc)
500 {
501 #if CHECKSUM_THE_DATA
502 cdst->c_hash_data = csrc->c_hash_data;
503 #endif
504 #if CHECKSUM_THE_COMPRESSED_DATA
505 cdst->c_hash_compressed_data = csrc->c_hash_compressed_data;
506 #endif
507 #if POPCOUNT_THE_COMPRESSED_DATA
508 cdst->c_pop_cdata = csrc->c_pop_cdata;
509 #endif
510 cdst->c_size = csrc->c_size;
511 cdst->c_packed_ptr = csrc->c_packed_ptr;
512 #if defined(__arm__) || defined(__arm64__)
513 cdst->c_codec = csrc->c_codec;
514 #endif
515 }
516
517 vm_map_t compressor_map;
518 uint64_t compressor_pool_max_size;
519 uint64_t compressor_pool_size;
520 uint32_t compressor_pool_multiplier;
521
522 #if DEVELOPMENT || DEBUG
523 /*
524 * Compressor segments are write-protected in development/debug
525 * kernels to help debug memory corruption.
526 * In cases where performance is a concern, this can be disabled
527 * via the boot-arg "-disable_cseg_write_protection".
528 */
529 boolean_t write_protect_c_segs = TRUE;
530 int vm_compressor_test_seg_wp;
531 uint32_t vm_ktrace_enabled;
532 #endif /* DEVELOPMENT || DEBUG */
533
534 void
535 vm_compressor_init(void)
536 {
537 thread_t thread;
538 struct c_slot cs_dummy;
539 c_slot_t cs = &cs_dummy;
540 int c_segment_min_size;
541 int c_segment_padded_size;
542 int attempts = 1;
543 kern_return_t retval = KERN_SUCCESS;
544 vm_offset_t start_addr = 0;
545 vm_size_t c_segments_arr_size = 0, compressor_submap_size = 0;
546 vm_map_kernel_flags_t vmk_flags;
547 #if RECORD_THE_COMPRESSED_DATA
548 vm_size_t c_compressed_record_sbuf_size = 0;
549 #endif /* RECORD_THE_COMPRESSED_DATA */
550
551 #if DEVELOPMENT || DEBUG
552 char bootarg_name[32];
553 if (PE_parse_boot_argn("-kill_on_no_paging_space", bootarg_name, sizeof(bootarg_name))) {
554 kill_on_no_paging_space = TRUE;
555 }
556 if (PE_parse_boot_argn("-disable_cseg_write_protection", bootarg_name, sizeof(bootarg_name))) {
557 write_protect_c_segs = FALSE;
558 }
559 int vmcval = 1;
560 PE_parse_boot_argn("vm_compressor_validation", &vmcval, sizeof(vmcval));
561
562 if (kern_feature_override(KF_COMPRSV_OVRD)) {
563 vmcval = 0;
564 }
565 if (vmcval == 0) {
566 #if POPCOUNT_THE_COMPRESSED_DATA
567 popcount_c_segs = FALSE;
568 #endif
569 #if CHECKSUM_THE_DATA || CHECKSUM_THE_COMPRESSED_DATA
570 checksum_c_segs = FALSE;
571 #endif
572 #if VALIDATE_C_SEGMENTS
573 validate_c_segs = FALSE;
574 #endif
575 write_protect_c_segs = FALSE;
576 }
577 #endif /* DEVELOPMENT || DEBUG */
578
579 /*
580 * ensure that any pointer that gets created from
581 * the vm_page zone can be packed properly
582 */
583 cs->c_packed_ptr = C_SLOT_PACK_PTR(zone_map_min_address);
584
585 if (C_SLOT_UNPACK_PTR(cs) != (uintptr_t)zone_map_min_address) {
586 panic("C_SLOT_UNPACK_PTR failed on zone_map_min_address - %p", (void *)zone_map_min_address);
587 }
588
589 cs->c_packed_ptr = C_SLOT_PACK_PTR(zone_map_max_address);
590
591 if (C_SLOT_UNPACK_PTR(cs) != (uintptr_t)zone_map_max_address) {
592 panic("C_SLOT_UNPACK_PTR failed on zone_map_max_address - %p", (void *)zone_map_max_address);
593 }
594
595
596 assert((C_SEGMENTS_PER_PAGE * sizeof(union c_segu)) == PAGE_SIZE);
597
598 PE_parse_boot_argn("vm_compression_limit", &vm_compression_limit, sizeof(vm_compression_limit));
599
600 #ifdef CONFIG_EMBEDDED
601 vm_compressor_minorcompact_threshold_divisor = 20;
602 vm_compressor_majorcompact_threshold_divisor = 30;
603 vm_compressor_unthrottle_threshold_divisor = 40;
604 vm_compressor_catchup_threshold_divisor = 60;
605 #else
606 if (max_mem <= (3ULL * 1024ULL * 1024ULL * 1024ULL)) {
607 vm_compressor_minorcompact_threshold_divisor = 11;
608 vm_compressor_majorcompact_threshold_divisor = 13;
609 vm_compressor_unthrottle_threshold_divisor = 20;
610 vm_compressor_catchup_threshold_divisor = 35;
611 } else {
612 vm_compressor_minorcompact_threshold_divisor = 20;
613 vm_compressor_majorcompact_threshold_divisor = 25;
614 vm_compressor_unthrottle_threshold_divisor = 35;
615 vm_compressor_catchup_threshold_divisor = 50;
616 }
617 #endif
618 /*
619 * vm_page_init_lck_grp is now responsible for calling vm_compressor_init_locks
620 * c_master_lock needs to be available early so that "vm_page_find_contiguous" can
621 * use PAGE_REPLACEMENT_ALLOWED to coordinate with the compressor.
622 */
623
624 c_list_lock = lck_mtx_alloc_init(&vm_compressor_lck_grp, &vm_compressor_lck_attr);
625
626 queue_init(&c_bad_list_head);
627 queue_init(&c_age_list_head);
628 queue_init(&c_minor_list_head);
629 queue_init(&c_major_list_head);
630 queue_init(&c_filling_list_head);
631 queue_init(&c_swapout_list_head);
632 queue_init(&c_swapio_list_head);
633 queue_init(&c_swappedin_list_head);
634 queue_init(&c_swappedout_list_head);
635 queue_init(&c_swappedout_sparse_list_head);
636
637 c_free_segno_head = -1;
638 c_segments_available = 0;
639
640 if (vm_compression_limit) {
641 compressor_pool_size = (uint64_t)vm_compression_limit * PAGE_SIZE_64;
642 }
643
644 compressor_pool_max_size = C_SEG_MAX_LIMIT;
645 compressor_pool_max_size *= C_SEG_BUFSIZE;
646
647 #if !CONFIG_EMBEDDED
648
649 if (vm_compression_limit == 0) {
650 if (max_mem <= (4ULL * 1024ULL * 1024ULL * 1024ULL)) {
651 compressor_pool_size = 16ULL * max_mem;
652 } else if (max_mem <= (8ULL * 1024ULL * 1024ULL * 1024ULL)) {
653 compressor_pool_size = 8ULL * max_mem;
654 } else if (max_mem <= (32ULL * 1024ULL * 1024ULL * 1024ULL)) {
655 compressor_pool_size = 4ULL * max_mem;
656 } else {
657 compressor_pool_size = 2ULL * max_mem;
658 }
659 }
660 if (max_mem <= (8ULL * 1024ULL * 1024ULL * 1024ULL)) {
661 compressor_pool_multiplier = 1;
662 } else if (max_mem <= (32ULL * 1024ULL * 1024ULL * 1024ULL)) {
663 compressor_pool_multiplier = 2;
664 } else {
665 compressor_pool_multiplier = 4;
666 }
667
668 #elif defined(__arm__)
669
670 #define VM_RESERVE_SIZE (1024 * 1024 * 256)
671 #define MAX_COMPRESSOR_POOL_SIZE (1024 * 1024 * 450)
672
673 if (compressor_pool_max_size > MAX_COMPRESSOR_POOL_SIZE) {
674 compressor_pool_max_size = MAX_COMPRESSOR_POOL_SIZE;
675 }
676
677 if (vm_compression_limit == 0) {
678 compressor_pool_size = ((kernel_map->max_offset - kernel_map->min_offset) - kernel_map->size) - VM_RESERVE_SIZE;
679 }
680 compressor_pool_multiplier = 1;
681 #else
682 if (compressor_pool_max_size > max_mem) {
683 compressor_pool_max_size = max_mem;
684 }
685
686 if (vm_compression_limit == 0) {
687 compressor_pool_size = max_mem;
688 }
689 compressor_pool_multiplier = 1;
690 #endif
691 if (compressor_pool_size > compressor_pool_max_size) {
692 compressor_pool_size = compressor_pool_max_size;
693 }
694
695 try_again:
696 c_segments_limit = (uint32_t)(compressor_pool_size / (vm_size_t)(C_SEG_ALLOCSIZE));
697 c_segments_nearing_limit = (uint32_t)(((uint64_t)c_segments_limit * 98ULL) / 100ULL);
698
699 c_segment_pages_compressed_limit = (c_segments_limit * (C_SEG_BUFSIZE / PAGE_SIZE) * compressor_pool_multiplier);
700
701 if (c_segment_pages_compressed_limit < (uint32_t)(max_mem / PAGE_SIZE)) {
702 c_segment_pages_compressed_limit = (uint32_t)(max_mem / PAGE_SIZE);
703 }
704
705 c_segment_pages_compressed_nearing_limit = (uint32_t)(((uint64_t)c_segment_pages_compressed_limit * 98ULL) / 100ULL);
706
707 /*
708 * Submap needs space for:
709 * - c_segments
710 * - c_buffers
711 * - swap reclaimations -- C_SEG_BUFSIZE
712 */
713 c_segments_arr_size = vm_map_round_page((sizeof(union c_segu) * c_segments_limit), VM_MAP_PAGE_MASK(kernel_map));
714 c_buffers_size = vm_map_round_page(((vm_size_t)C_SEG_ALLOCSIZE * (vm_size_t)c_segments_limit), VM_MAP_PAGE_MASK(kernel_map));
715
716 compressor_submap_size = c_segments_arr_size + c_buffers_size + C_SEG_BUFSIZE;
717
718 #if RECORD_THE_COMPRESSED_DATA
719 c_compressed_record_sbuf_size = (vm_size_t)C_SEG_ALLOCSIZE + (PAGE_SIZE * 2);
720 compressor_submap_size += c_compressed_record_sbuf_size;
721 #endif /* RECORD_THE_COMPRESSED_DATA */
722
723 vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
724 vmk_flags.vmkf_permanent = TRUE;
725 retval = kmem_suballoc(kernel_map, &start_addr, compressor_submap_size,
726 FALSE, VM_FLAGS_ANYWHERE, vmk_flags, VM_KERN_MEMORY_COMPRESSOR,
727 &compressor_map);
728
729 if (retval != KERN_SUCCESS) {
730 if (++attempts > 3) {
731 panic("vm_compressor_init: kmem_suballoc failed - 0x%llx", (uint64_t)compressor_submap_size);
732 }
733
734 compressor_pool_size = compressor_pool_size / 2;
735
736 kprintf("retrying creation of the compressor submap at 0x%llx bytes\n", compressor_pool_size);
737 goto try_again;
738 }
739 if (kernel_memory_allocate(compressor_map, (vm_offset_t *)(&c_segments), (sizeof(union c_segu) * c_segments_limit), 0, KMA_KOBJECT | KMA_VAONLY | KMA_PERMANENT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS) {
740 panic("vm_compressor_init: kernel_memory_allocate failed - c_segments\n");
741 }
742 if (kernel_memory_allocate(compressor_map, &c_buffers, c_buffers_size, 0, KMA_COMPRESSOR | KMA_VAONLY | KMA_PERMANENT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS) {
743 panic("vm_compressor_init: kernel_memory_allocate failed - c_buffers\n");
744 }
745
746
747 c_segment_min_size = sizeof(struct c_segment) + (C_SEG_SLOT_VAR_ARRAY_MIN_LEN * sizeof(struct c_slot));
748
749 for (c_segment_padded_size = 128; c_segment_padded_size < c_segment_min_size; c_segment_padded_size = c_segment_padded_size << 1) {
750 ;
751 }
752
753 compressor_segment_zone = zinit(c_segment_padded_size, c_segments_limit * c_segment_padded_size, PAGE_SIZE, "compressor_segment");
754 zone_change(compressor_segment_zone, Z_CALLERACCT, FALSE);
755 zone_change(compressor_segment_zone, Z_NOENCRYPT, TRUE);
756
757 c_seg_fixed_array_len = (c_segment_padded_size - sizeof(struct c_segment)) / sizeof(struct c_slot);
758
759 c_segments_busy = FALSE;
760
761 c_segments_next_page = (caddr_t)c_segments;
762 vm_compressor_algorithm_init();
763
764 {
765 host_basic_info_data_t hinfo;
766 mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
767
768 #define BSD_HOST 1
769 host_info((host_t)BSD_HOST, HOST_BASIC_INFO, (host_info_t)&hinfo, &count);
770
771 compressor_cpus = hinfo.max_cpus;
772 compressor_scratch_bufs = kalloc_tag(compressor_cpus * vm_compressor_get_decode_scratch_size(), VM_KERN_MEMORY_COMPRESSOR);
773
774 kdp_compressor_scratch_buf = kalloc_tag(vm_compressor_get_decode_scratch_size(), VM_KERN_MEMORY_COMPRESSOR);
775
776 /*
777 * kdp_compressor_decompressed_page must be page aligned because we access
778 * it through the physical apperture by page number. kalloc() does not
779 * guarantee alignment.
780 */
781 vm_offset_t addr;
782 if (kernel_memory_allocate(kernel_map, &addr, PAGE_SIZE, 0, KMA_KOBJECT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS) {
783 panic("vm_compressor_init: kernel_memory_allocate failed - kdp_compressor_decompressed_page\n");
784 }
785 assert((addr & PAGE_MASK) == 0);
786 kdp_compressor_decompressed_page = (void *)addr;
787 kdp_compressor_decompressed_page_paddr = kvtophys((vm_offset_t)kdp_compressor_decompressed_page);
788 kdp_compressor_decompressed_page_ppnum = (ppnum_t) atop(kdp_compressor_decompressed_page_paddr);
789 }
790 #if CONFIG_FREEZE
791 freezer_compressor_scratch_buf = kalloc_tag(vm_compressor_get_encode_scratch_size(), VM_KERN_MEMORY_COMPRESSOR);
792 #endif
793
794 #if RECORD_THE_COMPRESSED_DATA
795 if (kernel_memory_allocate(compressor_map, (vm_offset_t *)&c_compressed_record_sbuf, c_compressed_record_sbuf_size, 0, KMA_KOBJECT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS) {
796 panic("vm_compressor_init: kernel_memory_allocate failed - c_compressed_record_sbuf\n");
797 }
798
799 c_compressed_record_cptr = c_compressed_record_sbuf;
800 c_compressed_record_ebuf = c_compressed_record_sbuf + c_compressed_record_sbuf_size;
801 #endif
802
803 if (kernel_thread_start_priority((thread_continue_t)vm_compressor_swap_trigger_thread, NULL,
804 BASEPRI_VM, &thread) != KERN_SUCCESS) {
805 panic("vm_compressor_swap_trigger_thread: create failed");
806 }
807 thread_deallocate(thread);
808
809 if (vm_pageout_internal_start() != KERN_SUCCESS) {
810 panic("vm_compressor_init: Failed to start the internal pageout thread.\n");
811 }
812 if (VM_CONFIG_SWAP_IS_PRESENT) {
813 vm_compressor_swap_init();
814 }
815
816 if (VM_CONFIG_COMPRESSOR_IS_ACTIVE) {
817 vm_compressor_is_active = 1;
818 }
819
820 #if CONFIG_FREEZE
821 memorystatus_freeze_enabled = TRUE;
822 #endif /* CONFIG_FREEZE */
823
824 vm_compressor_available = 1;
825
826 vm_page_reactivate_all_throttled();
827 }
828
829
830 #if VALIDATE_C_SEGMENTS
831
832 static void
833 c_seg_validate(c_segment_t c_seg, boolean_t must_be_compact)
834 {
835 int c_indx;
836 int32_t bytes_used;
837 uint32_t c_rounded_size;
838 uint32_t c_size;
839 c_slot_t cs;
840
841 if (__probable(validate_c_segs == FALSE)) {
842 return;
843 }
844 if (c_seg->c_firstemptyslot < c_seg->c_nextslot) {
845 c_indx = c_seg->c_firstemptyslot;
846 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
847
848 if (cs == NULL) {
849 panic("c_seg_validate: no slot backing c_firstemptyslot");
850 }
851
852 if (cs->c_size) {
853 panic("c_seg_validate: c_firstemptyslot has non-zero size (%d)\n", cs->c_size);
854 }
855 }
856 bytes_used = 0;
857
858 for (c_indx = 0; c_indx < c_seg->c_nextslot; c_indx++) {
859 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
860
861 c_size = UNPACK_C_SIZE(cs);
862
863 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
864
865 bytes_used += c_rounded_size;
866
867 #if CHECKSUM_THE_COMPRESSED_DATA
868 unsigned csvhash;
869 if (c_size && cs->c_hash_compressed_data != (csvhash = vmc_hash((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size))) {
870 addr64_t csvphys = kvtophys((vm_offset_t)&c_seg->c_store.c_buffer[cs->c_offset]);
871 panic("Compressed data doesn't match original %p phys: 0x%llx %d %p %d %d 0x%x 0x%x", c_seg, csvphys, cs->c_offset, cs, c_indx, c_size, cs->c_hash_compressed_data, csvhash);
872 }
873 #endif
874 #if POPCOUNT_THE_COMPRESSED_DATA
875 unsigned csvpop;
876 if (c_size) {
877 uintptr_t csvaddr = (uintptr_t) &c_seg->c_store.c_buffer[cs->c_offset];
878 if (cs->c_pop_cdata != (csvpop = vmc_pop(csvaddr, c_size))) {
879 panic("Compressed data popcount doesn't match original, bit distance: %d %p (phys: %p) %p %p 0x%llx 0x%x 0x%x 0x%x", (csvpop - cs->c_pop_cdata), (void *)csvaddr, (void *) kvtophys(csvaddr), c_seg, cs, (uint64_t)cs->c_offset, c_size, csvpop, cs->c_pop_cdata);
880 }
881 }
882 #endif
883 }
884
885 if (bytes_used != c_seg->c_bytes_used) {
886 panic("c_seg_validate: bytes_used mismatch - found %d, segment has %d\n", bytes_used, c_seg->c_bytes_used);
887 }
888
889 if (c_seg->c_bytes_used > C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset)) {
890 panic("c_seg_validate: c_bytes_used > c_nextoffset - c_nextoffset = %d, c_bytes_used = %d\n",
891 (int32_t)C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset), c_seg->c_bytes_used);
892 }
893
894 if (must_be_compact) {
895 if (c_seg->c_bytes_used != C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset)) {
896 panic("c_seg_validate: c_bytes_used doesn't match c_nextoffset - c_nextoffset = %d, c_bytes_used = %d\n",
897 (int32_t)C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset), c_seg->c_bytes_used);
898 }
899 }
900 }
901
902 #endif
903
904
905 void
906 c_seg_need_delayed_compaction(c_segment_t c_seg, boolean_t c_list_lock_held)
907 {
908 boolean_t clear_busy = FALSE;
909
910 if (c_list_lock_held == FALSE) {
911 if (!lck_mtx_try_lock_spin_always(c_list_lock)) {
912 C_SEG_BUSY(c_seg);
913
914 lck_mtx_unlock_always(&c_seg->c_lock);
915 lck_mtx_lock_spin_always(c_list_lock);
916 lck_mtx_lock_spin_always(&c_seg->c_lock);
917
918 clear_busy = TRUE;
919 }
920 }
921 assert(c_seg->c_state != C_IS_FILLING);
922
923 if (!c_seg->c_on_minorcompact_q && !(C_SEG_IS_ON_DISK_OR_SOQ(c_seg))) {
924 queue_enter(&c_minor_list_head, c_seg, c_segment_t, c_list);
925 c_seg->c_on_minorcompact_q = 1;
926 c_minor_count++;
927 }
928 if (c_list_lock_held == FALSE) {
929 lck_mtx_unlock_always(c_list_lock);
930 }
931
932 if (clear_busy == TRUE) {
933 C_SEG_WAKEUP_DONE(c_seg);
934 }
935 }
936
937
938 unsigned int c_seg_moved_to_sparse_list = 0;
939
940 void
941 c_seg_move_to_sparse_list(c_segment_t c_seg)
942 {
943 boolean_t clear_busy = FALSE;
944
945 if (!lck_mtx_try_lock_spin_always(c_list_lock)) {
946 C_SEG_BUSY(c_seg);
947
948 lck_mtx_unlock_always(&c_seg->c_lock);
949 lck_mtx_lock_spin_always(c_list_lock);
950 lck_mtx_lock_spin_always(&c_seg->c_lock);
951
952 clear_busy = TRUE;
953 }
954 c_seg_switch_state(c_seg, C_ON_SWAPPEDOUTSPARSE_Q, FALSE);
955
956 c_seg_moved_to_sparse_list++;
957
958 lck_mtx_unlock_always(c_list_lock);
959
960 if (clear_busy == TRUE) {
961 C_SEG_WAKEUP_DONE(c_seg);
962 }
963 }
964
965
966 void
967 c_seg_insert_into_q(queue_head_t *qhead, c_segment_t c_seg)
968 {
969 c_segment_t c_seg_next;
970
971 if (queue_empty(qhead)) {
972 queue_enter(qhead, c_seg, c_segment_t, c_age_list);
973 } else {
974 c_seg_next = (c_segment_t)queue_first(qhead);
975
976 while (TRUE) {
977 if (c_seg->c_generation_id < c_seg_next->c_generation_id) {
978 queue_insert_before(qhead, c_seg, c_seg_next, c_segment_t, c_age_list);
979 break;
980 }
981 c_seg_next = (c_segment_t) queue_next(&c_seg_next->c_age_list);
982
983 if (queue_end(qhead, (queue_entry_t) c_seg_next)) {
984 queue_enter(qhead, c_seg, c_segment_t, c_age_list);
985 break;
986 }
987 }
988 }
989 }
990
991
992 int try_minor_compaction_failed = 0;
993 int try_minor_compaction_succeeded = 0;
994
995 void
996 c_seg_try_minor_compaction_and_unlock(c_segment_t c_seg)
997 {
998 assert(c_seg->c_on_minorcompact_q);
999 /*
1000 * c_seg is currently on the delayed minor compaction
1001 * queue and we have c_seg locked... if we can get the
1002 * c_list_lock w/o blocking (if we blocked we could deadlock
1003 * because the lock order is c_list_lock then c_seg's lock)
1004 * we'll pull it from the delayed list and free it directly
1005 */
1006 if (!lck_mtx_try_lock_spin_always(c_list_lock)) {
1007 /*
1008 * c_list_lock is held, we need to bail
1009 */
1010 try_minor_compaction_failed++;
1011
1012 lck_mtx_unlock_always(&c_seg->c_lock);
1013 } else {
1014 try_minor_compaction_succeeded++;
1015
1016 C_SEG_BUSY(c_seg);
1017 c_seg_do_minor_compaction_and_unlock(c_seg, TRUE, FALSE, FALSE);
1018 }
1019 }
1020
1021
1022 int
1023 c_seg_do_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy, boolean_t need_list_lock, boolean_t disallow_page_replacement)
1024 {
1025 int c_seg_freed;
1026
1027 assert(c_seg->c_busy);
1028 assert(!C_SEG_IS_ON_DISK_OR_SOQ(c_seg));
1029
1030 /*
1031 * check for the case that can occur when we are not swapping
1032 * and this segment has been major compacted in the past
1033 * and moved to the majorcompact q to remove it from further
1034 * consideration... if the occupancy falls too low we need
1035 * to put it back on the age_q so that it will be considered
1036 * in the next major compaction sweep... if we don't do this
1037 * we will eventually run into the c_segments_limit
1038 */
1039 if (c_seg->c_state == C_ON_MAJORCOMPACT_Q && C_SEG_SHOULD_MAJORCOMPACT_NOW(c_seg)) {
1040 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
1041 }
1042 if (!c_seg->c_on_minorcompact_q) {
1043 if (clear_busy == TRUE) {
1044 C_SEG_WAKEUP_DONE(c_seg);
1045 }
1046
1047 lck_mtx_unlock_always(&c_seg->c_lock);
1048
1049 return 0;
1050 }
1051 queue_remove(&c_minor_list_head, c_seg, c_segment_t, c_list);
1052 c_seg->c_on_minorcompact_q = 0;
1053 c_minor_count--;
1054
1055 lck_mtx_unlock_always(c_list_lock);
1056
1057 if (disallow_page_replacement == TRUE) {
1058 lck_mtx_unlock_always(&c_seg->c_lock);
1059
1060 PAGE_REPLACEMENT_DISALLOWED(TRUE);
1061
1062 lck_mtx_lock_spin_always(&c_seg->c_lock);
1063 }
1064 c_seg_freed = c_seg_minor_compaction_and_unlock(c_seg, clear_busy);
1065
1066 if (disallow_page_replacement == TRUE) {
1067 PAGE_REPLACEMENT_DISALLOWED(FALSE);
1068 }
1069
1070 if (need_list_lock == TRUE) {
1071 lck_mtx_lock_spin_always(c_list_lock);
1072 }
1073
1074 return c_seg_freed;
1075 }
1076
1077
1078 void
1079 c_seg_wait_on_busy(c_segment_t c_seg)
1080 {
1081 c_seg->c_wanted = 1;
1082 assert_wait((event_t) (c_seg), THREAD_UNINT);
1083
1084 lck_mtx_unlock_always(&c_seg->c_lock);
1085 thread_block(THREAD_CONTINUE_NULL);
1086 }
1087
1088
1089 void
1090 c_seg_switch_state(c_segment_t c_seg, int new_state, boolean_t insert_head)
1091 {
1092 int old_state = c_seg->c_state;
1093
1094 #if !CONFIG_EMBEDDED
1095 #if DEVELOPMENT || DEBUG
1096 if (new_state != C_IS_FILLING) {
1097 LCK_MTX_ASSERT(&c_seg->c_lock, LCK_MTX_ASSERT_OWNED);
1098 }
1099 LCK_MTX_ASSERT(c_list_lock, LCK_MTX_ASSERT_OWNED);
1100 #endif
1101 #endif /* !CONFIG_EMBEDDED */
1102 switch (old_state) {
1103 case C_IS_EMPTY:
1104 assert(new_state == C_IS_FILLING || new_state == C_IS_FREE);
1105
1106 c_empty_count--;
1107 break;
1108
1109 case C_IS_FILLING:
1110 assert(new_state == C_ON_AGE_Q || new_state == C_ON_SWAPOUT_Q);
1111
1112 queue_remove(&c_filling_list_head, c_seg, c_segment_t, c_age_list);
1113 c_filling_count--;
1114 break;
1115
1116 case C_ON_AGE_Q:
1117 assert(new_state == C_ON_SWAPOUT_Q || new_state == C_ON_MAJORCOMPACT_Q ||
1118 new_state == C_IS_FREE);
1119
1120 queue_remove(&c_age_list_head, c_seg, c_segment_t, c_age_list);
1121 c_age_count--;
1122 break;
1123
1124 case C_ON_SWAPPEDIN_Q:
1125 assert(new_state == C_ON_AGE_Q || new_state == C_IS_FREE);
1126
1127 queue_remove(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
1128 c_swappedin_count--;
1129 break;
1130
1131 case C_ON_SWAPOUT_Q:
1132 assert(new_state == C_ON_AGE_Q || new_state == C_IS_FREE || new_state == C_IS_EMPTY || new_state == C_ON_SWAPIO_Q);
1133
1134 queue_remove(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
1135 thread_wakeup((event_t)&compaction_swapper_running);
1136 c_swapout_count--;
1137 break;
1138
1139 case C_ON_SWAPIO_Q:
1140 assert(new_state == C_ON_SWAPPEDOUT_Q || new_state == C_ON_SWAPPEDOUTSPARSE_Q || new_state == C_ON_AGE_Q);
1141
1142 queue_remove(&c_swapio_list_head, c_seg, c_segment_t, c_age_list);
1143 c_swapio_count--;
1144 break;
1145
1146 case C_ON_SWAPPEDOUT_Q:
1147 assert(new_state == C_ON_SWAPPEDIN_Q || new_state == C_ON_AGE_Q ||
1148 new_state == C_ON_SWAPPEDOUTSPARSE_Q ||
1149 new_state == C_ON_BAD_Q || new_state == C_IS_EMPTY || new_state == C_IS_FREE);
1150
1151 queue_remove(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
1152 c_swappedout_count--;
1153 break;
1154
1155 case C_ON_SWAPPEDOUTSPARSE_Q:
1156 assert(new_state == C_ON_SWAPPEDIN_Q || new_state == C_ON_AGE_Q ||
1157 new_state == C_ON_BAD_Q || new_state == C_IS_EMPTY || new_state == C_IS_FREE);
1158
1159 queue_remove(&c_swappedout_sparse_list_head, c_seg, c_segment_t, c_age_list);
1160 c_swappedout_sparse_count--;
1161 break;
1162
1163 case C_ON_MAJORCOMPACT_Q:
1164 assert(new_state == C_ON_AGE_Q || new_state == C_IS_FREE);
1165
1166 queue_remove(&c_major_list_head, c_seg, c_segment_t, c_age_list);
1167 c_major_count--;
1168 break;
1169
1170 case C_ON_BAD_Q:
1171 assert(new_state == C_IS_FREE);
1172
1173 queue_remove(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
1174 c_bad_count--;
1175 break;
1176
1177 default:
1178 panic("c_seg %p has bad c_state = %d\n", c_seg, old_state);
1179 }
1180
1181 switch (new_state) {
1182 case C_IS_FREE:
1183 assert(old_state != C_IS_FILLING);
1184
1185 break;
1186
1187 case C_IS_EMPTY:
1188 assert(old_state == C_ON_SWAPOUT_Q || old_state == C_ON_SWAPPEDOUT_Q || old_state == C_ON_SWAPPEDOUTSPARSE_Q);
1189
1190 c_empty_count++;
1191 break;
1192
1193 case C_IS_FILLING:
1194 assert(old_state == C_IS_EMPTY);
1195
1196 queue_enter(&c_filling_list_head, c_seg, c_segment_t, c_age_list);
1197 c_filling_count++;
1198 break;
1199
1200 case C_ON_AGE_Q:
1201 assert(old_state == C_IS_FILLING || old_state == C_ON_SWAPPEDIN_Q ||
1202 old_state == C_ON_SWAPOUT_Q || old_state == C_ON_SWAPIO_Q ||
1203 old_state == C_ON_MAJORCOMPACT_Q || old_state == C_ON_SWAPPEDOUT_Q || old_state == C_ON_SWAPPEDOUTSPARSE_Q);
1204
1205 if (old_state == C_IS_FILLING) {
1206 queue_enter(&c_age_list_head, c_seg, c_segment_t, c_age_list);
1207 } else {
1208 if (!queue_empty(&c_age_list_head)) {
1209 c_segment_t c_first;
1210
1211 c_first = (c_segment_t)queue_first(&c_age_list_head);
1212 c_seg->c_creation_ts = c_first->c_creation_ts;
1213 }
1214 queue_enter_first(&c_age_list_head, c_seg, c_segment_t, c_age_list);
1215 }
1216 c_age_count++;
1217 break;
1218
1219 case C_ON_SWAPPEDIN_Q:
1220 assert(old_state == C_ON_SWAPPEDOUT_Q || old_state == C_ON_SWAPPEDOUTSPARSE_Q);
1221
1222 if (insert_head == TRUE) {
1223 queue_enter_first(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
1224 } else {
1225 queue_enter(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
1226 }
1227 c_swappedin_count++;
1228 break;
1229
1230 case C_ON_SWAPOUT_Q:
1231 assert(old_state == C_ON_AGE_Q || old_state == C_IS_FILLING);
1232
1233 if (insert_head == TRUE) {
1234 queue_enter_first(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
1235 } else {
1236 queue_enter(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
1237 }
1238 c_swapout_count++;
1239 break;
1240
1241 case C_ON_SWAPIO_Q:
1242 assert(old_state == C_ON_SWAPOUT_Q);
1243
1244 if (insert_head == TRUE) {
1245 queue_enter_first(&c_swapio_list_head, c_seg, c_segment_t, c_age_list);
1246 } else {
1247 queue_enter(&c_swapio_list_head, c_seg, c_segment_t, c_age_list);
1248 }
1249 c_swapio_count++;
1250 break;
1251
1252 case C_ON_SWAPPEDOUT_Q:
1253 assert(old_state == C_ON_SWAPIO_Q);
1254
1255 if (insert_head == TRUE) {
1256 queue_enter_first(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
1257 } else {
1258 queue_enter(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
1259 }
1260 c_swappedout_count++;
1261 break;
1262
1263 case C_ON_SWAPPEDOUTSPARSE_Q:
1264 assert(old_state == C_ON_SWAPIO_Q || old_state == C_ON_SWAPPEDOUT_Q);
1265
1266 if (insert_head == TRUE) {
1267 queue_enter_first(&c_swappedout_sparse_list_head, c_seg, c_segment_t, c_age_list);
1268 } else {
1269 queue_enter(&c_swappedout_sparse_list_head, c_seg, c_segment_t, c_age_list);
1270 }
1271
1272 c_swappedout_sparse_count++;
1273 break;
1274
1275 case C_ON_MAJORCOMPACT_Q:
1276 assert(old_state == C_ON_AGE_Q);
1277
1278 if (insert_head == TRUE) {
1279 queue_enter_first(&c_major_list_head, c_seg, c_segment_t, c_age_list);
1280 } else {
1281 queue_enter(&c_major_list_head, c_seg, c_segment_t, c_age_list);
1282 }
1283 c_major_count++;
1284 break;
1285
1286 case C_ON_BAD_Q:
1287 assert(old_state == C_ON_SWAPPEDOUT_Q || old_state == C_ON_SWAPPEDOUTSPARSE_Q);
1288
1289 if (insert_head == TRUE) {
1290 queue_enter_first(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
1291 } else {
1292 queue_enter(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
1293 }
1294 c_bad_count++;
1295 break;
1296
1297 default:
1298 panic("c_seg %p requesting bad c_state = %d\n", c_seg, new_state);
1299 }
1300 c_seg->c_state = new_state;
1301 }
1302
1303
1304
1305 void
1306 c_seg_free(c_segment_t c_seg)
1307 {
1308 assert(c_seg->c_busy);
1309
1310 lck_mtx_unlock_always(&c_seg->c_lock);
1311 lck_mtx_lock_spin_always(c_list_lock);
1312 lck_mtx_lock_spin_always(&c_seg->c_lock);
1313
1314 c_seg_free_locked(c_seg);
1315 }
1316
1317
1318 void
1319 c_seg_free_locked(c_segment_t c_seg)
1320 {
1321 int segno;
1322 int pages_populated = 0;
1323 int32_t *c_buffer = NULL;
1324 uint64_t c_swap_handle = 0;
1325
1326 assert(c_seg->c_busy);
1327 assert(c_seg->c_slots_used == 0);
1328 assert(!c_seg->c_on_minorcompact_q);
1329 assert(!c_seg->c_busy_swapping);
1330
1331 if (c_seg->c_overage_swap == TRUE) {
1332 c_overage_swapped_count--;
1333 c_seg->c_overage_swap = FALSE;
1334 }
1335 if (!(C_SEG_IS_ONDISK(c_seg))) {
1336 c_buffer = c_seg->c_store.c_buffer;
1337 } else {
1338 c_swap_handle = c_seg->c_store.c_swap_handle;
1339 }
1340
1341 c_seg_switch_state(c_seg, C_IS_FREE, FALSE);
1342
1343 lck_mtx_unlock_always(c_list_lock);
1344
1345 if (c_buffer) {
1346 pages_populated = (round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE;
1347 c_seg->c_store.c_buffer = NULL;
1348 } else {
1349 c_seg->c_store.c_swap_handle = (uint64_t)-1;
1350 }
1351
1352 lck_mtx_unlock_always(&c_seg->c_lock);
1353
1354 if (c_buffer) {
1355 if (pages_populated) {
1356 kernel_memory_depopulate(compressor_map, (vm_offset_t) c_buffer, pages_populated * PAGE_SIZE, KMA_COMPRESSOR);
1357 }
1358 } else if (c_swap_handle) {
1359 /*
1360 * Free swap space on disk.
1361 */
1362 vm_swap_free(c_swap_handle);
1363 }
1364 lck_mtx_lock_spin_always(&c_seg->c_lock);
1365 /*
1366 * c_seg must remain busy until
1367 * after the call to vm_swap_free
1368 */
1369 C_SEG_WAKEUP_DONE(c_seg);
1370 lck_mtx_unlock_always(&c_seg->c_lock);
1371
1372 segno = c_seg->c_mysegno;
1373
1374 lck_mtx_lock_spin_always(c_list_lock);
1375 /*
1376 * because the c_buffer is now associated with the segno,
1377 * we can't put the segno back on the free list until
1378 * after we have depopulated the c_buffer range, or
1379 * we run the risk of depopulating a range that is
1380 * now being used in one of the compressor heads
1381 */
1382 c_segments[segno].c_segno = c_free_segno_head;
1383 c_free_segno_head = segno;
1384 c_segment_count--;
1385
1386 lck_mtx_unlock_always(c_list_lock);
1387
1388 lck_mtx_destroy(&c_seg->c_lock, &vm_compressor_lck_grp);
1389
1390 if (c_seg->c_slot_var_array_len) {
1391 kfree(c_seg->c_slot_var_array, sizeof(struct c_slot) * c_seg->c_slot_var_array_len);
1392 }
1393
1394 zfree(compressor_segment_zone, c_seg);
1395 }
1396
1397 #if DEVELOPMENT || DEBUG
1398 int c_seg_trim_page_count = 0;
1399 #endif
1400
1401 void
1402 c_seg_trim_tail(c_segment_t c_seg)
1403 {
1404 c_slot_t cs;
1405 uint32_t c_size;
1406 uint32_t c_offset;
1407 uint32_t c_rounded_size;
1408 uint16_t current_nextslot;
1409 uint32_t current_populated_offset;
1410
1411 if (c_seg->c_bytes_used == 0) {
1412 return;
1413 }
1414 current_nextslot = c_seg->c_nextslot;
1415 current_populated_offset = c_seg->c_populated_offset;
1416
1417 while (c_seg->c_nextslot) {
1418 cs = C_SEG_SLOT_FROM_INDEX(c_seg, (c_seg->c_nextslot - 1));
1419
1420 c_size = UNPACK_C_SIZE(cs);
1421
1422 if (c_size) {
1423 if (current_nextslot != c_seg->c_nextslot) {
1424 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
1425 c_offset = cs->c_offset + C_SEG_BYTES_TO_OFFSET(c_rounded_size);
1426
1427 c_seg->c_nextoffset = c_offset;
1428 c_seg->c_populated_offset = (c_offset + (C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1)) &
1429 ~(C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1);
1430
1431 if (c_seg->c_firstemptyslot > c_seg->c_nextslot) {
1432 c_seg->c_firstemptyslot = c_seg->c_nextslot;
1433 }
1434 #if DEVELOPMENT || DEBUG
1435 c_seg_trim_page_count += ((round_page_32(C_SEG_OFFSET_TO_BYTES(current_populated_offset)) -
1436 round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) /
1437 PAGE_SIZE);
1438 #endif
1439 }
1440 break;
1441 }
1442 c_seg->c_nextslot--;
1443 }
1444 assert(c_seg->c_nextslot);
1445 }
1446
1447
1448 int
1449 c_seg_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy)
1450 {
1451 c_slot_mapping_t slot_ptr;
1452 uint32_t c_offset = 0;
1453 uint32_t old_populated_offset;
1454 uint32_t c_rounded_size;
1455 uint32_t c_size;
1456 int c_indx = 0;
1457 int i;
1458 c_slot_t c_dst;
1459 c_slot_t c_src;
1460
1461 assert(c_seg->c_busy);
1462
1463 #if VALIDATE_C_SEGMENTS
1464 c_seg_validate(c_seg, FALSE);
1465 #endif
1466 if (c_seg->c_bytes_used == 0) {
1467 c_seg_free(c_seg);
1468 return 1;
1469 }
1470 lck_mtx_unlock_always(&c_seg->c_lock);
1471
1472 if (c_seg->c_firstemptyslot >= c_seg->c_nextslot || C_SEG_UNUSED_BYTES(c_seg) < PAGE_SIZE) {
1473 goto done;
1474 }
1475
1476 /* TODO: assert first emptyslot's c_size is actually 0 */
1477
1478 #if DEVELOPMENT || DEBUG
1479 C_SEG_MAKE_WRITEABLE(c_seg);
1480 #endif
1481
1482 #if VALIDATE_C_SEGMENTS
1483 c_seg->c_was_minor_compacted++;
1484 #endif
1485 c_indx = c_seg->c_firstemptyslot;
1486 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
1487
1488 old_populated_offset = c_seg->c_populated_offset;
1489 c_offset = c_dst->c_offset;
1490
1491 for (i = c_indx + 1; i < c_seg->c_nextslot && c_offset < c_seg->c_nextoffset; i++) {
1492 c_src = C_SEG_SLOT_FROM_INDEX(c_seg, i);
1493
1494 c_size = UNPACK_C_SIZE(c_src);
1495
1496 if (c_size == 0) {
1497 continue;
1498 }
1499
1500 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
1501 /* N.B.: This memcpy may be an overlapping copy */
1502 memcpy(&c_seg->c_store.c_buffer[c_offset], &c_seg->c_store.c_buffer[c_src->c_offset], c_rounded_size);
1503
1504 cslot_copy(c_dst, c_src);
1505 c_dst->c_offset = c_offset;
1506
1507 slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
1508 slot_ptr->s_cindx = c_indx;
1509
1510 c_offset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
1511 PACK_C_SIZE(c_src, 0);
1512 c_indx++;
1513
1514 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
1515 }
1516 c_seg->c_firstemptyslot = c_indx;
1517 c_seg->c_nextslot = c_indx;
1518 c_seg->c_nextoffset = c_offset;
1519 c_seg->c_populated_offset = (c_offset + (C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1)) & ~(C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1);
1520 c_seg->c_bytes_unused = 0;
1521
1522 #if VALIDATE_C_SEGMENTS
1523 c_seg_validate(c_seg, TRUE);
1524 #endif
1525 if (old_populated_offset > c_seg->c_populated_offset) {
1526 uint32_t gc_size;
1527 int32_t *gc_ptr;
1528
1529 gc_size = C_SEG_OFFSET_TO_BYTES(old_populated_offset - c_seg->c_populated_offset);
1530 gc_ptr = &c_seg->c_store.c_buffer[c_seg->c_populated_offset];
1531
1532 kernel_memory_depopulate(compressor_map, (vm_offset_t)gc_ptr, gc_size, KMA_COMPRESSOR);
1533 }
1534
1535 #if DEVELOPMENT || DEBUG
1536 C_SEG_WRITE_PROTECT(c_seg);
1537 #endif
1538
1539 done:
1540 if (clear_busy == TRUE) {
1541 lck_mtx_lock_spin_always(&c_seg->c_lock);
1542 C_SEG_WAKEUP_DONE(c_seg);
1543 lck_mtx_unlock_always(&c_seg->c_lock);
1544 }
1545 return 0;
1546 }
1547
1548
1549 static void
1550 c_seg_alloc_nextslot(c_segment_t c_seg)
1551 {
1552 struct c_slot *old_slot_array = NULL;
1553 struct c_slot *new_slot_array = NULL;
1554 int newlen;
1555 int oldlen;
1556
1557 if (c_seg->c_nextslot < c_seg_fixed_array_len) {
1558 return;
1559 }
1560
1561 if ((c_seg->c_nextslot - c_seg_fixed_array_len) >= c_seg->c_slot_var_array_len) {
1562 oldlen = c_seg->c_slot_var_array_len;
1563 old_slot_array = c_seg->c_slot_var_array;
1564
1565 if (oldlen == 0) {
1566 newlen = C_SEG_SLOT_VAR_ARRAY_MIN_LEN;
1567 } else {
1568 newlen = oldlen * 2;
1569 }
1570
1571 new_slot_array = (struct c_slot *)kalloc(sizeof(struct c_slot) * newlen);
1572
1573 lck_mtx_lock_spin_always(&c_seg->c_lock);
1574
1575 if (old_slot_array) {
1576 memcpy((char *)new_slot_array, (char *)old_slot_array, sizeof(struct c_slot) * oldlen);
1577 }
1578
1579 c_seg->c_slot_var_array_len = newlen;
1580 c_seg->c_slot_var_array = new_slot_array;
1581
1582 lck_mtx_unlock_always(&c_seg->c_lock);
1583
1584 if (old_slot_array) {
1585 kfree(old_slot_array, sizeof(struct c_slot) * oldlen);
1586 }
1587 }
1588 }
1589
1590
1591
1592 struct {
1593 uint64_t asked_permission;
1594 uint64_t compactions;
1595 uint64_t moved_slots;
1596 uint64_t moved_bytes;
1597 uint64_t wasted_space_in_swapouts;
1598 uint64_t count_of_swapouts;
1599 uint64_t count_of_freed_segs;
1600 } c_seg_major_compact_stats;
1601
1602
1603 #define C_MAJOR_COMPACTION_SIZE_APPROPRIATE ((C_SEG_BUFSIZE * 90) / 100)
1604
1605
1606 boolean_t
1607 c_seg_major_compact_ok(
1608 c_segment_t c_seg_dst,
1609 c_segment_t c_seg_src)
1610 {
1611 c_seg_major_compact_stats.asked_permission++;
1612
1613 if (c_seg_src->c_bytes_used >= C_MAJOR_COMPACTION_SIZE_APPROPRIATE &&
1614 c_seg_dst->c_bytes_used >= C_MAJOR_COMPACTION_SIZE_APPROPRIATE) {
1615 return FALSE;
1616 }
1617
1618 if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX_INDEX) {
1619 /*
1620 * destination segment is full... can't compact
1621 */
1622 return FALSE;
1623 }
1624
1625 return TRUE;
1626 }
1627
1628
1629 boolean_t
1630 c_seg_major_compact(
1631 c_segment_t c_seg_dst,
1632 c_segment_t c_seg_src)
1633 {
1634 c_slot_mapping_t slot_ptr;
1635 uint32_t c_rounded_size;
1636 uint32_t c_size;
1637 uint16_t dst_slot;
1638 int i;
1639 c_slot_t c_dst;
1640 c_slot_t c_src;
1641 boolean_t keep_compacting = TRUE;
1642
1643 /*
1644 * segments are not locked but they are both marked c_busy
1645 * which keeps c_decompress from working on them...
1646 * we can safely allocate new pages, move compressed data
1647 * from c_seg_src to c_seg_dst and update both c_segment's
1648 * state w/o holding the master lock
1649 */
1650 #if DEVELOPMENT || DEBUG
1651 C_SEG_MAKE_WRITEABLE(c_seg_dst);
1652 #endif
1653
1654 #if VALIDATE_C_SEGMENTS
1655 c_seg_dst->c_was_major_compacted++;
1656 c_seg_src->c_was_major_donor++;
1657 #endif
1658 c_seg_major_compact_stats.compactions++;
1659
1660 dst_slot = c_seg_dst->c_nextslot;
1661
1662 for (i = 0; i < c_seg_src->c_nextslot; i++) {
1663 c_src = C_SEG_SLOT_FROM_INDEX(c_seg_src, i);
1664
1665 c_size = UNPACK_C_SIZE(c_src);
1666
1667 if (c_size == 0) {
1668 /* BATCH: move what we have so far; */
1669 continue;
1670 }
1671
1672 if (C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset - c_seg_dst->c_nextoffset) < (unsigned) c_size) {
1673 int size_to_populate;
1674
1675 /* doesn't fit */
1676 size_to_populate = C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset);
1677
1678 if (size_to_populate == 0) {
1679 /* can't fit */
1680 keep_compacting = FALSE;
1681 break;
1682 }
1683 if (size_to_populate > C_SEG_MAX_POPULATE_SIZE) {
1684 size_to_populate = C_SEG_MAX_POPULATE_SIZE;
1685 }
1686
1687 kernel_memory_populate(compressor_map,
1688 (vm_offset_t) &c_seg_dst->c_store.c_buffer[c_seg_dst->c_populated_offset],
1689 size_to_populate,
1690 KMA_COMPRESSOR,
1691 VM_KERN_MEMORY_COMPRESSOR);
1692
1693 c_seg_dst->c_populated_offset += C_SEG_BYTES_TO_OFFSET(size_to_populate);
1694 assert(C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset) <= C_SEG_BUFSIZE);
1695 }
1696 c_seg_alloc_nextslot(c_seg_dst);
1697
1698 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, c_seg_dst->c_nextslot);
1699
1700 memcpy(&c_seg_dst->c_store.c_buffer[c_seg_dst->c_nextoffset], &c_seg_src->c_store.c_buffer[c_src->c_offset], c_size);
1701
1702 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
1703
1704 c_seg_major_compact_stats.moved_slots++;
1705 c_seg_major_compact_stats.moved_bytes += c_size;
1706
1707 cslot_copy(c_dst, c_src);
1708 c_dst->c_offset = c_seg_dst->c_nextoffset;
1709
1710 if (c_seg_dst->c_firstemptyslot == c_seg_dst->c_nextslot) {
1711 c_seg_dst->c_firstemptyslot++;
1712 }
1713 c_seg_dst->c_slots_used++;
1714 c_seg_dst->c_nextslot++;
1715 c_seg_dst->c_bytes_used += c_rounded_size;
1716 c_seg_dst->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
1717
1718 PACK_C_SIZE(c_src, 0);
1719
1720 c_seg_src->c_bytes_used -= c_rounded_size;
1721 c_seg_src->c_bytes_unused += c_rounded_size;
1722 c_seg_src->c_firstemptyslot = 0;
1723
1724 assert(c_seg_src->c_slots_used);
1725 c_seg_src->c_slots_used--;
1726
1727 if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX_INDEX) {
1728 /* dest segment is now full */
1729 keep_compacting = FALSE;
1730 break;
1731 }
1732 }
1733 #if DEVELOPMENT || DEBUG
1734 C_SEG_WRITE_PROTECT(c_seg_dst);
1735 #endif
1736 if (dst_slot < c_seg_dst->c_nextslot) {
1737 PAGE_REPLACEMENT_ALLOWED(TRUE);
1738 /*
1739 * we've now locked out c_decompress from
1740 * converting the slot passed into it into
1741 * a c_segment_t which allows us to use
1742 * the backptr to change which c_segment and
1743 * index the slot points to
1744 */
1745 while (dst_slot < c_seg_dst->c_nextslot) {
1746 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, dst_slot);
1747
1748 slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
1749 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
1750 slot_ptr->s_cseg = c_seg_dst->c_mysegno + 1;
1751 slot_ptr->s_cindx = dst_slot++;
1752 }
1753 PAGE_REPLACEMENT_ALLOWED(FALSE);
1754 }
1755 return keep_compacting;
1756 }
1757
1758
1759 uint64_t
1760 vm_compressor_compute_elapsed_msecs(clock_sec_t end_sec, clock_nsec_t end_nsec, clock_sec_t start_sec, clock_nsec_t start_nsec)
1761 {
1762 uint64_t end_msecs;
1763 uint64_t start_msecs;
1764
1765 end_msecs = (end_sec * 1000) + end_nsec / 1000000;
1766 start_msecs = (start_sec * 1000) + start_nsec / 1000000;
1767
1768 return end_msecs - start_msecs;
1769 }
1770
1771
1772
1773 uint32_t compressor_eval_period_in_msecs = 250;
1774 uint32_t compressor_sample_min_in_msecs = 500;
1775 uint32_t compressor_sample_max_in_msecs = 10000;
1776 uint32_t compressor_thrashing_threshold_per_10msecs = 50;
1777 uint32_t compressor_thrashing_min_per_10msecs = 20;
1778
1779 /* When true, reset sample data next chance we get. */
1780 static boolean_t compressor_need_sample_reset = FALSE;
1781
1782
1783 void
1784 compute_swapout_target_age(void)
1785 {
1786 clock_sec_t cur_ts_sec;
1787 clock_nsec_t cur_ts_nsec;
1788 uint32_t min_operations_needed_in_this_sample;
1789 uint64_t elapsed_msecs_in_eval;
1790 uint64_t elapsed_msecs_in_sample;
1791 boolean_t need_eval_reset = FALSE;
1792
1793 clock_get_system_nanotime(&cur_ts_sec, &cur_ts_nsec);
1794
1795 elapsed_msecs_in_sample = vm_compressor_compute_elapsed_msecs(cur_ts_sec, cur_ts_nsec, start_of_sample_period_sec, start_of_sample_period_nsec);
1796
1797 if (compressor_need_sample_reset ||
1798 elapsed_msecs_in_sample >= compressor_sample_max_in_msecs) {
1799 compressor_need_sample_reset = TRUE;
1800 need_eval_reset = TRUE;
1801 goto done;
1802 }
1803 elapsed_msecs_in_eval = vm_compressor_compute_elapsed_msecs(cur_ts_sec, cur_ts_nsec, start_of_eval_period_sec, start_of_eval_period_nsec);
1804
1805 if (elapsed_msecs_in_eval < compressor_eval_period_in_msecs) {
1806 goto done;
1807 }
1808 need_eval_reset = TRUE;
1809
1810 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_START, elapsed_msecs_in_eval, sample_period_compression_count, sample_period_decompression_count, 0, 0);
1811
1812 min_operations_needed_in_this_sample = (compressor_thrashing_min_per_10msecs * (uint32_t)elapsed_msecs_in_eval) / 10;
1813
1814 if ((sample_period_compression_count - last_eval_compression_count) < min_operations_needed_in_this_sample ||
1815 (sample_period_decompression_count - last_eval_decompression_count) < min_operations_needed_in_this_sample) {
1816 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, sample_period_compression_count - last_eval_compression_count,
1817 sample_period_decompression_count - last_eval_decompression_count, 0, 1, 0);
1818
1819 swapout_target_age = 0;
1820
1821 compressor_need_sample_reset = TRUE;
1822 need_eval_reset = TRUE;
1823 goto done;
1824 }
1825 last_eval_compression_count = sample_period_compression_count;
1826 last_eval_decompression_count = sample_period_decompression_count;
1827
1828 if (elapsed_msecs_in_sample < compressor_sample_min_in_msecs) {
1829 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, swapout_target_age, 0, 0, 5, 0);
1830 goto done;
1831 }
1832 if (sample_period_decompression_count > ((compressor_thrashing_threshold_per_10msecs * elapsed_msecs_in_sample) / 10)) {
1833 uint64_t running_total;
1834 uint64_t working_target;
1835 uint64_t aging_target;
1836 uint32_t oldest_age_of_csegs_sampled = 0;
1837 uint64_t working_set_approximation = 0;
1838
1839 swapout_target_age = 0;
1840
1841 working_target = (sample_period_decompression_count / 100) * 95; /* 95 percent */
1842 aging_target = (sample_period_decompression_count / 100) * 1; /* 1 percent */
1843 running_total = 0;
1844
1845 for (oldest_age_of_csegs_sampled = 0; oldest_age_of_csegs_sampled < DECOMPRESSION_SAMPLE_MAX_AGE; oldest_age_of_csegs_sampled++) {
1846 running_total += age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
1847
1848 working_set_approximation += oldest_age_of_csegs_sampled * age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
1849
1850 if (running_total >= working_target) {
1851 break;
1852 }
1853 }
1854 if (oldest_age_of_csegs_sampled < DECOMPRESSION_SAMPLE_MAX_AGE) {
1855 working_set_approximation = (working_set_approximation * 1000) / elapsed_msecs_in_sample;
1856
1857 if (working_set_approximation < VM_PAGE_COMPRESSOR_COUNT) {
1858 running_total = overage_decompressions_during_sample_period;
1859
1860 for (oldest_age_of_csegs_sampled = DECOMPRESSION_SAMPLE_MAX_AGE - 1; oldest_age_of_csegs_sampled; oldest_age_of_csegs_sampled--) {
1861 running_total += age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
1862
1863 if (running_total >= aging_target) {
1864 break;
1865 }
1866 }
1867 swapout_target_age = (uint32_t)cur_ts_sec - oldest_age_of_csegs_sampled;
1868
1869 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, swapout_target_age, working_set_approximation, VM_PAGE_COMPRESSOR_COUNT, 2, 0);
1870 } else {
1871 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, working_set_approximation, VM_PAGE_COMPRESSOR_COUNT, 0, 3, 0);
1872 }
1873 } else {
1874 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, working_target, running_total, 0, 4, 0);
1875 }
1876
1877 compressor_need_sample_reset = TRUE;
1878 need_eval_reset = TRUE;
1879 } else {
1880 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, sample_period_decompression_count, (compressor_thrashing_threshold_per_10msecs * elapsed_msecs_in_sample) / 10, 0, 6, 0);
1881 }
1882 done:
1883 if (compressor_need_sample_reset == TRUE) {
1884 bzero(age_of_decompressions_during_sample_period, sizeof(age_of_decompressions_during_sample_period));
1885 overage_decompressions_during_sample_period = 0;
1886
1887 start_of_sample_period_sec = cur_ts_sec;
1888 start_of_sample_period_nsec = cur_ts_nsec;
1889 sample_period_decompression_count = 0;
1890 sample_period_compression_count = 0;
1891 last_eval_decompression_count = 0;
1892 last_eval_compression_count = 0;
1893 compressor_need_sample_reset = FALSE;
1894 }
1895 if (need_eval_reset == TRUE) {
1896 start_of_eval_period_sec = cur_ts_sec;
1897 start_of_eval_period_nsec = cur_ts_nsec;
1898 }
1899 }
1900
1901
1902 int compaction_swapper_init_now = 0;
1903 int compaction_swapper_running = 0;
1904 int compaction_swapper_awakened = 0;
1905 int compaction_swapper_abort = 0;
1906
1907
1908 #if CONFIG_JETSAM
1909 boolean_t memorystatus_kill_on_VM_compressor_thrashing(boolean_t);
1910 boolean_t memorystatus_kill_on_VM_compressor_space_shortage(boolean_t);
1911 boolean_t memorystatus_kill_on_FC_thrashing(boolean_t);
1912 int compressor_thrashing_induced_jetsam = 0;
1913 int filecache_thrashing_induced_jetsam = 0;
1914 static boolean_t vm_compressor_thrashing_detected = FALSE;
1915 #endif /* CONFIG_JETSAM */
1916
1917 static boolean_t
1918 compressor_needs_to_swap(void)
1919 {
1920 boolean_t should_swap = FALSE;
1921
1922 if (vm_swapout_ripe_segments == TRUE && c_overage_swapped_count < c_overage_swapped_limit) {
1923 c_segment_t c_seg;
1924 clock_sec_t now;
1925 clock_sec_t age;
1926 clock_nsec_t nsec;
1927
1928 clock_get_system_nanotime(&now, &nsec);
1929 age = 0;
1930
1931 lck_mtx_lock_spin_always(c_list_lock);
1932
1933 if (!queue_empty(&c_age_list_head)) {
1934 c_seg = (c_segment_t) queue_first(&c_age_list_head);
1935
1936 age = now - c_seg->c_creation_ts;
1937 }
1938 lck_mtx_unlock_always(c_list_lock);
1939
1940 if (age >= vm_ripe_target_age) {
1941 return TRUE;
1942 }
1943 }
1944 if (VM_CONFIG_SWAP_IS_ACTIVE) {
1945 if (COMPRESSOR_NEEDS_TO_SWAP()) {
1946 return TRUE;
1947 }
1948 if (VM_PAGE_Q_THROTTLED(&vm_pageout_queue_external) && vm_page_anonymous_count < (vm_page_inactive_count / 20)) {
1949 return TRUE;
1950 }
1951 if (vm_page_free_count < (vm_page_free_reserved - (COMPRESSOR_FREE_RESERVED_LIMIT * 2))) {
1952 return TRUE;
1953 }
1954 }
1955 compute_swapout_target_age();
1956
1957 if (swapout_target_age) {
1958 c_segment_t c_seg;
1959
1960 lck_mtx_lock_spin_always(c_list_lock);
1961
1962 if (!queue_empty(&c_age_list_head)) {
1963 c_seg = (c_segment_t) queue_first(&c_age_list_head);
1964
1965 if (c_seg->c_creation_ts > swapout_target_age) {
1966 swapout_target_age = 0;
1967 }
1968 }
1969 lck_mtx_unlock_always(c_list_lock);
1970 }
1971 #if CONFIG_PHANTOM_CACHE
1972 if (vm_phantom_cache_check_pressure()) {
1973 should_swap = TRUE;
1974 }
1975 #endif
1976 if (swapout_target_age) {
1977 should_swap = TRUE;
1978 }
1979
1980 #if CONFIG_JETSAM
1981 if (should_swap || vm_compressor_low_on_space() == TRUE) {
1982 if (vm_compressor_thrashing_detected == FALSE) {
1983 vm_compressor_thrashing_detected = TRUE;
1984
1985 if (swapout_target_age || vm_compressor_low_on_space() == TRUE) {
1986 if (swapout_target_age) {
1987 /* The compressor is thrashing. */
1988 memorystatus_kill_on_VM_compressor_thrashing(TRUE /* async */);
1989 } else {
1990 /* The compressor is running low on space. */
1991 memorystatus_kill_on_VM_compressor_space_shortage(TRUE /* async */);
1992 }
1993 compressor_thrashing_induced_jetsam++;
1994 } else {
1995 memorystatus_kill_on_FC_thrashing(TRUE /* async */);
1996 filecache_thrashing_induced_jetsam++;
1997 }
1998 }
1999 /*
2000 * let the jetsam take precedence over
2001 * any major compactions we might have
2002 * been able to do... otherwise we run
2003 * the risk of doing major compactions
2004 * on segments we're about to free up
2005 * due to the jetsam activity.
2006 */
2007 should_swap = FALSE;
2008 }
2009
2010 #endif /* CONFIG_JETSAM */
2011
2012 if (should_swap == FALSE) {
2013 /*
2014 * vm_compressor_needs_to_major_compact returns true only if we're
2015 * about to run out of available compressor segments... in this
2016 * case, we absolutely need to run a major compaction even if
2017 * we've just kicked off a jetsam or we don't otherwise need to
2018 * swap... terminating objects releases
2019 * pages back to the uncompressed cache, but does not guarantee
2020 * that we will free up even a single compression segment
2021 */
2022 should_swap = vm_compressor_needs_to_major_compact();
2023 }
2024
2025 /*
2026 * returning TRUE when swap_supported == FALSE
2027 * will cause the major compaction engine to
2028 * run, but will not trigger any swapping...
2029 * segments that have been major compacted
2030 * will be moved to the majorcompact queue
2031 */
2032 return should_swap;
2033 }
2034
2035 #if CONFIG_JETSAM
2036 /*
2037 * This function is called from the jetsam thread after killing something to
2038 * mitigate thrashing.
2039 *
2040 * We need to restart our thrashing detection heuristics since memory pressure
2041 * has potentially changed significantly, and we don't want to detect on old
2042 * data from before the jetsam.
2043 */
2044 void
2045 vm_thrashing_jetsam_done(void)
2046 {
2047 vm_compressor_thrashing_detected = FALSE;
2048
2049 /* Were we compressor-thrashing or filecache-thrashing? */
2050 if (swapout_target_age) {
2051 swapout_target_age = 0;
2052 compressor_need_sample_reset = TRUE;
2053 }
2054 #if CONFIG_PHANTOM_CACHE
2055 else {
2056 vm_phantom_cache_restart_sample();
2057 }
2058 #endif
2059 }
2060 #endif /* CONFIG_JETSAM */
2061
2062 uint32_t vm_wake_compactor_swapper_calls = 0;
2063 uint32_t vm_run_compactor_already_running = 0;
2064 uint32_t vm_run_compactor_empty_minor_q = 0;
2065 uint32_t vm_run_compactor_did_compact = 0;
2066 uint32_t vm_run_compactor_waited = 0;
2067
2068 void
2069 vm_run_compactor(void)
2070 {
2071 if (c_segment_count == 0) {
2072 return;
2073 }
2074
2075 lck_mtx_lock_spin_always(c_list_lock);
2076
2077 if (c_minor_count == 0) {
2078 vm_run_compactor_empty_minor_q++;
2079
2080 lck_mtx_unlock_always(c_list_lock);
2081 return;
2082 }
2083 if (compaction_swapper_running) {
2084 if (vm_pageout_state.vm_restricted_to_single_processor == FALSE) {
2085 vm_run_compactor_already_running++;
2086
2087 lck_mtx_unlock_always(c_list_lock);
2088 return;
2089 }
2090 vm_run_compactor_waited++;
2091
2092 assert_wait((event_t)&compaction_swapper_running, THREAD_UNINT);
2093
2094 lck_mtx_unlock_always(c_list_lock);
2095
2096 thread_block(THREAD_CONTINUE_NULL);
2097
2098 return;
2099 }
2100 vm_run_compactor_did_compact++;
2101
2102 fastwake_warmup = FALSE;
2103 compaction_swapper_running = 1;
2104
2105 vm_compressor_do_delayed_compactions(FALSE);
2106
2107 compaction_swapper_running = 0;
2108
2109 lck_mtx_unlock_always(c_list_lock);
2110
2111 thread_wakeup((event_t)&compaction_swapper_running);
2112 }
2113
2114
2115 void
2116 vm_wake_compactor_swapper(void)
2117 {
2118 if (compaction_swapper_running || compaction_swapper_awakened || c_segment_count == 0) {
2119 return;
2120 }
2121
2122 if (c_minor_count || vm_compressor_needs_to_major_compact()) {
2123 lck_mtx_lock_spin_always(c_list_lock);
2124
2125 fastwake_warmup = FALSE;
2126
2127 if (compaction_swapper_running == 0 && compaction_swapper_awakened == 0) {
2128 vm_wake_compactor_swapper_calls++;
2129
2130 compaction_swapper_awakened = 1;
2131 thread_wakeup((event_t)&c_compressor_swap_trigger);
2132 }
2133 lck_mtx_unlock_always(c_list_lock);
2134 }
2135 }
2136
2137
2138 void
2139 vm_consider_swapping()
2140 {
2141 c_segment_t c_seg, c_seg_next;
2142 clock_sec_t now;
2143 clock_nsec_t nsec;
2144
2145 assert(VM_CONFIG_SWAP_IS_PRESENT);
2146
2147 lck_mtx_lock_spin_always(c_list_lock);
2148
2149 compaction_swapper_abort = 1;
2150
2151 while (compaction_swapper_running) {
2152 assert_wait((event_t)&compaction_swapper_running, THREAD_UNINT);
2153
2154 lck_mtx_unlock_always(c_list_lock);
2155
2156 thread_block(THREAD_CONTINUE_NULL);
2157
2158 lck_mtx_lock_spin_always(c_list_lock);
2159 }
2160 compaction_swapper_abort = 0;
2161 compaction_swapper_running = 1;
2162
2163 vm_swapout_ripe_segments = TRUE;
2164
2165 if (!queue_empty(&c_major_list_head)) {
2166 clock_get_system_nanotime(&now, &nsec);
2167
2168 c_seg = (c_segment_t)queue_first(&c_major_list_head);
2169
2170 while (!queue_end(&c_major_list_head, (queue_entry_t)c_seg)) {
2171 if (c_overage_swapped_count >= c_overage_swapped_limit) {
2172 break;
2173 }
2174
2175 c_seg_next = (c_segment_t) queue_next(&c_seg->c_age_list);
2176
2177 if ((now - c_seg->c_creation_ts) >= vm_ripe_target_age) {
2178 lck_mtx_lock_spin_always(&c_seg->c_lock);
2179
2180 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
2181
2182 lck_mtx_unlock_always(&c_seg->c_lock);
2183 }
2184 c_seg = c_seg_next;
2185 }
2186 }
2187 vm_compressor_compact_and_swap(FALSE);
2188
2189 compaction_swapper_running = 0;
2190
2191 vm_swapout_ripe_segments = FALSE;
2192
2193 lck_mtx_unlock_always(c_list_lock);
2194
2195 thread_wakeup((event_t)&compaction_swapper_running);
2196 }
2197
2198
2199 void
2200 vm_consider_waking_compactor_swapper(void)
2201 {
2202 boolean_t need_wakeup = FALSE;
2203
2204 if (c_segment_count == 0) {
2205 return;
2206 }
2207
2208 if (compaction_swapper_running || compaction_swapper_awakened) {
2209 return;
2210 }
2211
2212 if (!compaction_swapper_inited && !compaction_swapper_init_now) {
2213 compaction_swapper_init_now = 1;
2214 need_wakeup = TRUE;
2215 }
2216
2217 if (c_minor_count && (COMPRESSOR_NEEDS_TO_MINOR_COMPACT())) {
2218 need_wakeup = TRUE;
2219 } else if (compressor_needs_to_swap()) {
2220 need_wakeup = TRUE;
2221 } else if (c_minor_count) {
2222 uint64_t total_bytes;
2223
2224 total_bytes = compressor_object->resident_page_count * PAGE_SIZE_64;
2225
2226 if ((total_bytes - compressor_bytes_used) > total_bytes / 10) {
2227 need_wakeup = TRUE;
2228 }
2229 }
2230 if (need_wakeup == TRUE) {
2231 lck_mtx_lock_spin_always(c_list_lock);
2232
2233 fastwake_warmup = FALSE;
2234
2235 if (compaction_swapper_running == 0 && compaction_swapper_awakened == 0) {
2236 memoryshot(VM_WAKEUP_COMPACTOR_SWAPPER, DBG_FUNC_NONE);
2237
2238 compaction_swapper_awakened = 1;
2239 thread_wakeup((event_t)&c_compressor_swap_trigger);
2240 }
2241 lck_mtx_unlock_always(c_list_lock);
2242 }
2243 }
2244
2245
2246 #define C_SWAPOUT_LIMIT 4
2247 #define DELAYED_COMPACTIONS_PER_PASS 30
2248
2249 void
2250 vm_compressor_do_delayed_compactions(boolean_t flush_all)
2251 {
2252 c_segment_t c_seg;
2253 int number_compacted = 0;
2254 boolean_t needs_to_swap = FALSE;
2255
2256
2257 #if !CONFIG_EMBEDDED
2258 LCK_MTX_ASSERT(c_list_lock, LCK_MTX_ASSERT_OWNED);
2259 #endif /* !CONFIG_EMBEDDED */
2260
2261 while (!queue_empty(&c_minor_list_head) && needs_to_swap == FALSE) {
2262 c_seg = (c_segment_t)queue_first(&c_minor_list_head);
2263
2264 lck_mtx_lock_spin_always(&c_seg->c_lock);
2265
2266 if (c_seg->c_busy) {
2267 lck_mtx_unlock_always(c_list_lock);
2268 c_seg_wait_on_busy(c_seg);
2269 lck_mtx_lock_spin_always(c_list_lock);
2270
2271 continue;
2272 }
2273 C_SEG_BUSY(c_seg);
2274
2275 c_seg_do_minor_compaction_and_unlock(c_seg, TRUE, FALSE, TRUE);
2276
2277 if (VM_CONFIG_SWAP_IS_ACTIVE && (number_compacted++ > DELAYED_COMPACTIONS_PER_PASS)) {
2278 if ((flush_all == TRUE || compressor_needs_to_swap() == TRUE) && c_swapout_count < C_SWAPOUT_LIMIT) {
2279 needs_to_swap = TRUE;
2280 }
2281
2282 number_compacted = 0;
2283 }
2284 lck_mtx_lock_spin_always(c_list_lock);
2285 }
2286 }
2287
2288
2289 #define C_SEGMENT_SWAPPEDIN_AGE_LIMIT 10
2290
2291 static void
2292 vm_compressor_age_swapped_in_segments(boolean_t flush_all)
2293 {
2294 c_segment_t c_seg;
2295 clock_sec_t now;
2296 clock_nsec_t nsec;
2297
2298 clock_get_system_nanotime(&now, &nsec);
2299
2300 while (!queue_empty(&c_swappedin_list_head)) {
2301 c_seg = (c_segment_t)queue_first(&c_swappedin_list_head);
2302
2303 if (flush_all == FALSE && (now - c_seg->c_swappedin_ts) < C_SEGMENT_SWAPPEDIN_AGE_LIMIT) {
2304 break;
2305 }
2306
2307 lck_mtx_lock_spin_always(&c_seg->c_lock);
2308
2309 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
2310
2311 lck_mtx_unlock_always(&c_seg->c_lock);
2312 }
2313 }
2314
2315
2316 extern int vm_num_swap_files;
2317 extern int vm_num_pinned_swap_files;
2318 extern int vm_swappin_enabled;
2319
2320 extern unsigned int vm_swapfile_total_segs_used;
2321 extern unsigned int vm_swapfile_total_segs_alloced;
2322
2323
2324 void
2325 vm_compressor_flush(void)
2326 {
2327 uint64_t vm_swap_put_failures_at_start;
2328 wait_result_t wait_result = 0;
2329 AbsoluteTime startTime, endTime;
2330 clock_sec_t now_sec;
2331 clock_nsec_t now_nsec;
2332 uint64_t nsec;
2333
2334 HIBLOG("vm_compressor_flush - starting\n");
2335
2336 clock_get_uptime(&startTime);
2337
2338 lck_mtx_lock_spin_always(c_list_lock);
2339
2340 fastwake_warmup = FALSE;
2341 compaction_swapper_abort = 1;
2342
2343 while (compaction_swapper_running) {
2344 assert_wait((event_t)&compaction_swapper_running, THREAD_UNINT);
2345
2346 lck_mtx_unlock_always(c_list_lock);
2347
2348 thread_block(THREAD_CONTINUE_NULL);
2349
2350 lck_mtx_lock_spin_always(c_list_lock);
2351 }
2352 compaction_swapper_abort = 0;
2353 compaction_swapper_running = 1;
2354
2355 hibernate_flushing = TRUE;
2356 hibernate_no_swapspace = FALSE;
2357 c_generation_id_flush_barrier = c_generation_id + 1000;
2358
2359 clock_get_system_nanotime(&now_sec, &now_nsec);
2360 hibernate_flushing_deadline = now_sec + HIBERNATE_FLUSHING_SECS_TO_COMPLETE;
2361
2362 vm_swap_put_failures_at_start = vm_swap_put_failures;
2363
2364 vm_compressor_compact_and_swap(TRUE);
2365
2366 while (!queue_empty(&c_swapout_list_head)) {
2367 assert_wait_timeout((event_t) &compaction_swapper_running, THREAD_INTERRUPTIBLE, 5000, 1000 * NSEC_PER_USEC);
2368
2369 lck_mtx_unlock_always(c_list_lock);
2370
2371 wait_result = thread_block(THREAD_CONTINUE_NULL);
2372
2373 lck_mtx_lock_spin_always(c_list_lock);
2374
2375 if (wait_result == THREAD_TIMED_OUT) {
2376 break;
2377 }
2378 }
2379 hibernate_flushing = FALSE;
2380 compaction_swapper_running = 0;
2381
2382 if (vm_swap_put_failures > vm_swap_put_failures_at_start) {
2383 HIBLOG("vm_compressor_flush failed to clean %llu segments - vm_page_compressor_count(%d)\n",
2384 vm_swap_put_failures - vm_swap_put_failures_at_start, VM_PAGE_COMPRESSOR_COUNT);
2385 }
2386
2387 lck_mtx_unlock_always(c_list_lock);
2388
2389 thread_wakeup((event_t)&compaction_swapper_running);
2390
2391 clock_get_uptime(&endTime);
2392 SUB_ABSOLUTETIME(&endTime, &startTime);
2393 absolutetime_to_nanoseconds(endTime, &nsec);
2394
2395 HIBLOG("vm_compressor_flush completed - took %qd msecs - vm_num_swap_files = %d, vm_num_pinned_swap_files = %d, vm_swappin_enabled = %d\n",
2396 nsec / 1000000ULL, vm_num_swap_files, vm_num_pinned_swap_files, vm_swappin_enabled);
2397 }
2398
2399
2400 int compaction_swap_trigger_thread_awakened = 0;
2401
2402 static void
2403 vm_compressor_swap_trigger_thread(void)
2404 {
2405 current_thread()->options |= TH_OPT_VMPRIV;
2406
2407 /*
2408 * compaction_swapper_init_now is set when the first call to
2409 * vm_consider_waking_compactor_swapper is made from
2410 * vm_pageout_scan... since this function is called upon
2411 * thread creation, we want to make sure to delay adjusting
2412 * the tuneables until we are awakened via vm_pageout_scan
2413 * so that we are at a point where the vm_swapfile_open will
2414 * be operating on the correct directory (in case the default
2415 * of /var/vm/ is overridden by the dymanic_pager
2416 */
2417 if (compaction_swapper_init_now) {
2418 vm_compaction_swapper_do_init();
2419
2420 if (vm_pageout_state.vm_restricted_to_single_processor == TRUE) {
2421 thread_vm_bind_group_add();
2422 }
2423 thread_set_thread_name(current_thread(), "VM_cswap_trigger");
2424 compaction_swapper_init_now = 0;
2425 }
2426 lck_mtx_lock_spin_always(c_list_lock);
2427
2428 compaction_swap_trigger_thread_awakened++;
2429 compaction_swapper_awakened = 0;
2430
2431 if (compaction_swapper_running == 0) {
2432 compaction_swapper_running = 1;
2433
2434 vm_compressor_compact_and_swap(FALSE);
2435
2436 compaction_swapper_running = 0;
2437 }
2438 assert_wait((event_t)&c_compressor_swap_trigger, THREAD_UNINT);
2439
2440 if (compaction_swapper_running == 0) {
2441 thread_wakeup((event_t)&compaction_swapper_running);
2442 }
2443
2444 lck_mtx_unlock_always(c_list_lock);
2445
2446 thread_block((thread_continue_t)vm_compressor_swap_trigger_thread);
2447
2448 /* NOTREACHED */
2449 }
2450
2451
2452 void
2453 vm_compressor_record_warmup_start(void)
2454 {
2455 c_segment_t c_seg;
2456
2457 lck_mtx_lock_spin_always(c_list_lock);
2458
2459 if (first_c_segment_to_warm_generation_id == 0) {
2460 if (!queue_empty(&c_age_list_head)) {
2461 c_seg = (c_segment_t)queue_last(&c_age_list_head);
2462
2463 first_c_segment_to_warm_generation_id = c_seg->c_generation_id;
2464 } else {
2465 first_c_segment_to_warm_generation_id = 0;
2466 }
2467
2468 fastwake_recording_in_progress = TRUE;
2469 }
2470 lck_mtx_unlock_always(c_list_lock);
2471 }
2472
2473
2474 void
2475 vm_compressor_record_warmup_end(void)
2476 {
2477 c_segment_t c_seg;
2478
2479 lck_mtx_lock_spin_always(c_list_lock);
2480
2481 if (fastwake_recording_in_progress == TRUE) {
2482 if (!queue_empty(&c_age_list_head)) {
2483 c_seg = (c_segment_t)queue_last(&c_age_list_head);
2484
2485 last_c_segment_to_warm_generation_id = c_seg->c_generation_id;
2486 } else {
2487 last_c_segment_to_warm_generation_id = first_c_segment_to_warm_generation_id;
2488 }
2489
2490 fastwake_recording_in_progress = FALSE;
2491
2492 HIBLOG("vm_compressor_record_warmup (%qd - %qd)\n", first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id);
2493 }
2494 lck_mtx_unlock_always(c_list_lock);
2495 }
2496
2497
2498 #define DELAY_TRIM_ON_WAKE_SECS 25
2499
2500 void
2501 vm_compressor_delay_trim(void)
2502 {
2503 clock_sec_t sec;
2504 clock_nsec_t nsec;
2505
2506 clock_get_system_nanotime(&sec, &nsec);
2507 dont_trim_until_ts = sec + DELAY_TRIM_ON_WAKE_SECS;
2508 }
2509
2510
2511 void
2512 vm_compressor_do_warmup(void)
2513 {
2514 lck_mtx_lock_spin_always(c_list_lock);
2515
2516 if (first_c_segment_to_warm_generation_id == last_c_segment_to_warm_generation_id) {
2517 first_c_segment_to_warm_generation_id = last_c_segment_to_warm_generation_id = 0;
2518
2519 lck_mtx_unlock_always(c_list_lock);
2520 return;
2521 }
2522
2523 if (compaction_swapper_running == 0 && compaction_swapper_awakened == 0) {
2524 fastwake_warmup = TRUE;
2525
2526 compaction_swapper_awakened = 1;
2527 thread_wakeup((event_t)&c_compressor_swap_trigger);
2528 }
2529 lck_mtx_unlock_always(c_list_lock);
2530 }
2531
2532 void
2533 do_fastwake_warmup_all(void)
2534 {
2535 lck_mtx_lock_spin_always(c_list_lock);
2536
2537 if (queue_empty(&c_swappedout_list_head) && queue_empty(&c_swappedout_sparse_list_head)) {
2538 lck_mtx_unlock_always(c_list_lock);
2539 return;
2540 }
2541
2542 fastwake_warmup = TRUE;
2543
2544 do_fastwake_warmup(&c_swappedout_list_head, TRUE);
2545
2546 do_fastwake_warmup(&c_swappedout_sparse_list_head, TRUE);
2547
2548 fastwake_warmup = FALSE;
2549
2550 lck_mtx_unlock_always(c_list_lock);
2551 }
2552
2553 void
2554 do_fastwake_warmup(queue_head_t *c_queue, boolean_t consider_all_cseg)
2555 {
2556 c_segment_t c_seg = NULL;
2557 AbsoluteTime startTime, endTime;
2558 uint64_t nsec;
2559
2560
2561 HIBLOG("vm_compressor_fastwake_warmup (%qd - %qd) - starting\n", first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id);
2562
2563 clock_get_uptime(&startTime);
2564
2565 lck_mtx_unlock_always(c_list_lock);
2566
2567 proc_set_thread_policy(current_thread(),
2568 TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER2);
2569
2570 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2571
2572 lck_mtx_lock_spin_always(c_list_lock);
2573
2574 while (!queue_empty(c_queue) && fastwake_warmup == TRUE) {
2575 c_seg = (c_segment_t) queue_first(c_queue);
2576
2577 if (consider_all_cseg == FALSE) {
2578 if (c_seg->c_generation_id < first_c_segment_to_warm_generation_id ||
2579 c_seg->c_generation_id > last_c_segment_to_warm_generation_id) {
2580 break;
2581 }
2582
2583 if (vm_page_free_count < (AVAILABLE_MEMORY / 4)) {
2584 break;
2585 }
2586 }
2587
2588 lck_mtx_lock_spin_always(&c_seg->c_lock);
2589 lck_mtx_unlock_always(c_list_lock);
2590
2591 if (c_seg->c_busy) {
2592 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2593 c_seg_wait_on_busy(c_seg);
2594 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2595 } else {
2596 if (c_seg_swapin(c_seg, TRUE, FALSE) == 0) {
2597 lck_mtx_unlock_always(&c_seg->c_lock);
2598 }
2599 c_segment_warmup_count++;
2600
2601 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2602 vm_pageout_io_throttle();
2603 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2604 }
2605 lck_mtx_lock_spin_always(c_list_lock);
2606 }
2607 lck_mtx_unlock_always(c_list_lock);
2608
2609 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2610
2611 proc_set_thread_policy(current_thread(),
2612 TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER0);
2613
2614 clock_get_uptime(&endTime);
2615 SUB_ABSOLUTETIME(&endTime, &startTime);
2616 absolutetime_to_nanoseconds(endTime, &nsec);
2617
2618 HIBLOG("vm_compressor_fastwake_warmup completed - took %qd msecs\n", nsec / 1000000ULL);
2619
2620 lck_mtx_lock_spin_always(c_list_lock);
2621
2622 if (consider_all_cseg == FALSE) {
2623 first_c_segment_to_warm_generation_id = last_c_segment_to_warm_generation_id = 0;
2624 }
2625 }
2626
2627
2628 void
2629 vm_compressor_compact_and_swap(boolean_t flush_all)
2630 {
2631 c_segment_t c_seg, c_seg_next;
2632 boolean_t keep_compacting;
2633 clock_sec_t now;
2634 clock_nsec_t nsec;
2635
2636
2637 if (fastwake_warmup == TRUE) {
2638 uint64_t starting_warmup_count;
2639
2640 starting_warmup_count = c_segment_warmup_count;
2641
2642 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 11) | DBG_FUNC_START, c_segment_warmup_count,
2643 first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id, 0, 0);
2644 do_fastwake_warmup(&c_swappedout_list_head, FALSE);
2645 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 11) | DBG_FUNC_END, c_segment_warmup_count, c_segment_warmup_count - starting_warmup_count, 0, 0, 0);
2646
2647 fastwake_warmup = FALSE;
2648 }
2649
2650 /*
2651 * it's possible for the c_age_list_head to be empty if we
2652 * hit our limits for growing the compressor pool and we subsequently
2653 * hibernated... on the next hibernation we could see the queue as
2654 * empty and not proceeed even though we have a bunch of segments on
2655 * the swapped in queue that need to be dealt with.
2656 */
2657 vm_compressor_do_delayed_compactions(flush_all);
2658
2659 vm_compressor_age_swapped_in_segments(flush_all);
2660
2661 /*
2662 * we only need to grab the timestamp once per
2663 * invocation of this function since the
2664 * timescale we're interested in is measured
2665 * in days
2666 */
2667 clock_get_system_nanotime(&now, &nsec);
2668
2669 while (!queue_empty(&c_age_list_head) && compaction_swapper_abort == 0) {
2670 if (hibernate_flushing == TRUE) {
2671 clock_sec_t sec;
2672
2673 if (hibernate_should_abort()) {
2674 HIBLOG("vm_compressor_flush - hibernate_should_abort returned TRUE\n");
2675 break;
2676 }
2677 if (hibernate_no_swapspace == TRUE) {
2678 HIBLOG("vm_compressor_flush - out of swap space\n");
2679 break;
2680 }
2681 if (vm_swap_files_pinned() == FALSE) {
2682 HIBLOG("vm_compressor_flush - unpinned swap files\n");
2683 break;
2684 }
2685 if (hibernate_in_progress_with_pinned_swap == TRUE &&
2686 (vm_swapfile_total_segs_alloced == vm_swapfile_total_segs_used)) {
2687 HIBLOG("vm_compressor_flush - out of pinned swap space\n");
2688 break;
2689 }
2690 clock_get_system_nanotime(&sec, &nsec);
2691
2692 if (sec > hibernate_flushing_deadline) {
2693 HIBLOG("vm_compressor_flush - failed to finish before deadline\n");
2694 break;
2695 }
2696 }
2697 if (c_swapout_count >= C_SWAPOUT_LIMIT) {
2698 assert_wait_timeout((event_t) &compaction_swapper_running, THREAD_INTERRUPTIBLE, 100, 1000 * NSEC_PER_USEC);
2699
2700 lck_mtx_unlock_always(c_list_lock);
2701
2702 thread_block(THREAD_CONTINUE_NULL);
2703
2704 lck_mtx_lock_spin_always(c_list_lock);
2705 }
2706 /*
2707 * Minor compactions
2708 */
2709 vm_compressor_do_delayed_compactions(flush_all);
2710
2711 vm_compressor_age_swapped_in_segments(flush_all);
2712
2713 if (c_swapout_count >= C_SWAPOUT_LIMIT) {
2714 /*
2715 * we timed out on the above thread_block
2716 * let's loop around and try again
2717 * the timeout allows us to continue
2718 * to do minor compactions to make
2719 * more memory available
2720 */
2721 continue;
2722 }
2723
2724 /*
2725 * Swap out segments?
2726 */
2727 if (flush_all == FALSE) {
2728 boolean_t needs_to_swap;
2729
2730 lck_mtx_unlock_always(c_list_lock);
2731
2732 needs_to_swap = compressor_needs_to_swap();
2733
2734 #if !CONFIG_EMBEDDED
2735 if (needs_to_swap == TRUE && vm_swap_low_on_space()) {
2736 vm_compressor_take_paging_space_action();
2737 }
2738 #endif /* !CONFIG_EMBEDDED */
2739
2740 lck_mtx_lock_spin_always(c_list_lock);
2741
2742 if (needs_to_swap == FALSE) {
2743 break;
2744 }
2745 }
2746 if (queue_empty(&c_age_list_head)) {
2747 break;
2748 }
2749 c_seg = (c_segment_t) queue_first(&c_age_list_head);
2750
2751 assert(c_seg->c_state == C_ON_AGE_Q);
2752
2753 if (flush_all == TRUE && c_seg->c_generation_id > c_generation_id_flush_barrier) {
2754 break;
2755 }
2756
2757 lck_mtx_lock_spin_always(&c_seg->c_lock);
2758
2759 if (c_seg->c_busy) {
2760 lck_mtx_unlock_always(c_list_lock);
2761 c_seg_wait_on_busy(c_seg);
2762 lck_mtx_lock_spin_always(c_list_lock);
2763
2764 continue;
2765 }
2766 C_SEG_BUSY(c_seg);
2767
2768 if (c_seg_do_minor_compaction_and_unlock(c_seg, FALSE, TRUE, TRUE)) {
2769 /*
2770 * found an empty c_segment and freed it
2771 * so go grab the next guy in the queue
2772 */
2773 c_seg_major_compact_stats.count_of_freed_segs++;
2774 continue;
2775 }
2776 /*
2777 * Major compaction
2778 */
2779 keep_compacting = TRUE;
2780
2781 while (keep_compacting == TRUE) {
2782 assert(c_seg->c_busy);
2783
2784 /* look for another segment to consolidate */
2785
2786 c_seg_next = (c_segment_t) queue_next(&c_seg->c_age_list);
2787
2788 if (queue_end(&c_age_list_head, (queue_entry_t)c_seg_next)) {
2789 break;
2790 }
2791
2792 assert(c_seg_next->c_state == C_ON_AGE_Q);
2793
2794 if (c_seg_major_compact_ok(c_seg, c_seg_next) == FALSE) {
2795 break;
2796 }
2797
2798 lck_mtx_lock_spin_always(&c_seg_next->c_lock);
2799
2800 if (c_seg_next->c_busy) {
2801 lck_mtx_unlock_always(c_list_lock);
2802 c_seg_wait_on_busy(c_seg_next);
2803 lck_mtx_lock_spin_always(c_list_lock);
2804
2805 continue;
2806 }
2807 /* grab that segment */
2808 C_SEG_BUSY(c_seg_next);
2809
2810 if (c_seg_do_minor_compaction_and_unlock(c_seg_next, FALSE, TRUE, TRUE)) {
2811 /*
2812 * found an empty c_segment and freed it
2813 * so we can't continue to use c_seg_next
2814 */
2815 c_seg_major_compact_stats.count_of_freed_segs++;
2816 continue;
2817 }
2818
2819 /* unlock the list ... */
2820 lck_mtx_unlock_always(c_list_lock);
2821
2822 /* do the major compaction */
2823
2824 keep_compacting = c_seg_major_compact(c_seg, c_seg_next);
2825
2826 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2827
2828 lck_mtx_lock_spin_always(&c_seg_next->c_lock);
2829 /*
2830 * run a minor compaction on the donor segment
2831 * since we pulled at least some of it's
2832 * data into our target... if we've emptied
2833 * it, now is a good time to free it which
2834 * c_seg_minor_compaction_and_unlock also takes care of
2835 *
2836 * by passing TRUE, we ask for c_busy to be cleared
2837 * and c_wanted to be taken care of
2838 */
2839 if (c_seg_minor_compaction_and_unlock(c_seg_next, TRUE)) {
2840 c_seg_major_compact_stats.count_of_freed_segs++;
2841 }
2842
2843 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2844
2845 /* relock the list */
2846 lck_mtx_lock_spin_always(c_list_lock);
2847 } /* major compaction */
2848
2849 lck_mtx_lock_spin_always(&c_seg->c_lock);
2850
2851 assert(c_seg->c_busy);
2852 assert(!c_seg->c_on_minorcompact_q);
2853
2854 if (VM_CONFIG_SWAP_IS_ACTIVE) {
2855 /*
2856 * This mode of putting a generic c_seg on the swapout list is
2857 * only supported when we have general swapping enabled
2858 */
2859 c_seg_switch_state(c_seg, C_ON_SWAPOUT_Q, FALSE);
2860 } else {
2861 if ((vm_swapout_ripe_segments == TRUE && c_overage_swapped_count < c_overage_swapped_limit)) {
2862 assert(VM_CONFIG_SWAP_IS_PRESENT);
2863 /*
2864 * we are running compressor sweeps with swap-behind
2865 * make sure the c_seg has aged enough before swapping it
2866 * out...
2867 */
2868 if ((now - c_seg->c_creation_ts) >= vm_ripe_target_age) {
2869 c_seg->c_overage_swap = TRUE;
2870 c_overage_swapped_count++;
2871 c_seg_switch_state(c_seg, C_ON_SWAPOUT_Q, FALSE);
2872 }
2873 }
2874 }
2875 if (c_seg->c_state == C_ON_AGE_Q) {
2876 /*
2877 * this c_seg didn't get moved to the swapout queue
2878 * so we need to move it out of the way...
2879 * we just did a major compaction on it so put it
2880 * on that queue
2881 */
2882 c_seg_switch_state(c_seg, C_ON_MAJORCOMPACT_Q, FALSE);
2883 } else {
2884 c_seg_major_compact_stats.wasted_space_in_swapouts += C_SEG_BUFSIZE - c_seg->c_bytes_used;
2885 c_seg_major_compact_stats.count_of_swapouts++;
2886 }
2887 C_SEG_WAKEUP_DONE(c_seg);
2888
2889 lck_mtx_unlock_always(&c_seg->c_lock);
2890
2891 if (c_swapout_count) {
2892 lck_mtx_unlock_always(c_list_lock);
2893
2894 thread_wakeup((event_t)&c_swapout_list_head);
2895
2896 lck_mtx_lock_spin_always(c_list_lock);
2897 }
2898 }
2899 }
2900
2901
2902 static c_segment_t
2903 c_seg_allocate(c_segment_t *current_chead)
2904 {
2905 c_segment_t c_seg;
2906 int min_needed;
2907 int size_to_populate;
2908
2909 #if !CONFIG_EMBEDDED
2910 if (vm_compressor_low_on_space()) {
2911 vm_compressor_take_paging_space_action();
2912 }
2913 #endif /* !CONFIG_EMBEDDED */
2914
2915 if ((c_seg = *current_chead) == NULL) {
2916 uint32_t c_segno;
2917
2918 lck_mtx_lock_spin_always(c_list_lock);
2919
2920 while (c_segments_busy == TRUE) {
2921 assert_wait((event_t) (&c_segments_busy), THREAD_UNINT);
2922
2923 lck_mtx_unlock_always(c_list_lock);
2924
2925 thread_block(THREAD_CONTINUE_NULL);
2926
2927 lck_mtx_lock_spin_always(c_list_lock);
2928 }
2929 if (c_free_segno_head == (uint32_t)-1) {
2930 uint32_t c_segments_available_new;
2931
2932 if (c_segments_available >= c_segments_limit || c_segment_pages_compressed >= c_segment_pages_compressed_limit) {
2933 lck_mtx_unlock_always(c_list_lock);
2934
2935 return NULL;
2936 }
2937 c_segments_busy = TRUE;
2938 lck_mtx_unlock_always(c_list_lock);
2939
2940 kernel_memory_populate(compressor_map, (vm_offset_t)c_segments_next_page,
2941 PAGE_SIZE, KMA_KOBJECT, VM_KERN_MEMORY_COMPRESSOR);
2942 c_segments_next_page += PAGE_SIZE;
2943
2944 c_segments_available_new = c_segments_available + C_SEGMENTS_PER_PAGE;
2945
2946 if (c_segments_available_new > c_segments_limit) {
2947 c_segments_available_new = c_segments_limit;
2948 }
2949
2950 for (c_segno = c_segments_available + 1; c_segno < c_segments_available_new; c_segno++) {
2951 c_segments[c_segno - 1].c_segno = c_segno;
2952 }
2953
2954 lck_mtx_lock_spin_always(c_list_lock);
2955
2956 c_segments[c_segno - 1].c_segno = c_free_segno_head;
2957 c_free_segno_head = c_segments_available;
2958 c_segments_available = c_segments_available_new;
2959
2960 c_segments_busy = FALSE;
2961 thread_wakeup((event_t) (&c_segments_busy));
2962 }
2963 c_segno = c_free_segno_head;
2964 assert(c_segno >= 0 && c_segno < c_segments_limit);
2965
2966 c_free_segno_head = (uint32_t)c_segments[c_segno].c_segno;
2967
2968 /*
2969 * do the rest of the bookkeeping now while we're still behind
2970 * the list lock and grab our generation id now into a local
2971 * so that we can install it once we have the c_seg allocated
2972 */
2973 c_segment_count++;
2974 if (c_segment_count > c_segment_count_max) {
2975 c_segment_count_max = c_segment_count;
2976 }
2977
2978 lck_mtx_unlock_always(c_list_lock);
2979
2980 c_seg = (c_segment_t)zalloc(compressor_segment_zone);
2981 bzero((char *)c_seg, sizeof(struct c_segment));
2982
2983 c_seg->c_store.c_buffer = (int32_t *)C_SEG_BUFFER_ADDRESS(c_segno);
2984
2985 lck_mtx_init(&c_seg->c_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
2986
2987 c_seg->c_state = C_IS_EMPTY;
2988 c_seg->c_firstemptyslot = C_SLOT_MAX_INDEX;
2989 c_seg->c_mysegno = c_segno;
2990
2991 lck_mtx_lock_spin_always(c_list_lock);
2992 c_empty_count++;
2993 c_seg_switch_state(c_seg, C_IS_FILLING, FALSE);
2994 c_segments[c_segno].c_seg = c_seg;
2995 assert(c_segments[c_segno].c_segno > c_segments_available);
2996 lck_mtx_unlock_always(c_list_lock);
2997
2998 *current_chead = c_seg;
2999
3000 #if DEVELOPMENT || DEBUG
3001 C_SEG_MAKE_WRITEABLE(c_seg);
3002 #endif
3003 }
3004 c_seg_alloc_nextslot(c_seg);
3005
3006 size_to_populate = C_SEG_ALLOCSIZE - C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset);
3007
3008 if (size_to_populate) {
3009 min_needed = PAGE_SIZE + (C_SEG_ALLOCSIZE - C_SEG_BUFSIZE);
3010
3011 if (C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset - c_seg->c_nextoffset) < (unsigned) min_needed) {
3012 if (size_to_populate > C_SEG_MAX_POPULATE_SIZE) {
3013 size_to_populate = C_SEG_MAX_POPULATE_SIZE;
3014 }
3015
3016 OSAddAtomic64(size_to_populate / PAGE_SIZE, &vm_pageout_vminfo.vm_compressor_pages_grabbed);
3017
3018 kernel_memory_populate(compressor_map,
3019 (vm_offset_t) &c_seg->c_store.c_buffer[c_seg->c_populated_offset],
3020 size_to_populate,
3021 KMA_COMPRESSOR,
3022 VM_KERN_MEMORY_COMPRESSOR);
3023 } else {
3024 size_to_populate = 0;
3025 }
3026 }
3027 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3028
3029 lck_mtx_lock_spin_always(&c_seg->c_lock);
3030
3031 if (size_to_populate) {
3032 c_seg->c_populated_offset += C_SEG_BYTES_TO_OFFSET(size_to_populate);
3033 }
3034
3035 return c_seg;
3036 }
3037
3038 #if DEVELOPMENT || DEBUG
3039 #if CONFIG_FREEZE
3040 extern boolean_t memorystatus_freeze_to_memory;
3041 #endif /* CONFIG_FREEZE */
3042 #endif /* DEVELOPMENT || DEBUG */
3043
3044 static void
3045 c_current_seg_filled(c_segment_t c_seg, c_segment_t *current_chead)
3046 {
3047 uint32_t unused_bytes;
3048 uint32_t offset_to_depopulate;
3049 int new_state = C_ON_AGE_Q;
3050 clock_sec_t sec;
3051 clock_nsec_t nsec;
3052 boolean_t head_insert = FALSE;
3053
3054 unused_bytes = trunc_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset - c_seg->c_nextoffset));
3055
3056 if (unused_bytes) {
3057 offset_to_depopulate = C_SEG_BYTES_TO_OFFSET(round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_nextoffset)));
3058
3059 /*
3060 * release the extra physical page(s) at the end of the segment
3061 */
3062 lck_mtx_unlock_always(&c_seg->c_lock);
3063
3064 kernel_memory_depopulate(
3065 compressor_map,
3066 (vm_offset_t) &c_seg->c_store.c_buffer[offset_to_depopulate],
3067 unused_bytes,
3068 KMA_COMPRESSOR);
3069
3070 lck_mtx_lock_spin_always(&c_seg->c_lock);
3071
3072 c_seg->c_populated_offset = offset_to_depopulate;
3073 }
3074 assert(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset) <= C_SEG_BUFSIZE);
3075
3076 #if DEVELOPMENT || DEBUG
3077 {
3078 boolean_t c_seg_was_busy = FALSE;
3079
3080 if (!c_seg->c_busy) {
3081 C_SEG_BUSY(c_seg);
3082 } else {
3083 c_seg_was_busy = TRUE;
3084 }
3085
3086 lck_mtx_unlock_always(&c_seg->c_lock);
3087
3088 C_SEG_WRITE_PROTECT(c_seg);
3089
3090 lck_mtx_lock_spin_always(&c_seg->c_lock);
3091
3092 if (c_seg_was_busy == FALSE) {
3093 C_SEG_WAKEUP_DONE(c_seg);
3094 }
3095 }
3096 #endif
3097
3098 #if CONFIG_FREEZE
3099 if (current_chead == (c_segment_t*)&freezer_chead &&
3100 VM_CONFIG_SWAP_IS_PRESENT &&
3101 VM_CONFIG_FREEZER_SWAP_IS_ACTIVE
3102 #if DEVELOPMENT || DEBUG
3103 && !memorystatus_freeze_to_memory
3104 #endif /* DEVELOPMENT || DEBUG */
3105 ) {
3106 new_state = C_ON_SWAPOUT_Q;
3107 }
3108 #endif /* CONFIG_FREEZE */
3109
3110 if (vm_darkwake_mode == TRUE) {
3111 new_state = C_ON_SWAPOUT_Q;
3112 head_insert = TRUE;
3113 }
3114
3115 clock_get_system_nanotime(&sec, &nsec);
3116 c_seg->c_creation_ts = (uint32_t)sec;
3117
3118 lck_mtx_lock_spin_always(c_list_lock);
3119
3120 c_seg->c_generation_id = c_generation_id++;
3121 c_seg_switch_state(c_seg, new_state, head_insert);
3122
3123 #if CONFIG_FREEZE
3124 if (c_seg->c_state == C_ON_SWAPOUT_Q) {
3125 /*
3126 * darkwake and freezer can't co-exist together
3127 * We'll need to fix this accounting as a start.
3128 */
3129 assert(vm_darkwake_mode == FALSE);
3130 c_freezer_swapout_page_count += (C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset)) / PAGE_SIZE_64;
3131 }
3132 #endif /* CONFIG_FREEZE */
3133
3134 if (c_seg->c_state == C_ON_AGE_Q && C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE) {
3135 c_seg_need_delayed_compaction(c_seg, TRUE);
3136 }
3137
3138 lck_mtx_unlock_always(c_list_lock);
3139
3140 if (c_seg->c_state == C_ON_SWAPOUT_Q) {
3141 thread_wakeup((event_t)&c_swapout_list_head);
3142 }
3143
3144 *current_chead = NULL;
3145 }
3146
3147
3148 /*
3149 * returns with c_seg locked
3150 */
3151 void
3152 c_seg_swapin_requeue(c_segment_t c_seg, boolean_t has_data, boolean_t minor_compact_ok, boolean_t age_on_swapin_q)
3153 {
3154 clock_sec_t sec;
3155 clock_nsec_t nsec;
3156
3157 clock_get_system_nanotime(&sec, &nsec);
3158
3159 lck_mtx_lock_spin_always(c_list_lock);
3160 lck_mtx_lock_spin_always(&c_seg->c_lock);
3161
3162 assert(c_seg->c_busy_swapping);
3163 assert(c_seg->c_busy);
3164
3165 c_seg->c_busy_swapping = 0;
3166
3167 if (c_seg->c_overage_swap == TRUE) {
3168 c_overage_swapped_count--;
3169 c_seg->c_overage_swap = FALSE;
3170 }
3171 if (has_data == TRUE) {
3172 if (age_on_swapin_q == TRUE) {
3173 c_seg_switch_state(c_seg, C_ON_SWAPPEDIN_Q, FALSE);
3174 } else {
3175 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
3176 }
3177
3178 if (minor_compact_ok == TRUE && !c_seg->c_on_minorcompact_q && C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE) {
3179 c_seg_need_delayed_compaction(c_seg, TRUE);
3180 }
3181 } else {
3182 c_seg->c_store.c_buffer = (int32_t*) NULL;
3183 c_seg->c_populated_offset = C_SEG_BYTES_TO_OFFSET(0);
3184
3185 c_seg_switch_state(c_seg, C_ON_BAD_Q, FALSE);
3186 }
3187 c_seg->c_swappedin_ts = (uint32_t)sec;
3188
3189 lck_mtx_unlock_always(c_list_lock);
3190 }
3191
3192
3193
3194 /*
3195 * c_seg has to be locked and is returned locked if the c_seg isn't freed
3196 * PAGE_REPLACMENT_DISALLOWED has to be TRUE on entry and is returned TRUE
3197 * c_seg_swapin returns 1 if the c_seg was freed, 0 otherwise
3198 */
3199
3200 int
3201 c_seg_swapin(c_segment_t c_seg, boolean_t force_minor_compaction, boolean_t age_on_swapin_q)
3202 {
3203 vm_offset_t addr = 0;
3204 uint32_t io_size = 0;
3205 uint64_t f_offset;
3206
3207 assert(C_SEG_IS_ONDISK(c_seg));
3208
3209 #if !CHECKSUM_THE_SWAP
3210 c_seg_trim_tail(c_seg);
3211 #endif
3212 io_size = round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset));
3213 f_offset = c_seg->c_store.c_swap_handle;
3214
3215 C_SEG_BUSY(c_seg);
3216 c_seg->c_busy_swapping = 1;
3217
3218 /*
3219 * This thread is likely going to block for I/O.
3220 * Make sure it is ready to run when the I/O completes because
3221 * it needs to clear the busy bit on the c_seg so that other
3222 * waiting threads can make progress too. To do that, boost
3223 * the rwlock_count so that the priority is boosted.
3224 */
3225 set_thread_rwlock_boost();
3226 lck_mtx_unlock_always(&c_seg->c_lock);
3227
3228 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3229
3230 addr = (vm_offset_t)C_SEG_BUFFER_ADDRESS(c_seg->c_mysegno);
3231 c_seg->c_store.c_buffer = (int32_t*) addr;
3232
3233 kernel_memory_populate(compressor_map, addr, io_size, KMA_COMPRESSOR, VM_KERN_MEMORY_COMPRESSOR);
3234
3235 if (vm_swap_get(c_seg, f_offset, io_size) != KERN_SUCCESS) {
3236 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3237
3238 kernel_memory_depopulate(compressor_map, addr, io_size, KMA_COMPRESSOR);
3239
3240 c_seg_swapin_requeue(c_seg, FALSE, TRUE, age_on_swapin_q);
3241 } else {
3242 #if ENCRYPTED_SWAP
3243 vm_swap_decrypt(c_seg);
3244 #endif /* ENCRYPTED_SWAP */
3245
3246 #if CHECKSUM_THE_SWAP
3247 if (c_seg->cseg_swap_size != io_size) {
3248 panic("swapin size doesn't match swapout size");
3249 }
3250
3251 if (c_seg->cseg_hash != vmc_hash((char*) c_seg->c_store.c_buffer, (int)io_size)) {
3252 panic("c_seg_swapin - Swap hash mismatch\n");
3253 }
3254 #endif /* CHECKSUM_THE_SWAP */
3255
3256 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3257
3258 c_seg_swapin_requeue(c_seg, TRUE, force_minor_compaction == TRUE ? FALSE : TRUE, age_on_swapin_q);
3259
3260 OSAddAtomic64(c_seg->c_bytes_used, &compressor_bytes_used);
3261
3262 if (force_minor_compaction == TRUE) {
3263 if (c_seg_minor_compaction_and_unlock(c_seg, FALSE)) {
3264 /*
3265 * c_seg was completely empty so it was freed,
3266 * so be careful not to reference it again
3267 *
3268 * Drop the rwlock_count so that the thread priority
3269 * is returned back to where it is supposed to be.
3270 */
3271 clear_thread_rwlock_boost();
3272 return 1;
3273 }
3274
3275 lck_mtx_lock_spin_always(&c_seg->c_lock);
3276 }
3277 }
3278 C_SEG_WAKEUP_DONE(c_seg);
3279
3280 /*
3281 * Drop the rwlock_count so that the thread priority
3282 * is returned back to where it is supposed to be.
3283 */
3284 clear_thread_rwlock_boost();
3285
3286 return 0;
3287 }
3288
3289
3290 static void
3291 c_segment_sv_hash_drop_ref(int hash_indx)
3292 {
3293 struct c_sv_hash_entry o_sv_he, n_sv_he;
3294
3295 while (1) {
3296 o_sv_he.he_record = c_segment_sv_hash_table[hash_indx].he_record;
3297
3298 n_sv_he.he_ref = o_sv_he.he_ref - 1;
3299 n_sv_he.he_data = o_sv_he.he_data;
3300
3301 if (OSCompareAndSwap64((UInt64)o_sv_he.he_record, (UInt64)n_sv_he.he_record, (UInt64 *) &c_segment_sv_hash_table[hash_indx].he_record) == TRUE) {
3302 if (n_sv_he.he_ref == 0) {
3303 OSAddAtomic(-1, &c_segment_svp_in_hash);
3304 }
3305 break;
3306 }
3307 }
3308 }
3309
3310
3311 static int
3312 c_segment_sv_hash_insert(uint32_t data)
3313 {
3314 int hash_sindx;
3315 int misses;
3316 struct c_sv_hash_entry o_sv_he, n_sv_he;
3317 boolean_t got_ref = FALSE;
3318
3319 if (data == 0) {
3320 OSAddAtomic(1, &c_segment_svp_zero_compressions);
3321 } else {
3322 OSAddAtomic(1, &c_segment_svp_nonzero_compressions);
3323 }
3324
3325 hash_sindx = data & C_SV_HASH_MASK;
3326
3327 for (misses = 0; misses < C_SV_HASH_MAX_MISS; misses++) {
3328 o_sv_he.he_record = c_segment_sv_hash_table[hash_sindx].he_record;
3329
3330 while (o_sv_he.he_data == data || o_sv_he.he_ref == 0) {
3331 n_sv_he.he_ref = o_sv_he.he_ref + 1;
3332 n_sv_he.he_data = data;
3333
3334 if (OSCompareAndSwap64((UInt64)o_sv_he.he_record, (UInt64)n_sv_he.he_record, (UInt64 *) &c_segment_sv_hash_table[hash_sindx].he_record) == TRUE) {
3335 if (n_sv_he.he_ref == 1) {
3336 OSAddAtomic(1, &c_segment_svp_in_hash);
3337 }
3338 got_ref = TRUE;
3339 break;
3340 }
3341 o_sv_he.he_record = c_segment_sv_hash_table[hash_sindx].he_record;
3342 }
3343 if (got_ref == TRUE) {
3344 break;
3345 }
3346 hash_sindx++;
3347
3348 if (hash_sindx == C_SV_HASH_SIZE) {
3349 hash_sindx = 0;
3350 }
3351 }
3352 if (got_ref == FALSE) {
3353 return -1;
3354 }
3355
3356 return hash_sindx;
3357 }
3358
3359
3360 #if RECORD_THE_COMPRESSED_DATA
3361
3362 static void
3363 c_compressed_record_data(char *src, int c_size)
3364 {
3365 if ((c_compressed_record_cptr + c_size + 4) >= c_compressed_record_ebuf) {
3366 panic("c_compressed_record_cptr >= c_compressed_record_ebuf");
3367 }
3368
3369 *(int *)((void *)c_compressed_record_cptr) = c_size;
3370
3371 c_compressed_record_cptr += 4;
3372
3373 memcpy(c_compressed_record_cptr, src, c_size);
3374 c_compressed_record_cptr += c_size;
3375 }
3376 #endif
3377
3378
3379 static int
3380 c_compress_page(char *src, c_slot_mapping_t slot_ptr, c_segment_t *current_chead, char *scratch_buf)
3381 {
3382 int c_size;
3383 int c_rounded_size = 0;
3384 int max_csize;
3385 c_slot_t cs;
3386 c_segment_t c_seg;
3387
3388 KERNEL_DEBUG(0xe0400000 | DBG_FUNC_START, *current_chead, 0, 0, 0, 0);
3389 retry:
3390 if ((c_seg = c_seg_allocate(current_chead)) == NULL) {
3391 return 1;
3392 }
3393 /*
3394 * returns with c_seg lock held
3395 * and PAGE_REPLACEMENT_DISALLOWED(TRUE)...
3396 * c_nextslot has been allocated and
3397 * c_store.c_buffer populated
3398 */
3399 assert(c_seg->c_state == C_IS_FILLING);
3400
3401 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_seg->c_nextslot);
3402
3403 cs->c_packed_ptr = C_SLOT_PACK_PTR(slot_ptr);
3404 assert(slot_ptr == (c_slot_mapping_t)C_SLOT_UNPACK_PTR(cs));
3405
3406 cs->c_offset = c_seg->c_nextoffset;
3407
3408 max_csize = C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES((int32_t)cs->c_offset);
3409
3410 if (max_csize > PAGE_SIZE) {
3411 max_csize = PAGE_SIZE;
3412 }
3413
3414 #if CHECKSUM_THE_DATA
3415 cs->c_hash_data = vmc_hash(src, PAGE_SIZE);
3416 #endif
3417 boolean_t incomp_copy = FALSE;
3418 int max_csize_adj = (max_csize - 4);
3419
3420 if (vm_compressor_algorithm() != VM_COMPRESSOR_DEFAULT_CODEC) {
3421 #if defined(__arm__) || defined(__arm64__)
3422 uint16_t ccodec = CINVALID;
3423
3424 if (max_csize >= C_SEG_OFFSET_ALIGNMENT_BOUNDARY) {
3425 c_size = metacompressor((const uint8_t *) src,
3426 (uint8_t *) &c_seg->c_store.c_buffer[cs->c_offset],
3427 max_csize_adj, &ccodec,
3428 scratch_buf, &incomp_copy);
3429 #if C_SEG_OFFSET_ALIGNMENT_BOUNDARY > 4
3430 if (c_size > max_csize_adj) {
3431 c_size = -1;
3432 }
3433 #endif
3434 } else {
3435 c_size = -1;
3436 }
3437 assert(ccodec == CCWK || ccodec == CCLZ4);
3438 cs->c_codec = ccodec;
3439 #endif
3440 } else {
3441 #if defined(__arm__) || defined(__arm64__)
3442 cs->c_codec = CCWK;
3443 #endif
3444 #if defined(__arm64__)
3445 __unreachable_ok_push
3446 if (PAGE_SIZE == 4096) {
3447 c_size = WKdm_compress_4k((WK_word *)(uintptr_t)src, (WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
3448 (WK_word *)(uintptr_t)scratch_buf, max_csize_adj);
3449 } else {
3450 c_size = WKdm_compress_16k((WK_word *)(uintptr_t)src, (WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
3451 (WK_word *)(uintptr_t)scratch_buf, max_csize_adj);
3452 }
3453 __unreachable_ok_pop
3454 #else
3455 c_size = WKdm_compress_new((const WK_word *)(uintptr_t)src, (WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
3456 (WK_word *)(uintptr_t)scratch_buf, max_csize_adj);
3457 #endif
3458 }
3459 assertf(((c_size <= max_csize_adj) && (c_size >= -1)),
3460 "c_size invalid (%d, %d), cur compressions: %d", c_size, max_csize_adj, c_segment_pages_compressed);
3461
3462 if (c_size == -1) {
3463 if (max_csize < PAGE_SIZE) {
3464 c_current_seg_filled(c_seg, current_chead);
3465 assert(*current_chead == NULL);
3466
3467 lck_mtx_unlock_always(&c_seg->c_lock);
3468 /* TODO: it may be worth requiring codecs to distinguish
3469 * between incompressible inputs and failures due to
3470 * budget exhaustion.
3471 */
3472 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3473 goto retry;
3474 }
3475 c_size = PAGE_SIZE;
3476
3477 if (incomp_copy == FALSE) {
3478 memcpy(&c_seg->c_store.c_buffer[cs->c_offset], src, c_size);
3479 }
3480
3481 OSAddAtomic(1, &c_segment_noncompressible_pages);
3482 } else if (c_size == 0) {
3483 int hash_index;
3484
3485 /*
3486 * special case - this is a page completely full of a single 32 bit value
3487 */
3488 hash_index = c_segment_sv_hash_insert(*(uint32_t *)(uintptr_t)src);
3489
3490 if (hash_index != -1) {
3491 slot_ptr->s_cindx = hash_index;
3492 slot_ptr->s_cseg = C_SV_CSEG_ID;
3493
3494 OSAddAtomic(1, &c_segment_svp_hash_succeeded);
3495 #if RECORD_THE_COMPRESSED_DATA
3496 c_compressed_record_data(src, 4);
3497 #endif
3498 goto sv_compression;
3499 }
3500 c_size = 4;
3501
3502 memcpy(&c_seg->c_store.c_buffer[cs->c_offset], src, c_size);
3503
3504 OSAddAtomic(1, &c_segment_svp_hash_failed);
3505 }
3506
3507 #if RECORD_THE_COMPRESSED_DATA
3508 c_compressed_record_data((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size);
3509 #endif
3510 #if CHECKSUM_THE_COMPRESSED_DATA
3511 cs->c_hash_compressed_data = vmc_hash((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size);
3512 #endif
3513 #if POPCOUNT_THE_COMPRESSED_DATA
3514 cs->c_pop_cdata = vmc_pop((uintptr_t) &c_seg->c_store.c_buffer[cs->c_offset], c_size);
3515 #endif
3516 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
3517
3518 PACK_C_SIZE(cs, c_size);
3519 c_seg->c_bytes_used += c_rounded_size;
3520 c_seg->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
3521 c_seg->c_slots_used++;
3522
3523 slot_ptr->s_cindx = c_seg->c_nextslot++;
3524 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
3525 slot_ptr->s_cseg = c_seg->c_mysegno + 1;
3526
3527 sv_compression:
3528 if (c_seg->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg->c_nextslot >= C_SLOT_MAX_INDEX) {
3529 c_current_seg_filled(c_seg, current_chead);
3530 assert(*current_chead == NULL);
3531 }
3532 lck_mtx_unlock_always(&c_seg->c_lock);
3533
3534 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3535
3536 #if RECORD_THE_COMPRESSED_DATA
3537 if ((c_compressed_record_cptr - c_compressed_record_sbuf) >= C_SEG_ALLOCSIZE) {
3538 c_compressed_record_write(c_compressed_record_sbuf, (int)(c_compressed_record_cptr - c_compressed_record_sbuf));
3539 c_compressed_record_cptr = c_compressed_record_sbuf;
3540 }
3541 #endif
3542 if (c_size) {
3543 OSAddAtomic64(c_size, &c_segment_compressed_bytes);
3544 OSAddAtomic64(c_rounded_size, &compressor_bytes_used);
3545 }
3546 OSAddAtomic64(PAGE_SIZE, &c_segment_input_bytes);
3547
3548 OSAddAtomic(1, &c_segment_pages_compressed);
3549 OSAddAtomic(1, &sample_period_compression_count);
3550
3551 KERNEL_DEBUG(0xe0400000 | DBG_FUNC_END, *current_chead, c_size, c_segment_input_bytes, c_segment_compressed_bytes, 0);
3552
3553 return 0;
3554 }
3555
3556 static inline void
3557 sv_decompress(int32_t *ddst, int32_t pattern)
3558 {
3559 // assert(__builtin_constant_p(PAGE_SIZE) != 0);
3560 #if defined(__x86_64__)
3561 memset_word(ddst, pattern, PAGE_SIZE / sizeof(int32_t));
3562 #elif defined(__arm64__)
3563 assert((PAGE_SIZE % 128) == 0);
3564 if (pattern == 0) {
3565 fill32_dczva((addr64_t)ddst, PAGE_SIZE);
3566 } else {
3567 fill32_nt((addr64_t)ddst, PAGE_SIZE, pattern);
3568 }
3569 #else
3570 size_t i;
3571
3572 /* Unroll the pattern fill loop 4x to encourage the
3573 * compiler to emit NEON stores, cf.
3574 * <rdar://problem/25839866> Loop autovectorization
3575 * anomalies.
3576 */
3577 /* * We use separate loops for each PAGE_SIZE
3578 * to allow the autovectorizer to engage, as PAGE_SIZE
3579 * may not be a constant.
3580 */
3581
3582 __unreachable_ok_push
3583 if (PAGE_SIZE == 4096) {
3584 for (i = 0; i < (4096U / sizeof(int32_t)); i += 4) {
3585 *ddst++ = pattern;
3586 *ddst++ = pattern;
3587 *ddst++ = pattern;
3588 *ddst++ = pattern;
3589 }
3590 } else {
3591 assert(PAGE_SIZE == 16384);
3592 for (i = 0; i < (int)(16384U / sizeof(int32_t)); i += 4) {
3593 *ddst++ = pattern;
3594 *ddst++ = pattern;
3595 *ddst++ = pattern;
3596 *ddst++ = pattern;
3597 }
3598 }
3599 __unreachable_ok_pop
3600 #endif
3601 }
3602
3603 static int
3604 c_decompress_page(char *dst, volatile c_slot_mapping_t slot_ptr, int flags, int *zeroslot)
3605 {
3606 c_slot_t cs;
3607 c_segment_t c_seg;
3608 uint32_t c_segno;
3609 int c_indx;
3610 int c_rounded_size;
3611 uint32_t c_size;
3612 int retval = 0;
3613 boolean_t need_unlock = TRUE;
3614 boolean_t consider_defragmenting = FALSE;
3615 boolean_t kdp_mode = FALSE;
3616
3617 if (__improbable(flags & C_KDP)) {
3618 if (not_in_kdp) {
3619 panic("C_KDP passed to decompress page from outside of debugger context");
3620 }
3621
3622 assert((flags & C_KEEP) == C_KEEP);
3623 assert((flags & C_DONT_BLOCK) == C_DONT_BLOCK);
3624
3625 if ((flags & (C_DONT_BLOCK | C_KEEP)) != (C_DONT_BLOCK | C_KEEP)) {
3626 return -2;
3627 }
3628
3629 kdp_mode = TRUE;
3630 *zeroslot = 0;
3631 }
3632
3633 ReTry:
3634 if (__probable(!kdp_mode)) {
3635 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3636 } else {
3637 if (kdp_lck_rw_lock_is_acquired_exclusive(&c_master_lock)) {
3638 return -2;
3639 }
3640 }
3641
3642 #if HIBERNATION
3643 /*
3644 * if hibernation is enabled, it indicates (via a call
3645 * to 'vm_decompressor_lock' that no further
3646 * decompressions are allowed once it reaches
3647 * the point of flushing all of the currently dirty
3648 * anonymous memory through the compressor and out
3649 * to disk... in this state we allow freeing of compressed
3650 * pages and must honor the C_DONT_BLOCK case
3651 */
3652 if (__improbable(dst && decompressions_blocked == TRUE)) {
3653 if (flags & C_DONT_BLOCK) {
3654 if (__probable(!kdp_mode)) {
3655 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3656 }
3657
3658 *zeroslot = 0;
3659 return -2;
3660 }
3661 /*
3662 * it's safe to atomically assert and block behind the
3663 * lock held in shared mode because "decompressions_blocked" is
3664 * only set and cleared and the thread_wakeup done when the lock
3665 * is held exclusively
3666 */
3667 assert_wait((event_t)&decompressions_blocked, THREAD_UNINT);
3668
3669 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3670
3671 thread_block(THREAD_CONTINUE_NULL);
3672
3673 goto ReTry;
3674 }
3675 #endif
3676 /* s_cseg is actually "segno+1" */
3677 c_segno = slot_ptr->s_cseg - 1;
3678
3679 if (__improbable(c_segno >= c_segments_available)) {
3680 panic("c_decompress_page: c_segno %d >= c_segments_available %d, slot_ptr(%p), slot_data(%x)",
3681 c_segno, c_segments_available, slot_ptr, *(int *)((void *)slot_ptr));
3682 }
3683
3684 if (__improbable(c_segments[c_segno].c_segno < c_segments_available)) {
3685 panic("c_decompress_page: c_segno %d is free, slot_ptr(%p), slot_data(%x)",
3686 c_segno, slot_ptr, *(int *)((void *)slot_ptr));
3687 }
3688
3689 c_seg = c_segments[c_segno].c_seg;
3690
3691 if (__probable(!kdp_mode)) {
3692 lck_mtx_lock_spin_always(&c_seg->c_lock);
3693 } else {
3694 if (kdp_lck_mtx_lock_spin_is_acquired(&c_seg->c_lock)) {
3695 return -2;
3696 }
3697 }
3698
3699 assert(c_seg->c_state != C_IS_EMPTY && c_seg->c_state != C_IS_FREE);
3700
3701 if (dst == NULL && c_seg->c_busy_swapping) {
3702 assert(c_seg->c_busy);
3703
3704 goto bypass_busy_check;
3705 }
3706 if (flags & C_DONT_BLOCK) {
3707 if (c_seg->c_busy || (C_SEG_IS_ONDISK(c_seg) && dst)) {
3708 *zeroslot = 0;
3709
3710 retval = -2;
3711 goto done;
3712 }
3713 }
3714 if (c_seg->c_busy) {
3715 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3716
3717 c_seg_wait_on_busy(c_seg);
3718
3719 goto ReTry;
3720 }
3721 bypass_busy_check:
3722
3723 c_indx = slot_ptr->s_cindx;
3724
3725 if (__improbable(c_indx >= c_seg->c_nextslot)) {
3726 panic("c_decompress_page: c_indx %d >= c_nextslot %d, c_seg(%p), slot_ptr(%p), slot_data(%x)",
3727 c_indx, c_seg->c_nextslot, c_seg, slot_ptr, *(int *)((void *)slot_ptr));
3728 }
3729
3730 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
3731
3732 c_size = UNPACK_C_SIZE(cs);
3733
3734 if (__improbable(c_size == 0)) {
3735 panic("c_decompress_page: c_size == 0, c_seg(%p), slot_ptr(%p), slot_data(%x)",
3736 c_seg, slot_ptr, *(int *)((void *)slot_ptr));
3737 }
3738
3739 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
3740
3741 if (dst) {
3742 uint32_t age_of_cseg;
3743 clock_sec_t cur_ts_sec;
3744 clock_nsec_t cur_ts_nsec;
3745
3746 if (C_SEG_IS_ONDISK(c_seg)) {
3747 assert(kdp_mode == FALSE);
3748 retval = c_seg_swapin(c_seg, FALSE, TRUE);
3749 assert(retval == 0);
3750
3751 retval = 1;
3752 }
3753 if (c_seg->c_state == C_ON_BAD_Q) {
3754 assert(c_seg->c_store.c_buffer == NULL);
3755 *zeroslot = 0;
3756
3757 retval = -1;
3758 goto done;
3759 }
3760
3761 #if POPCOUNT_THE_COMPRESSED_DATA
3762 unsigned csvpop;
3763 uintptr_t csvaddr = (uintptr_t) &c_seg->c_store.c_buffer[cs->c_offset];
3764 if (cs->c_pop_cdata != (csvpop = vmc_pop(csvaddr, c_size))) {
3765 panic("Compressed data popcount doesn't match original, bit distance: %d %p (phys: %p) %p %p 0x%x 0x%x 0x%x 0x%x", (csvpop - cs->c_pop_cdata), (void *)csvaddr, (void *) kvtophys(csvaddr), c_seg, cs, cs->c_offset, c_size, csvpop, cs->c_pop_cdata);
3766 }
3767 #endif
3768
3769 #if CHECKSUM_THE_COMPRESSED_DATA
3770 unsigned csvhash;
3771 if (cs->c_hash_compressed_data != (csvhash = vmc_hash((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size))) {
3772 panic("Compressed data doesn't match original %p %p %u %u %u", c_seg, cs, c_size, cs->c_hash_compressed_data, csvhash);
3773 }
3774 #endif
3775 if (c_rounded_size == PAGE_SIZE) {
3776 /*
3777 * page wasn't compressible... just copy it out
3778 */
3779 memcpy(dst, &c_seg->c_store.c_buffer[cs->c_offset], PAGE_SIZE);
3780 } else if (c_size == 4) {
3781 int32_t data;
3782 int32_t *dptr;
3783
3784 /*
3785 * page was populated with a single value
3786 * that didn't fit into our fast hash
3787 * so we packed it in as a single non-compressed value
3788 * that we need to populate the page with
3789 */
3790 dptr = (int32_t *)(uintptr_t)dst;
3791 data = *(int32_t *)(&c_seg->c_store.c_buffer[cs->c_offset]);
3792 sv_decompress(dptr, data);
3793 } else {
3794 uint32_t my_cpu_no;
3795 char *scratch_buf;
3796
3797 if (__probable(!kdp_mode)) {
3798 /*
3799 * we're behind the c_seg lock held in spin mode
3800 * which means pre-emption is disabled... therefore
3801 * the following sequence is atomic and safe
3802 */
3803 my_cpu_no = cpu_number();
3804
3805 assert(my_cpu_no < compressor_cpus);
3806
3807 scratch_buf = &compressor_scratch_bufs[my_cpu_no * vm_compressor_get_decode_scratch_size()];
3808 } else {
3809 scratch_buf = kdp_compressor_scratch_buf;
3810 }
3811
3812 if (vm_compressor_algorithm() != VM_COMPRESSOR_DEFAULT_CODEC) {
3813 #if defined(__arm__) || defined(__arm64__)
3814 uint16_t c_codec = cs->c_codec;
3815 metadecompressor((const uint8_t *) &c_seg->c_store.c_buffer[cs->c_offset],
3816 (uint8_t *)dst, c_size, c_codec, (void *)scratch_buf);
3817 #endif
3818 } else {
3819 #if defined(__arm64__)
3820 __unreachable_ok_push
3821 if (PAGE_SIZE == 4096) {
3822 WKdm_decompress_4k((WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
3823 (WK_word *)(uintptr_t)dst, (WK_word *)(uintptr_t)scratch_buf, c_size);
3824 } else {
3825 WKdm_decompress_16k((WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
3826 (WK_word *)(uintptr_t)dst, (WK_word *)(uintptr_t)scratch_buf, c_size);
3827 }
3828 __unreachable_ok_pop
3829 #else
3830 WKdm_decompress_new((WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
3831 (WK_word *)(uintptr_t)dst, (WK_word *)(uintptr_t)scratch_buf, c_size);
3832 #endif
3833 }
3834 }
3835
3836 #if CHECKSUM_THE_DATA
3837 if (cs->c_hash_data != vmc_hash(dst, PAGE_SIZE)) {
3838 #if defined(__arm__) || defined(__arm64__)
3839 int32_t *dinput = &c_seg->c_store.c_buffer[cs->c_offset];
3840 panic("decompressed data doesn't match original cs: %p, hash: 0x%x, offset: %d, c_size: %d, c_rounded_size: %d, codec: %d, header: 0x%x 0x%x 0x%x", cs, cs->c_hash_data, cs->c_offset, c_size, c_rounded_size, cs->c_codec, *dinput, *(dinput + 1), *(dinput + 2));
3841 #else
3842 panic("decompressed data doesn't match original cs: %p, hash: %d, offset: 0x%x, c_size: %d", cs, cs->c_hash_data, cs->c_offset, c_size);
3843 #endif
3844 }
3845 #endif
3846 if (c_seg->c_swappedin_ts == 0 && !kdp_mode) {
3847 clock_get_system_nanotime(&cur_ts_sec, &cur_ts_nsec);
3848
3849 age_of_cseg = (uint32_t)cur_ts_sec - c_seg->c_creation_ts;
3850 if (age_of_cseg < DECOMPRESSION_SAMPLE_MAX_AGE) {
3851 OSAddAtomic(1, &age_of_decompressions_during_sample_period[age_of_cseg]);
3852 } else {
3853 OSAddAtomic(1, &overage_decompressions_during_sample_period);
3854 }
3855
3856 OSAddAtomic(1, &sample_period_decompression_count);
3857 }
3858 }
3859 if (flags & C_KEEP) {
3860 *zeroslot = 0;
3861 goto done;
3862 }
3863 assert(kdp_mode == FALSE);
3864
3865 c_seg->c_bytes_unused += c_rounded_size;
3866 c_seg->c_bytes_used -= c_rounded_size;
3867
3868 assert(c_seg->c_slots_used);
3869 c_seg->c_slots_used--;
3870
3871 PACK_C_SIZE(cs, 0);
3872
3873 if (c_indx < c_seg->c_firstemptyslot) {
3874 c_seg->c_firstemptyslot = c_indx;
3875 }
3876
3877 OSAddAtomic(-1, &c_segment_pages_compressed);
3878
3879 if (c_seg->c_state != C_ON_BAD_Q && !(C_SEG_IS_ONDISK(c_seg))) {
3880 /*
3881 * C_SEG_IS_ONDISK == TRUE can occur when we're doing a
3882 * free of a compressed page (i.e. dst == NULL)
3883 */
3884 OSAddAtomic64(-c_rounded_size, &compressor_bytes_used);
3885 }
3886 if (c_seg->c_busy_swapping) {
3887 /*
3888 * bypass case for c_busy_swapping...
3889 * let the swapin/swapout paths deal with putting
3890 * the c_seg on the minor compaction queue if needed
3891 */
3892 assert(c_seg->c_busy);
3893 goto done;
3894 }
3895 assert(!c_seg->c_busy);
3896
3897 if (c_seg->c_state != C_IS_FILLING) {
3898 if (c_seg->c_bytes_used == 0) {
3899 if (!(C_SEG_IS_ONDISK(c_seg))) {
3900 int pages_populated;
3901
3902 pages_populated = (round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE;
3903 c_seg->c_populated_offset = C_SEG_BYTES_TO_OFFSET(0);
3904
3905 if (pages_populated) {
3906 assert(c_seg->c_state != C_ON_BAD_Q);
3907 assert(c_seg->c_store.c_buffer != NULL);
3908
3909 C_SEG_BUSY(c_seg);
3910 lck_mtx_unlock_always(&c_seg->c_lock);
3911
3912 kernel_memory_depopulate(compressor_map, (vm_offset_t) c_seg->c_store.c_buffer, pages_populated * PAGE_SIZE, KMA_COMPRESSOR);
3913
3914 lck_mtx_lock_spin_always(&c_seg->c_lock);
3915 C_SEG_WAKEUP_DONE(c_seg);
3916 }
3917 if (!c_seg->c_on_minorcompact_q && c_seg->c_state != C_ON_SWAPOUT_Q && c_seg->c_state != C_ON_SWAPIO_Q) {
3918 c_seg_need_delayed_compaction(c_seg, FALSE);
3919 }
3920 } else {
3921 if (c_seg->c_state != C_ON_SWAPPEDOUTSPARSE_Q) {
3922 c_seg_move_to_sparse_list(c_seg);
3923 consider_defragmenting = TRUE;
3924 }
3925 }
3926 } else if (c_seg->c_on_minorcompact_q) {
3927 assert(c_seg->c_state != C_ON_BAD_Q);
3928 assert(!C_SEG_IS_ON_DISK_OR_SOQ(c_seg));
3929
3930 if (C_SEG_SHOULD_MINORCOMPACT_NOW(c_seg)) {
3931 c_seg_try_minor_compaction_and_unlock(c_seg);
3932 need_unlock = FALSE;
3933 }
3934 } else if (!(C_SEG_IS_ONDISK(c_seg))) {
3935 if (c_seg->c_state != C_ON_BAD_Q && c_seg->c_state != C_ON_SWAPOUT_Q && c_seg->c_state != C_ON_SWAPIO_Q &&
3936 C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE) {
3937 c_seg_need_delayed_compaction(c_seg, FALSE);
3938 }
3939 } else if (c_seg->c_state != C_ON_SWAPPEDOUTSPARSE_Q && C_SEG_ONDISK_IS_SPARSE(c_seg)) {
3940 c_seg_move_to_sparse_list(c_seg);
3941 consider_defragmenting = TRUE;
3942 }
3943 }
3944 done:
3945 if (__improbable(kdp_mode)) {
3946 return retval;
3947 }
3948
3949 if (need_unlock == TRUE) {
3950 lck_mtx_unlock_always(&c_seg->c_lock);
3951 }
3952
3953 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3954
3955 if (consider_defragmenting == TRUE) {
3956 vm_swap_consider_defragmenting(VM_SWAP_FLAGS_NONE);
3957 }
3958
3959 #if CONFIG_EMBEDDED
3960 if ((c_minor_count && COMPRESSOR_NEEDS_TO_MINOR_COMPACT()) || vm_compressor_needs_to_major_compact()) {
3961 vm_wake_compactor_swapper();
3962 }
3963 #endif
3964
3965 return retval;
3966 }
3967
3968
3969 int
3970 vm_compressor_get(ppnum_t pn, int *slot, int flags)
3971 {
3972 c_slot_mapping_t slot_ptr;
3973 char *dst;
3974 int zeroslot = 1;
3975 int retval;
3976
3977 dst = pmap_map_compressor_page(pn);
3978 slot_ptr = (c_slot_mapping_t)slot;
3979
3980 assert(dst != NULL);
3981
3982 if (slot_ptr->s_cseg == C_SV_CSEG_ID) {
3983 int32_t data;
3984 int32_t *dptr;
3985
3986 /*
3987 * page was populated with a single value
3988 * that found a home in our hash table
3989 * grab that value from the hash and populate the page
3990 * that we need to populate the page with
3991 */
3992 dptr = (int32_t *)(uintptr_t)dst;
3993 data = c_segment_sv_hash_table[slot_ptr->s_cindx].he_data;
3994 sv_decompress(dptr, data);
3995 if (!(flags & C_KEEP)) {
3996 c_segment_sv_hash_drop_ref(slot_ptr->s_cindx);
3997
3998 OSAddAtomic(-1, &c_segment_pages_compressed);
3999 *slot = 0;
4000 }
4001 if (data) {
4002 OSAddAtomic(1, &c_segment_svp_nonzero_decompressions);
4003 } else {
4004 OSAddAtomic(1, &c_segment_svp_zero_decompressions);
4005 }
4006
4007 pmap_unmap_compressor_page(pn, dst);
4008 return 0;
4009 }
4010
4011 retval = c_decompress_page(dst, slot_ptr, flags, &zeroslot);
4012
4013 /*
4014 * zeroslot will be set to 0 by c_decompress_page if (flags & C_KEEP)
4015 * or (flags & C_DONT_BLOCK) and we found 'c_busy' or 'C_SEG_IS_ONDISK' to be TRUE
4016 */
4017 if (zeroslot) {
4018 *slot = 0;
4019 }
4020
4021 pmap_unmap_compressor_page(pn, dst);
4022
4023 /*
4024 * returns 0 if we successfully decompressed a page from a segment already in memory
4025 * returns 1 if we had to first swap in the segment, before successfully decompressing the page
4026 * returns -1 if we encountered an error swapping in the segment - decompression failed
4027 * returns -2 if (flags & C_DONT_BLOCK) and we found 'c_busy' or 'C_SEG_IS_ONDISK' to be true
4028 */
4029 return retval;
4030 }
4031
4032
4033 int
4034 vm_compressor_free(int *slot, int flags)
4035 {
4036 c_slot_mapping_t slot_ptr;
4037 int zeroslot = 1;
4038 int retval;
4039
4040 assert(flags == 0 || flags == C_DONT_BLOCK);
4041
4042 slot_ptr = (c_slot_mapping_t)slot;
4043
4044 if (slot_ptr->s_cseg == C_SV_CSEG_ID) {
4045 c_segment_sv_hash_drop_ref(slot_ptr->s_cindx);
4046 OSAddAtomic(-1, &c_segment_pages_compressed);
4047
4048 *slot = 0;
4049 return 0;
4050 }
4051 retval = c_decompress_page(NULL, slot_ptr, flags, &zeroslot);
4052 /*
4053 * returns 0 if we successfully freed the specified compressed page
4054 * returns -2 if (flags & C_DONT_BLOCK) and we found 'c_busy' set
4055 */
4056
4057 if (retval == 0) {
4058 *slot = 0;
4059 } else {
4060 assert(retval == -2);
4061 }
4062
4063 return retval;
4064 }
4065
4066
4067 int
4068 vm_compressor_put(ppnum_t pn, int *slot, void **current_chead, char *scratch_buf)
4069 {
4070 char *src;
4071 int retval;
4072
4073 src = pmap_map_compressor_page(pn);
4074 assert(src != NULL);
4075
4076 retval = c_compress_page(src, (c_slot_mapping_t)slot, (c_segment_t *)current_chead, scratch_buf);
4077 pmap_unmap_compressor_page(pn, src);
4078
4079 return retval;
4080 }
4081
4082 void
4083 vm_compressor_transfer(
4084 int *dst_slot_p,
4085 int *src_slot_p)
4086 {
4087 c_slot_mapping_t dst_slot, src_slot;
4088 c_segment_t c_seg;
4089 int c_indx;
4090 c_slot_t cs;
4091
4092 src_slot = (c_slot_mapping_t) src_slot_p;
4093
4094 if (src_slot->s_cseg == C_SV_CSEG_ID) {
4095 *dst_slot_p = *src_slot_p;
4096 *src_slot_p = 0;
4097 return;
4098 }
4099 dst_slot = (c_slot_mapping_t) dst_slot_p;
4100 Retry:
4101 PAGE_REPLACEMENT_DISALLOWED(TRUE);
4102 /* get segment for src_slot */
4103 c_seg = c_segments[src_slot->s_cseg - 1].c_seg;
4104 /* lock segment */
4105 lck_mtx_lock_spin_always(&c_seg->c_lock);
4106 /* wait if it's busy */
4107 if (c_seg->c_busy && !c_seg->c_busy_swapping) {
4108 PAGE_REPLACEMENT_DISALLOWED(FALSE);
4109 c_seg_wait_on_busy(c_seg);
4110 goto Retry;
4111 }
4112 /* find the c_slot */
4113 c_indx = src_slot->s_cindx;
4114 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
4115 /* point the c_slot back to dst_slot instead of src_slot */
4116 cs->c_packed_ptr = C_SLOT_PACK_PTR(dst_slot);
4117 /* transfer */
4118 *dst_slot_p = *src_slot_p;
4119 *src_slot_p = 0;
4120 lck_mtx_unlock_always(&c_seg->c_lock);
4121 PAGE_REPLACEMENT_DISALLOWED(FALSE);
4122 }
4123
4124 #if CONFIG_FREEZE
4125
4126 int freezer_finished_filling = 0;
4127
4128 void
4129 vm_compressor_finished_filling(
4130 void **current_chead)
4131 {
4132 c_segment_t c_seg;
4133
4134 if ((c_seg = *(c_segment_t *)current_chead) == NULL) {
4135 return;
4136 }
4137
4138 assert(c_seg->c_state == C_IS_FILLING);
4139
4140 lck_mtx_lock_spin_always(&c_seg->c_lock);
4141
4142 c_current_seg_filled(c_seg, (c_segment_t *)current_chead);
4143
4144 lck_mtx_unlock_always(&c_seg->c_lock);
4145
4146 freezer_finished_filling++;
4147 }
4148
4149
4150 /*
4151 * This routine is used to transfer the compressed chunks from
4152 * the c_seg/cindx pointed to by slot_p into a new c_seg headed
4153 * by the current_chead and a new cindx within that c_seg.
4154 *
4155 * Currently, this routine is only used by the "freezer backed by
4156 * compressor with swap" mode to create a series of c_segs that
4157 * only contain compressed data belonging to one task. So, we
4158 * move a task's previously compressed data into a set of new
4159 * c_segs which will also hold the task's yet to be compressed data.
4160 */
4161
4162 kern_return_t
4163 vm_compressor_relocate(
4164 void **current_chead,
4165 int *slot_p)
4166 {
4167 c_slot_mapping_t slot_ptr;
4168 c_slot_mapping_t src_slot;
4169 uint32_t c_rounded_size;
4170 uint32_t c_size;
4171 uint16_t dst_slot;
4172 c_slot_t c_dst;
4173 c_slot_t c_src;
4174 int c_indx;
4175 c_segment_t c_seg_dst = NULL;
4176 c_segment_t c_seg_src = NULL;
4177 kern_return_t kr = KERN_SUCCESS;
4178
4179
4180 src_slot = (c_slot_mapping_t) slot_p;
4181
4182 if (src_slot->s_cseg == C_SV_CSEG_ID) {
4183 /*
4184 * no need to relocate... this is a page full of a single
4185 * value which is hashed to a single entry not contained
4186 * in a c_segment_t
4187 */
4188 return kr;
4189 }
4190
4191 Relookup_dst:
4192 c_seg_dst = c_seg_allocate((c_segment_t *)current_chead);
4193 /*
4194 * returns with c_seg lock held
4195 * and PAGE_REPLACEMENT_DISALLOWED(TRUE)...
4196 * c_nextslot has been allocated and
4197 * c_store.c_buffer populated
4198 */
4199 if (c_seg_dst == NULL) {
4200 /*
4201 * Out of compression segments?
4202 */
4203 kr = KERN_RESOURCE_SHORTAGE;
4204 goto out;
4205 }
4206
4207 assert(c_seg_dst->c_busy == 0);
4208
4209 C_SEG_BUSY(c_seg_dst);
4210
4211 dst_slot = c_seg_dst->c_nextslot;
4212
4213 lck_mtx_unlock_always(&c_seg_dst->c_lock);
4214
4215 Relookup_src:
4216 c_seg_src = c_segments[src_slot->s_cseg - 1].c_seg;
4217
4218 assert(c_seg_dst != c_seg_src);
4219
4220 lck_mtx_lock_spin_always(&c_seg_src->c_lock);
4221
4222 if (C_SEG_IS_ONDISK(c_seg_src)) {
4223 /*
4224 * A "thaw" can mark a process as eligible for
4225 * another freeze cycle without bringing any of
4226 * its swapped out c_segs back from disk (because
4227 * that is done on-demand).
4228 *
4229 * If the src c_seg we find for our pre-compressed
4230 * data is already on-disk, then we are dealing
4231 * with an app's data that is already packed and
4232 * swapped out. Don't do anything.
4233 */
4234
4235 PAGE_REPLACEMENT_DISALLOWED(FALSE);
4236
4237 lck_mtx_unlock_always(&c_seg_src->c_lock);
4238
4239 c_seg_src = NULL;
4240
4241 goto out;
4242 }
4243
4244 if (c_seg_src->c_busy) {
4245 PAGE_REPLACEMENT_DISALLOWED(FALSE);
4246 c_seg_wait_on_busy(c_seg_src);
4247
4248 c_seg_src = NULL;
4249
4250 PAGE_REPLACEMENT_DISALLOWED(TRUE);
4251
4252 goto Relookup_src;
4253 }
4254
4255 C_SEG_BUSY(c_seg_src);
4256
4257 lck_mtx_unlock_always(&c_seg_src->c_lock);
4258
4259 PAGE_REPLACEMENT_DISALLOWED(FALSE);
4260
4261 /* find the c_slot */
4262 c_indx = src_slot->s_cindx;
4263
4264 c_src = C_SEG_SLOT_FROM_INDEX(c_seg_src, c_indx);
4265
4266 c_size = UNPACK_C_SIZE(c_src);
4267
4268 assert(c_size);
4269
4270 if (c_size > (uint32_t)(C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES((int32_t)c_seg_dst->c_nextoffset))) {
4271 /*
4272 * This segment is full. We need a new one.
4273 */
4274
4275 PAGE_REPLACEMENT_DISALLOWED(TRUE);
4276
4277 lck_mtx_lock_spin_always(&c_seg_src->c_lock);
4278 C_SEG_WAKEUP_DONE(c_seg_src);
4279 lck_mtx_unlock_always(&c_seg_src->c_lock);
4280
4281 c_seg_src = NULL;
4282
4283 lck_mtx_lock_spin_always(&c_seg_dst->c_lock);
4284
4285 assert(c_seg_dst->c_busy);
4286 assert(c_seg_dst->c_state == C_IS_FILLING);
4287 assert(!c_seg_dst->c_on_minorcompact_q);
4288
4289 c_current_seg_filled(c_seg_dst, (c_segment_t *)current_chead);
4290 assert(*current_chead == NULL);
4291
4292 C_SEG_WAKEUP_DONE(c_seg_dst);
4293
4294 lck_mtx_unlock_always(&c_seg_dst->c_lock);
4295
4296 c_seg_dst = NULL;
4297
4298 PAGE_REPLACEMENT_DISALLOWED(FALSE);
4299
4300 goto Relookup_dst;
4301 }
4302
4303 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, c_seg_dst->c_nextslot);
4304
4305 memcpy(&c_seg_dst->c_store.c_buffer[c_seg_dst->c_nextoffset], &c_seg_src->c_store.c_buffer[c_src->c_offset], c_size);
4306 //is platform alignment actually necessary since wkdm aligns its output?
4307 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
4308
4309 cslot_copy(c_dst, c_src);
4310 c_dst->c_offset = c_seg_dst->c_nextoffset;
4311
4312 if (c_seg_dst->c_firstemptyslot == c_seg_dst->c_nextslot) {
4313 c_seg_dst->c_firstemptyslot++;
4314 }
4315
4316 c_seg_dst->c_slots_used++;
4317 c_seg_dst->c_nextslot++;
4318 c_seg_dst->c_bytes_used += c_rounded_size;
4319 c_seg_dst->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
4320
4321
4322 PACK_C_SIZE(c_src, 0);
4323
4324 c_seg_src->c_bytes_used -= c_rounded_size;
4325 c_seg_src->c_bytes_unused += c_rounded_size;
4326
4327 assert(c_seg_src->c_slots_used);
4328 c_seg_src->c_slots_used--;
4329
4330 if (c_indx < c_seg_src->c_firstemptyslot) {
4331 c_seg_src->c_firstemptyslot = c_indx;
4332 }
4333
4334 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, dst_slot);
4335
4336 PAGE_REPLACEMENT_ALLOWED(TRUE);
4337 slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
4338 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
4339 slot_ptr->s_cseg = c_seg_dst->c_mysegno + 1;
4340 slot_ptr->s_cindx = dst_slot;
4341
4342 PAGE_REPLACEMENT_ALLOWED(FALSE);
4343
4344 out:
4345 if (c_seg_src) {
4346 lck_mtx_lock_spin_always(&c_seg_src->c_lock);
4347
4348 C_SEG_WAKEUP_DONE(c_seg_src);
4349
4350 if (c_seg_src->c_bytes_used == 0 && c_seg_src->c_state != C_IS_FILLING) {
4351 if (!c_seg_src->c_on_minorcompact_q) {
4352 c_seg_need_delayed_compaction(c_seg_src, FALSE);
4353 }
4354 }
4355
4356 lck_mtx_unlock_always(&c_seg_src->c_lock);
4357 }
4358
4359 if (c_seg_dst) {
4360 PAGE_REPLACEMENT_DISALLOWED(TRUE);
4361
4362 lck_mtx_lock_spin_always(&c_seg_dst->c_lock);
4363
4364 if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX_INDEX) {
4365 /*
4366 * Nearing or exceeded maximum slot and offset capacity.
4367 */
4368 assert(c_seg_dst->c_busy);
4369 assert(c_seg_dst->c_state == C_IS_FILLING);
4370 assert(!c_seg_dst->c_on_minorcompact_q);
4371
4372 c_current_seg_filled(c_seg_dst, (c_segment_t *)current_chead);
4373 assert(*current_chead == NULL);
4374 }
4375
4376 C_SEG_WAKEUP_DONE(c_seg_dst);
4377
4378 lck_mtx_unlock_always(&c_seg_dst->c_lock);
4379
4380 c_seg_dst = NULL;
4381
4382 PAGE_REPLACEMENT_DISALLOWED(FALSE);
4383 }
4384
4385 return kr;
4386 }
4387 #endif /* CONFIG_FREEZE */
mirror of XNU