]> git.saurik.com Git - apple/xnu.git/blob - osfmk/vm/vm_compressor.c
projects / apple / xnu.git / blob
? search:


071a66d4e3adc3729a1091b5ca3e2af71da49a07
[apple/xnu.git] / osfmk / vm / vm_compressor.c
1 /*
2 * Copyright (c) 2000-2013 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <vm/vm_compressor.h>
30
31 #if CONFIG_PHANTOM_CACHE
32 #include <vm/vm_phantom_cache.h>
33 #endif
34
35 #include <vm/vm_map.h>
36 #include <vm/vm_pageout.h>
37 #include <vm/memory_object.h>
38 #include <vm/vm_compressor_algorithms.h>
39 #include <vm/vm_fault.h>
40 #include <vm/vm_protos.h>
41 #include <mach/mach_host.h> /* for host_info() */
42 #include <kern/ledger.h>
43 #include <kern/policy_internal.h>
44 #include <kern/thread_group.h>
45 #include <san/kasan.h>
46
47 #if defined(__x86_64__)
48 #include <i386/misc_protos.h>
49 #endif
50 #if defined(__arm64__)
51 #include <arm/machine_routines.h>
52 #endif
53
54 #include <IOKit/IOHibernatePrivate.h>
55
56 extern boolean_t vm_darkwake_mode;
57
58 #if DEVELOPMENT || DEBUG
59 int do_cseg_wedge_thread(void);
60 int do_cseg_unwedge_thread(void);
61 static event_t debug_cseg_wait_event = NULL;
62 #endif /* DEVELOPMENT || DEBUG */
63
64 #if POPCOUNT_THE_COMPRESSED_DATA
65 boolean_t popcount_c_segs = TRUE;
66
67 static inline uint32_t
68 vmc_pop(uintptr_t ins, int sz)
69 {
70 uint32_t rv = 0;
71
72 if (__probable(popcount_c_segs == FALSE)) {
73 return 0xDEAD707C;
74 }
75
76 while (sz >= 16) {
77 uint32_t rv1, rv2;
78 uint64_t *ins64 = (uint64_t *) ins;
79 uint64_t *ins642 = (uint64_t *) (ins + 8);
80 rv1 = __builtin_popcountll(*ins64);
81 rv2 = __builtin_popcountll(*ins642);
82 rv += rv1 + rv2;
83 sz -= 16;
84 ins += 16;
85 }
86
87 while (sz >= 4) {
88 uint32_t *ins32 = (uint32_t *) ins;
89 rv += __builtin_popcount(*ins32);
90 sz -= 4;
91 ins += 4;
92 }
93
94 while (sz > 0) {
95 char *ins8 = (char *)ins;
96 rv += __builtin_popcount(*ins8);
97 sz--;
98 ins++;
99 }
100 return rv;
101 }
102 #endif
103
104 #if VALIDATE_C_SEGMENTS
105 boolean_t validate_c_segs = TRUE;
106 #endif
107 /*
108 * vm_compressor_mode has a heirarchy of control to set its value.
109 * boot-args are checked first, then device-tree, and finally
110 * the default value that is defined below. See vm_fault_init() for
111 * the boot-arg & device-tree code.
112 */
113
114 #if CONFIG_EMBEDDED
115
116 #if CONFIG_FREEZE
117 int vm_compressor_mode = VM_PAGER_FREEZER_DEFAULT;
118
119 void *freezer_chead; /* The chead used to track c_segs allocated for the exclusive use of holding just one task's compressed memory.*/
120 char *freezer_compressor_scratch_buf = NULL;
121
122 extern int c_freezer_swapout_page_count; /* This count keeps track of the # of compressed pages holding just one task's compressed memory on the swapout queue. This count is used during each freeze i.e. on a per-task basis.*/
123
124 #else /* CONFIG_FREEZE */
125 int vm_compressor_mode = VM_PAGER_NOT_CONFIGURED;
126 #endif /* CONFIG_FREEZE */
127
128 int vm_scale = 1;
129
130 #else /* CONFIG_EMBEDDED */
131 int vm_compressor_mode = VM_PAGER_COMPRESSOR_WITH_SWAP;
132 int vm_scale = 16;
133
134 #endif /* CONFIG_EMBEDDED */
135
136 int vm_compressor_is_active = 0;
137 int vm_compression_limit = 0;
138 int vm_compressor_available = 0;
139
140 extern void vm_pageout_io_throttle(void);
141
142 #if CHECKSUM_THE_DATA || CHECKSUM_THE_SWAP || CHECKSUM_THE_COMPRESSED_DATA
143 extern unsigned int hash_string(char *cp, int len);
144 static unsigned int vmc_hash(char *, int);
145 boolean_t checksum_c_segs = TRUE;
146
147 unsigned int
148 vmc_hash(char *cp, int len)
149 {
150 if (__probable(checksum_c_segs == FALSE)) {
151 return 0xDEAD7A37;
152 }
153 return hash_string(cp, len);
154 }
155 #endif
156
157 #define UNPACK_C_SIZE(cs) ((cs->c_size == (PAGE_SIZE-1)) ? PAGE_SIZE : cs->c_size)
158 #define PACK_C_SIZE(cs, size) (cs->c_size = ((size == PAGE_SIZE) ? PAGE_SIZE - 1 : size))
159
160
161 struct c_sv_hash_entry {
162 union {
163 struct {
164 uint32_t c_sv_he_ref;
165 uint32_t c_sv_he_data;
166 } c_sv_he;
167 uint64_t c_sv_he_record;
168 } c_sv_he_un;
169 };
170
171 #define he_ref c_sv_he_un.c_sv_he.c_sv_he_ref
172 #define he_data c_sv_he_un.c_sv_he.c_sv_he_data
173 #define he_record c_sv_he_un.c_sv_he_record
174
175 #define C_SV_HASH_MAX_MISS 32
176 #define C_SV_HASH_SIZE ((1 << 10))
177 #define C_SV_HASH_MASK ((1 << 10) - 1)
178 #define C_SV_CSEG_ID ((1 << 22) - 1)
179
180
181 union c_segu {
182 c_segment_t c_seg;
183 uintptr_t c_segno;
184 };
185
186
187
188 #define C_SLOT_PACK_PTR(ptr) (((uintptr_t)ptr - (uintptr_t) KERNEL_PMAP_HEAP_RANGE_START) >> 2)
189 #define C_SLOT_UNPACK_PTR(cslot) ((uintptr_t)(cslot->c_packed_ptr << 2) + (uintptr_t) KERNEL_PMAP_HEAP_RANGE_START)
190
191
192 uint32_t c_segment_count = 0;
193 uint32_t c_segment_count_max = 0;
194
195 uint64_t c_generation_id = 0;
196 uint64_t c_generation_id_flush_barrier;
197
198
199 #define HIBERNATE_FLUSHING_SECS_TO_COMPLETE 120
200
201 boolean_t hibernate_no_swapspace = FALSE;
202 clock_sec_t hibernate_flushing_deadline = 0;
203
204
205 #if RECORD_THE_COMPRESSED_DATA
206 char *c_compressed_record_sbuf;
207 char *c_compressed_record_ebuf;
208 char *c_compressed_record_cptr;
209 #endif
210
211
212 queue_head_t c_age_list_head;
213 queue_head_t c_swappedin_list_head;
214 queue_head_t c_swapout_list_head;
215 queue_head_t c_swapio_list_head;
216 queue_head_t c_swappedout_list_head;
217 queue_head_t c_swappedout_sparse_list_head;
218 queue_head_t c_major_list_head;
219 queue_head_t c_filling_list_head;
220 queue_head_t c_bad_list_head;
221
222 uint32_t c_age_count = 0;
223 uint32_t c_swappedin_count = 0;
224 uint32_t c_swapout_count = 0;
225 uint32_t c_swapio_count = 0;
226 uint32_t c_swappedout_count = 0;
227 uint32_t c_swappedout_sparse_count = 0;
228 uint32_t c_major_count = 0;
229 uint32_t c_filling_count = 0;
230 uint32_t c_empty_count = 0;
231 uint32_t c_bad_count = 0;
232
233
234 queue_head_t c_minor_list_head;
235 uint32_t c_minor_count = 0;
236
237 int c_overage_swapped_count = 0;
238 int c_overage_swapped_limit = 0;
239
240 int c_seg_fixed_array_len;
241 union c_segu *c_segments;
242 vm_offset_t c_buffers;
243 vm_size_t c_buffers_size;
244 caddr_t c_segments_next_page;
245 boolean_t c_segments_busy;
246 uint32_t c_segments_available;
247 uint32_t c_segments_limit;
248 uint32_t c_segments_nearing_limit;
249
250 uint32_t c_segment_svp_in_hash;
251 uint32_t c_segment_svp_hash_succeeded;
252 uint32_t c_segment_svp_hash_failed;
253 uint32_t c_segment_svp_zero_compressions;
254 uint32_t c_segment_svp_nonzero_compressions;
255 uint32_t c_segment_svp_zero_decompressions;
256 uint32_t c_segment_svp_nonzero_decompressions;
257
258 uint32_t c_segment_noncompressible_pages;
259
260 uint32_t c_segment_pages_compressed;
261 uint32_t c_segment_pages_compressed_limit;
262 uint32_t c_segment_pages_compressed_nearing_limit;
263 uint32_t c_free_segno_head = (uint32_t)-1;
264
265 uint32_t vm_compressor_minorcompact_threshold_divisor = 10;
266 uint32_t vm_compressor_majorcompact_threshold_divisor = 10;
267 uint32_t vm_compressor_unthrottle_threshold_divisor = 10;
268 uint32_t vm_compressor_catchup_threshold_divisor = 10;
269
270 uint32_t vm_compressor_minorcompact_threshold_divisor_overridden = 0;
271 uint32_t vm_compressor_majorcompact_threshold_divisor_overridden = 0;
272 uint32_t vm_compressor_unthrottle_threshold_divisor_overridden = 0;
273 uint32_t vm_compressor_catchup_threshold_divisor_overridden = 0;
274
275 #define C_SEGMENTS_PER_PAGE (PAGE_SIZE / sizeof(union c_segu))
276
277
278 lck_grp_attr_t vm_compressor_lck_grp_attr;
279 lck_attr_t vm_compressor_lck_attr;
280 lck_grp_t vm_compressor_lck_grp;
281 lck_mtx_t *c_list_lock;
282 lck_rw_t c_master_lock;
283 boolean_t decompressions_blocked = FALSE;
284
285 zone_t compressor_segment_zone;
286 int c_compressor_swap_trigger = 0;
287
288 uint32_t compressor_cpus;
289 char *compressor_scratch_bufs;
290 char *kdp_compressor_scratch_buf;
291 char *kdp_compressor_decompressed_page;
292 addr64_t kdp_compressor_decompressed_page_paddr;
293 ppnum_t kdp_compressor_decompressed_page_ppnum;
294
295 clock_sec_t start_of_sample_period_sec = 0;
296 clock_nsec_t start_of_sample_period_nsec = 0;
297 clock_sec_t start_of_eval_period_sec = 0;
298 clock_nsec_t start_of_eval_period_nsec = 0;
299 uint32_t sample_period_decompression_count = 0;
300 uint32_t sample_period_compression_count = 0;
301 uint32_t last_eval_decompression_count = 0;
302 uint32_t last_eval_compression_count = 0;
303
304 #define DECOMPRESSION_SAMPLE_MAX_AGE (60 * 30)
305
306 boolean_t vm_swapout_ripe_segments = FALSE;
307 uint32_t vm_ripe_target_age = (60 * 60 * 48);
308
309 uint32_t swapout_target_age = 0;
310 uint32_t age_of_decompressions_during_sample_period[DECOMPRESSION_SAMPLE_MAX_AGE];
311 uint32_t overage_decompressions_during_sample_period = 0;
312
313
314 void do_fastwake_warmup(queue_head_t *, boolean_t);
315 boolean_t fastwake_warmup = FALSE;
316 boolean_t fastwake_recording_in_progress = FALSE;
317 clock_sec_t dont_trim_until_ts = 0;
318
319 uint64_t c_segment_warmup_count;
320 uint64_t first_c_segment_to_warm_generation_id = 0;
321 uint64_t last_c_segment_to_warm_generation_id = 0;
322 boolean_t hibernate_flushing = FALSE;
323
324 int64_t c_segment_input_bytes __attribute__((aligned(8))) = 0;
325 int64_t c_segment_compressed_bytes __attribute__((aligned(8))) = 0;
326 int64_t compressor_bytes_used __attribute__((aligned(8))) = 0;
327
328
329 struct c_sv_hash_entry c_segment_sv_hash_table[C_SV_HASH_SIZE] __attribute__ ((aligned(8)));
330
331 static boolean_t compressor_needs_to_swap(void);
332 static void vm_compressor_swap_trigger_thread(void);
333 static void vm_compressor_do_delayed_compactions(boolean_t);
334 static void vm_compressor_compact_and_swap(boolean_t);
335 static void vm_compressor_age_swapped_in_segments(boolean_t);
336
337 #if !CONFIG_EMBEDDED
338 static void vm_compressor_take_paging_space_action(void);
339 #endif
340
341 void compute_swapout_target_age(void);
342
343 boolean_t c_seg_major_compact(c_segment_t, c_segment_t);
344 boolean_t c_seg_major_compact_ok(c_segment_t, c_segment_t);
345
346 int c_seg_minor_compaction_and_unlock(c_segment_t, boolean_t);
347 int c_seg_do_minor_compaction_and_unlock(c_segment_t, boolean_t, boolean_t, boolean_t);
348 void c_seg_try_minor_compaction_and_unlock(c_segment_t c_seg);
349
350 void c_seg_move_to_sparse_list(c_segment_t);
351 void c_seg_insert_into_q(queue_head_t *, c_segment_t);
352
353 uint64_t vm_available_memory(void);
354 uint64_t vm_compressor_pages_compressed(void);
355
356 /*
357 * indicate the need to do a major compaction if
358 * the overall set of in-use compression segments
359 * becomes sparse... on systems that support pressure
360 * driven swapping, this will also cause swapouts to
361 * be initiated.
362 */
363 static inline boolean_t
364 vm_compressor_needs_to_major_compact()
365 {
366 uint32_t incore_seg_count;
367
368 incore_seg_count = c_segment_count - c_swappedout_count - c_swappedout_sparse_count;
369
370 if ((c_segment_count >= (c_segments_nearing_limit / 8)) &&
371 ((incore_seg_count * C_SEG_MAX_PAGES) - VM_PAGE_COMPRESSOR_COUNT) >
372 ((incore_seg_count / 8) * C_SEG_MAX_PAGES)) {
373 return 1;
374 }
375 return 0;
376 }
377
378
379 uint64_t
380 vm_available_memory(void)
381 {
382 return ((uint64_t)AVAILABLE_NON_COMPRESSED_MEMORY) * PAGE_SIZE_64;
383 }
384
385
386 uint64_t
387 vm_compressor_pages_compressed(void)
388 {
389 return c_segment_pages_compressed * PAGE_SIZE_64;
390 }
391
392
393 boolean_t
394 vm_compressor_low_on_space(void)
395 {
396 if ((c_segment_pages_compressed > c_segment_pages_compressed_nearing_limit) ||
397 (c_segment_count > c_segments_nearing_limit)) {
398 return TRUE;
399 }
400
401 return FALSE;
402 }
403
404
405 boolean_t
406 vm_compressor_out_of_space(void)
407 {
408 if ((c_segment_pages_compressed >= c_segment_pages_compressed_limit) ||
409 (c_segment_count >= c_segments_limit)) {
410 return TRUE;
411 }
412
413 return FALSE;
414 }
415
416
417 int
418 vm_wants_task_throttled(task_t task)
419 {
420 if (task == kernel_task) {
421 return 0;
422 }
423
424 if (VM_CONFIG_SWAP_IS_ACTIVE) {
425 if ((vm_compressor_low_on_space() || HARD_THROTTLE_LIMIT_REACHED()) &&
426 (unsigned int)pmap_compressed(task->map->pmap) > (c_segment_pages_compressed / 4)) {
427 return 1;
428 }
429 }
430 return 0;
431 }
432
433
434 #if DEVELOPMENT || DEBUG
435 boolean_t kill_on_no_paging_space = FALSE; /* On compressor/swap exhaustion, kill the largest process regardless of
436 * its chosen process policy. Controlled by a boot-arg of the same name. */
437 #endif /* DEVELOPMENT || DEBUG */
438
439 #if !CONFIG_EMBEDDED
440
441 static uint32_t no_paging_space_action_in_progress = 0;
442 extern void memorystatus_send_low_swap_note(void);
443
444 static void
445 vm_compressor_take_paging_space_action(void)
446 {
447 if (no_paging_space_action_in_progress == 0) {
448 if (OSCompareAndSwap(0, 1, (UInt32 *)&no_paging_space_action_in_progress)) {
449 if (no_paging_space_action()) {
450 #if DEVELOPMENT || DEBUG
451 if (kill_on_no_paging_space == TRUE) {
452 /*
453 * Since we are choosing to always kill a process, we don't need the
454 * "out of application memory" dialog box in this mode. And, hence we won't
455 * send the knote.
456 */
457 no_paging_space_action_in_progress = 0;
458 return;
459 }
460 #endif /* DEVELOPMENT || DEBUG */
461 memorystatus_send_low_swap_note();
462 }
463
464 no_paging_space_action_in_progress = 0;
465 }
466 }
467 }
468 #endif /* !CONFIG_EMBEDDED */
469
470
471 void
472 vm_compressor_init_locks(void)
473 {
474 lck_grp_attr_setdefault(&vm_compressor_lck_grp_attr);
475 lck_grp_init(&vm_compressor_lck_grp, "vm_compressor", &vm_compressor_lck_grp_attr);
476 lck_attr_setdefault(&vm_compressor_lck_attr);
477
478 lck_rw_init(&c_master_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
479 }
480
481
482 void
483 vm_decompressor_lock(void)
484 {
485 PAGE_REPLACEMENT_ALLOWED(TRUE);
486
487 decompressions_blocked = TRUE;
488
489 PAGE_REPLACEMENT_ALLOWED(FALSE);
490 }
491
492 void
493 vm_decompressor_unlock(void)
494 {
495 PAGE_REPLACEMENT_ALLOWED(TRUE);
496
497 decompressions_blocked = FALSE;
498
499 PAGE_REPLACEMENT_ALLOWED(FALSE);
500
501 thread_wakeup((event_t)&decompressions_blocked);
502 }
503
504 static inline void
505 cslot_copy(c_slot_t cdst, c_slot_t csrc)
506 {
507 #if CHECKSUM_THE_DATA
508 cdst->c_hash_data = csrc->c_hash_data;
509 #endif
510 #if CHECKSUM_THE_COMPRESSED_DATA
511 cdst->c_hash_compressed_data = csrc->c_hash_compressed_data;
512 #endif
513 #if POPCOUNT_THE_COMPRESSED_DATA
514 cdst->c_pop_cdata = csrc->c_pop_cdata;
515 #endif
516 cdst->c_size = csrc->c_size;
517 cdst->c_packed_ptr = csrc->c_packed_ptr;
518 #if defined(__arm__) || defined(__arm64__)
519 cdst->c_codec = csrc->c_codec;
520 #endif
521 }
522
523 vm_map_t compressor_map;
524 uint64_t compressor_pool_max_size;
525 uint64_t compressor_pool_size;
526 uint32_t compressor_pool_multiplier;
527
528 #if DEVELOPMENT || DEBUG
529 /*
530 * Compressor segments are write-protected in development/debug
531 * kernels to help debug memory corruption.
532 * In cases where performance is a concern, this can be disabled
533 * via the boot-arg "-disable_cseg_write_protection".
534 */
535 boolean_t write_protect_c_segs = TRUE;
536 int vm_compressor_test_seg_wp;
537 uint32_t vm_ktrace_enabled;
538 #endif /* DEVELOPMENT || DEBUG */
539
540 void
541 vm_compressor_init(void)
542 {
543 thread_t thread;
544 struct c_slot cs_dummy;
545 c_slot_t cs = &cs_dummy;
546 int c_segment_min_size;
547 int c_segment_padded_size;
548 int attempts = 1;
549 kern_return_t retval = KERN_SUCCESS;
550 vm_offset_t start_addr = 0;
551 vm_size_t c_segments_arr_size = 0, compressor_submap_size = 0;
552 vm_map_kernel_flags_t vmk_flags;
553 #if RECORD_THE_COMPRESSED_DATA
554 vm_size_t c_compressed_record_sbuf_size = 0;
555 #endif /* RECORD_THE_COMPRESSED_DATA */
556
557 #if DEVELOPMENT || DEBUG
558 char bootarg_name[32];
559 if (PE_parse_boot_argn("-kill_on_no_paging_space", bootarg_name, sizeof(bootarg_name))) {
560 kill_on_no_paging_space = TRUE;
561 }
562 if (PE_parse_boot_argn("-disable_cseg_write_protection", bootarg_name, sizeof(bootarg_name))) {
563 write_protect_c_segs = FALSE;
564 }
565 int vmcval = 1;
566 PE_parse_boot_argn("vm_compressor_validation", &vmcval, sizeof(vmcval));
567
568 if (kern_feature_override(KF_COMPRSV_OVRD)) {
569 vmcval = 0;
570 }
571 if (vmcval == 0) {
572 #if POPCOUNT_THE_COMPRESSED_DATA
573 popcount_c_segs = FALSE;
574 #endif
575 #if CHECKSUM_THE_DATA || CHECKSUM_THE_COMPRESSED_DATA
576 checksum_c_segs = FALSE;
577 #endif
578 #if VALIDATE_C_SEGMENTS
579 validate_c_segs = FALSE;
580 #endif
581 write_protect_c_segs = FALSE;
582 }
583 #endif /* DEVELOPMENT || DEBUG */
584
585 /*
586 * ensure that any pointer that gets created from
587 * the vm_page zone can be packed properly
588 */
589 cs->c_packed_ptr = C_SLOT_PACK_PTR(zone_map_min_address);
590
591 if (C_SLOT_UNPACK_PTR(cs) != (uintptr_t)zone_map_min_address) {
592 panic("C_SLOT_UNPACK_PTR failed on zone_map_min_address - %p", (void *)zone_map_min_address);
593 }
594
595 cs->c_packed_ptr = C_SLOT_PACK_PTR(zone_map_max_address);
596
597 if (C_SLOT_UNPACK_PTR(cs) != (uintptr_t)zone_map_max_address) {
598 panic("C_SLOT_UNPACK_PTR failed on zone_map_max_address - %p", (void *)zone_map_max_address);
599 }
600
601
602 assert((C_SEGMENTS_PER_PAGE * sizeof(union c_segu)) == PAGE_SIZE);
603
604 PE_parse_boot_argn("vm_compression_limit", &vm_compression_limit, sizeof(vm_compression_limit));
605
606 #ifdef CONFIG_EMBEDDED
607 vm_compressor_minorcompact_threshold_divisor = 20;
608 vm_compressor_majorcompact_threshold_divisor = 30;
609 vm_compressor_unthrottle_threshold_divisor = 40;
610 vm_compressor_catchup_threshold_divisor = 60;
611 #else
612 if (max_mem <= (3ULL * 1024ULL * 1024ULL * 1024ULL)) {
613 vm_compressor_minorcompact_threshold_divisor = 11;
614 vm_compressor_majorcompact_threshold_divisor = 13;
615 vm_compressor_unthrottle_threshold_divisor = 20;
616 vm_compressor_catchup_threshold_divisor = 35;
617 } else {
618 vm_compressor_minorcompact_threshold_divisor = 20;
619 vm_compressor_majorcompact_threshold_divisor = 25;
620 vm_compressor_unthrottle_threshold_divisor = 35;
621 vm_compressor_catchup_threshold_divisor = 50;
622 }
623 #endif
624 /*
625 * vm_page_init_lck_grp is now responsible for calling vm_compressor_init_locks
626 * c_master_lock needs to be available early so that "vm_page_find_contiguous" can
627 * use PAGE_REPLACEMENT_ALLOWED to coordinate with the compressor.
628 */
629
630 c_list_lock = lck_mtx_alloc_init(&vm_compressor_lck_grp, &vm_compressor_lck_attr);
631
632 queue_init(&c_bad_list_head);
633 queue_init(&c_age_list_head);
634 queue_init(&c_minor_list_head);
635 queue_init(&c_major_list_head);
636 queue_init(&c_filling_list_head);
637 queue_init(&c_swapout_list_head);
638 queue_init(&c_swapio_list_head);
639 queue_init(&c_swappedin_list_head);
640 queue_init(&c_swappedout_list_head);
641 queue_init(&c_swappedout_sparse_list_head);
642
643 c_free_segno_head = -1;
644 c_segments_available = 0;
645
646 if (vm_compression_limit) {
647 compressor_pool_size = (uint64_t)vm_compression_limit * PAGE_SIZE_64;
648 }
649
650 compressor_pool_max_size = C_SEG_MAX_LIMIT;
651 compressor_pool_max_size *= C_SEG_BUFSIZE;
652
653 #if !CONFIG_EMBEDDED
654
655 if (vm_compression_limit == 0) {
656 if (max_mem <= (4ULL * 1024ULL * 1024ULL * 1024ULL)) {
657 compressor_pool_size = 16ULL * max_mem;
658 } else if (max_mem <= (8ULL * 1024ULL * 1024ULL * 1024ULL)) {
659 compressor_pool_size = 8ULL * max_mem;
660 } else if (max_mem <= (32ULL * 1024ULL * 1024ULL * 1024ULL)) {
661 compressor_pool_size = 4ULL * max_mem;
662 } else {
663 compressor_pool_size = 2ULL * max_mem;
664 }
665 }
666 if (max_mem <= (8ULL * 1024ULL * 1024ULL * 1024ULL)) {
667 compressor_pool_multiplier = 1;
668 } else if (max_mem <= (32ULL * 1024ULL * 1024ULL * 1024ULL)) {
669 compressor_pool_multiplier = 2;
670 } else {
671 compressor_pool_multiplier = 4;
672 }
673
674 #elif defined(__arm__)
675
676 #define VM_RESERVE_SIZE (1024 * 1024 * 256)
677 #define MAX_COMPRESSOR_POOL_SIZE (1024 * 1024 * 450)
678
679 if (compressor_pool_max_size > MAX_COMPRESSOR_POOL_SIZE) {
680 compressor_pool_max_size = MAX_COMPRESSOR_POOL_SIZE;
681 }
682
683 if (vm_compression_limit == 0) {
684 compressor_pool_size = ((kernel_map->max_offset - kernel_map->min_offset) - kernel_map->size) - VM_RESERVE_SIZE;
685 }
686 compressor_pool_multiplier = 1;
687
688 #elif defined(__arm64__) && defined(XNU_TARGET_OS_WATCH)
689
690 /*
691 * On M9 watches the compressor can become big and can lead to
692 * churn in workingset resulting in audio drops. Setting a cap
693 * on the compressor size favors reclaiming unused memory
694 * sitting in idle band via jetsams
695 */
696
697 #define COMPRESSOR_CAP_PERCENTAGE 30ULL
698
699 if (compressor_pool_max_size > max_mem) {
700 compressor_pool_max_size = max_mem;
701 }
702
703 if (vm_compression_limit == 0) {
704 compressor_pool_size = (max_mem * COMPRESSOR_CAP_PERCENTAGE) / 100ULL;
705 }
706 compressor_pool_multiplier = 1;
707
708 #else
709
710 if (compressor_pool_max_size > max_mem) {
711 compressor_pool_max_size = max_mem;
712 }
713
714 if (vm_compression_limit == 0) {
715 compressor_pool_size = max_mem;
716 }
717 compressor_pool_multiplier = 1;
718 #endif
719 if (compressor_pool_size > compressor_pool_max_size) {
720 compressor_pool_size = compressor_pool_max_size;
721 }
722
723 try_again:
724 c_segments_limit = (uint32_t)(compressor_pool_size / (vm_size_t)(C_SEG_ALLOCSIZE));
725 c_segments_nearing_limit = (uint32_t)(((uint64_t)c_segments_limit * 98ULL) / 100ULL);
726
727 c_segment_pages_compressed_limit = (c_segments_limit * (C_SEG_BUFSIZE / PAGE_SIZE) * compressor_pool_multiplier);
728
729 if (c_segment_pages_compressed_limit < (uint32_t)(max_mem / PAGE_SIZE)) {
730 c_segment_pages_compressed_limit = (uint32_t)(max_mem / PAGE_SIZE);
731 }
732
733 c_segment_pages_compressed_nearing_limit = (uint32_t)(((uint64_t)c_segment_pages_compressed_limit * 98ULL) / 100ULL);
734
735 /*
736 * Submap needs space for:
737 * - c_segments
738 * - c_buffers
739 * - swap reclaimations -- C_SEG_BUFSIZE
740 */
741 c_segments_arr_size = vm_map_round_page((sizeof(union c_segu) * c_segments_limit), VM_MAP_PAGE_MASK(kernel_map));
742 c_buffers_size = vm_map_round_page(((vm_size_t)C_SEG_ALLOCSIZE * (vm_size_t)c_segments_limit), VM_MAP_PAGE_MASK(kernel_map));
743
744 compressor_submap_size = c_segments_arr_size + c_buffers_size + C_SEG_BUFSIZE;
745
746 #if RECORD_THE_COMPRESSED_DATA
747 c_compressed_record_sbuf_size = (vm_size_t)C_SEG_ALLOCSIZE + (PAGE_SIZE * 2);
748 compressor_submap_size += c_compressed_record_sbuf_size;
749 #endif /* RECORD_THE_COMPRESSED_DATA */
750
751 vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
752 vmk_flags.vmkf_permanent = TRUE;
753 retval = kmem_suballoc(kernel_map, &start_addr, compressor_submap_size,
754 FALSE, VM_FLAGS_ANYWHERE, vmk_flags, VM_KERN_MEMORY_COMPRESSOR,
755 &compressor_map);
756
757 if (retval != KERN_SUCCESS) {
758 if (++attempts > 3) {
759 panic("vm_compressor_init: kmem_suballoc failed - 0x%llx", (uint64_t)compressor_submap_size);
760 }
761
762 compressor_pool_size = compressor_pool_size / 2;
763
764 kprintf("retrying creation of the compressor submap at 0x%llx bytes\n", compressor_pool_size);
765 goto try_again;
766 }
767 if (kernel_memory_allocate(compressor_map, (vm_offset_t *)(&c_segments), (sizeof(union c_segu) * c_segments_limit), 0, KMA_KOBJECT | KMA_VAONLY | KMA_PERMANENT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS) {
768 panic("vm_compressor_init: kernel_memory_allocate failed - c_segments\n");
769 }
770 if (kernel_memory_allocate(compressor_map, &c_buffers, c_buffers_size, 0, KMA_COMPRESSOR | KMA_VAONLY | KMA_PERMANENT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS) {
771 panic("vm_compressor_init: kernel_memory_allocate failed - c_buffers\n");
772 }
773
774
775 c_segment_min_size = sizeof(struct c_segment) + (C_SEG_SLOT_VAR_ARRAY_MIN_LEN * sizeof(struct c_slot));
776
777 for (c_segment_padded_size = 128; c_segment_padded_size < c_segment_min_size; c_segment_padded_size = c_segment_padded_size << 1) {
778 ;
779 }
780
781 compressor_segment_zone = zinit(c_segment_padded_size, c_segments_limit * c_segment_padded_size, PAGE_SIZE, "compressor_segment");
782 zone_change(compressor_segment_zone, Z_CALLERACCT, FALSE);
783 zone_change(compressor_segment_zone, Z_NOENCRYPT, TRUE);
784
785 c_seg_fixed_array_len = (c_segment_padded_size - sizeof(struct c_segment)) / sizeof(struct c_slot);
786
787 c_segments_busy = FALSE;
788
789 c_segments_next_page = (caddr_t)c_segments;
790 vm_compressor_algorithm_init();
791
792 {
793 host_basic_info_data_t hinfo;
794 mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
795
796 #define BSD_HOST 1
797 host_info((host_t)BSD_HOST, HOST_BASIC_INFO, (host_info_t)&hinfo, &count);
798
799 compressor_cpus = hinfo.max_cpus;
800 compressor_scratch_bufs = kalloc_tag(compressor_cpus * vm_compressor_get_decode_scratch_size(), VM_KERN_MEMORY_COMPRESSOR);
801
802 kdp_compressor_scratch_buf = kalloc_tag(vm_compressor_get_decode_scratch_size(), VM_KERN_MEMORY_COMPRESSOR);
803
804 /*
805 * kdp_compressor_decompressed_page must be page aligned because we access
806 * it through the physical apperture by page number. kalloc() does not
807 * guarantee alignment.
808 */
809 vm_offset_t addr;
810 if (kernel_memory_allocate(kernel_map, &addr, PAGE_SIZE, 0, KMA_KOBJECT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS) {
811 panic("vm_compressor_init: kernel_memory_allocate failed - kdp_compressor_decompressed_page\n");
812 }
813 assert((addr & PAGE_MASK) == 0);
814 kdp_compressor_decompressed_page = (void *)addr;
815 kdp_compressor_decompressed_page_paddr = kvtophys((vm_offset_t)kdp_compressor_decompressed_page);
816 kdp_compressor_decompressed_page_ppnum = (ppnum_t) atop(kdp_compressor_decompressed_page_paddr);
817 }
818 #if CONFIG_FREEZE
819 freezer_compressor_scratch_buf = kalloc_tag(vm_compressor_get_encode_scratch_size(), VM_KERN_MEMORY_COMPRESSOR);
820 #endif
821
822 #if RECORD_THE_COMPRESSED_DATA
823 if (kernel_memory_allocate(compressor_map, (vm_offset_t *)&c_compressed_record_sbuf, c_compressed_record_sbuf_size, 0, KMA_KOBJECT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS) {
824 panic("vm_compressor_init: kernel_memory_allocate failed - c_compressed_record_sbuf\n");
825 }
826
827 c_compressed_record_cptr = c_compressed_record_sbuf;
828 c_compressed_record_ebuf = c_compressed_record_sbuf + c_compressed_record_sbuf_size;
829 #endif
830
831 if (kernel_thread_start_priority((thread_continue_t)vm_compressor_swap_trigger_thread, NULL,
832 BASEPRI_VM, &thread) != KERN_SUCCESS) {
833 panic("vm_compressor_swap_trigger_thread: create failed");
834 }
835 thread_deallocate(thread);
836
837 if (vm_pageout_internal_start() != KERN_SUCCESS) {
838 panic("vm_compressor_init: Failed to start the internal pageout thread.\n");
839 }
840 if (VM_CONFIG_SWAP_IS_PRESENT) {
841 vm_compressor_swap_init();
842 }
843
844 if (VM_CONFIG_COMPRESSOR_IS_ACTIVE) {
845 vm_compressor_is_active = 1;
846 }
847
848 #if CONFIG_FREEZE
849 memorystatus_freeze_enabled = TRUE;
850 #endif /* CONFIG_FREEZE */
851
852 vm_compressor_available = 1;
853
854 vm_page_reactivate_all_throttled();
855 }
856
857
858 #if VALIDATE_C_SEGMENTS
859
860 static void
861 c_seg_validate(c_segment_t c_seg, boolean_t must_be_compact)
862 {
863 int c_indx;
864 int32_t bytes_used;
865 uint32_t c_rounded_size;
866 uint32_t c_size;
867 c_slot_t cs;
868
869 if (__probable(validate_c_segs == FALSE)) {
870 return;
871 }
872 if (c_seg->c_firstemptyslot < c_seg->c_nextslot) {
873 c_indx = c_seg->c_firstemptyslot;
874 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
875
876 if (cs == NULL) {
877 panic("c_seg_validate: no slot backing c_firstemptyslot");
878 }
879
880 if (cs->c_size) {
881 panic("c_seg_validate: c_firstemptyslot has non-zero size (%d)\n", cs->c_size);
882 }
883 }
884 bytes_used = 0;
885
886 for (c_indx = 0; c_indx < c_seg->c_nextslot; c_indx++) {
887 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
888
889 c_size = UNPACK_C_SIZE(cs);
890
891 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
892
893 bytes_used += c_rounded_size;
894
895 #if CHECKSUM_THE_COMPRESSED_DATA
896 unsigned csvhash;
897 if (c_size && cs->c_hash_compressed_data != (csvhash = vmc_hash((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size))) {
898 addr64_t csvphys = kvtophys((vm_offset_t)&c_seg->c_store.c_buffer[cs->c_offset]);
899 panic("Compressed data doesn't match original %p phys: 0x%llx %d %p %d %d 0x%x 0x%x", c_seg, csvphys, cs->c_offset, cs, c_indx, c_size, cs->c_hash_compressed_data, csvhash);
900 }
901 #endif
902 #if POPCOUNT_THE_COMPRESSED_DATA
903 unsigned csvpop;
904 if (c_size) {
905 uintptr_t csvaddr = (uintptr_t) &c_seg->c_store.c_buffer[cs->c_offset];
906 if (cs->c_pop_cdata != (csvpop = vmc_pop(csvaddr, c_size))) {
907 panic("Compressed data popcount doesn't match original, bit distance: %d %p (phys: %p) %p %p 0x%llx 0x%x 0x%x 0x%x", (csvpop - cs->c_pop_cdata), (void *)csvaddr, (void *) kvtophys(csvaddr), c_seg, cs, (uint64_t)cs->c_offset, c_size, csvpop, cs->c_pop_cdata);
908 }
909 }
910 #endif
911 }
912
913 if (bytes_used != c_seg->c_bytes_used) {
914 panic("c_seg_validate: bytes_used mismatch - found %d, segment has %d\n", bytes_used, c_seg->c_bytes_used);
915 }
916
917 if (c_seg->c_bytes_used > C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset)) {
918 panic("c_seg_validate: c_bytes_used > c_nextoffset - c_nextoffset = %d, c_bytes_used = %d\n",
919 (int32_t)C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset), c_seg->c_bytes_used);
920 }
921
922 if (must_be_compact) {
923 if (c_seg->c_bytes_used != C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset)) {
924 panic("c_seg_validate: c_bytes_used doesn't match c_nextoffset - c_nextoffset = %d, c_bytes_used = %d\n",
925 (int32_t)C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset), c_seg->c_bytes_used);
926 }
927 }
928 }
929
930 #endif
931
932
933 void
934 c_seg_need_delayed_compaction(c_segment_t c_seg, boolean_t c_list_lock_held)
935 {
936 boolean_t clear_busy = FALSE;
937
938 if (c_list_lock_held == FALSE) {
939 if (!lck_mtx_try_lock_spin_always(c_list_lock)) {
940 C_SEG_BUSY(c_seg);
941
942 lck_mtx_unlock_always(&c_seg->c_lock);
943 lck_mtx_lock_spin_always(c_list_lock);
944 lck_mtx_lock_spin_always(&c_seg->c_lock);
945
946 clear_busy = TRUE;
947 }
948 }
949 assert(c_seg->c_state != C_IS_FILLING);
950
951 if (!c_seg->c_on_minorcompact_q && !(C_SEG_IS_ON_DISK_OR_SOQ(c_seg))) {
952 queue_enter(&c_minor_list_head, c_seg, c_segment_t, c_list);
953 c_seg->c_on_minorcompact_q = 1;
954 c_minor_count++;
955 }
956 if (c_list_lock_held == FALSE) {
957 lck_mtx_unlock_always(c_list_lock);
958 }
959
960 if (clear_busy == TRUE) {
961 C_SEG_WAKEUP_DONE(c_seg);
962 }
963 }
964
965
966 unsigned int c_seg_moved_to_sparse_list = 0;
967
968 void
969 c_seg_move_to_sparse_list(c_segment_t c_seg)
970 {
971 boolean_t clear_busy = FALSE;
972
973 if (!lck_mtx_try_lock_spin_always(c_list_lock)) {
974 C_SEG_BUSY(c_seg);
975
976 lck_mtx_unlock_always(&c_seg->c_lock);
977 lck_mtx_lock_spin_always(c_list_lock);
978 lck_mtx_lock_spin_always(&c_seg->c_lock);
979
980 clear_busy = TRUE;
981 }
982 c_seg_switch_state(c_seg, C_ON_SWAPPEDOUTSPARSE_Q, FALSE);
983
984 c_seg_moved_to_sparse_list++;
985
986 lck_mtx_unlock_always(c_list_lock);
987
988 if (clear_busy == TRUE) {
989 C_SEG_WAKEUP_DONE(c_seg);
990 }
991 }
992
993
994 void
995 c_seg_insert_into_q(queue_head_t *qhead, c_segment_t c_seg)
996 {
997 c_segment_t c_seg_next;
998
999 if (queue_empty(qhead)) {
1000 queue_enter(qhead, c_seg, c_segment_t, c_age_list);
1001 } else {
1002 c_seg_next = (c_segment_t)queue_first(qhead);
1003
1004 while (TRUE) {
1005 if (c_seg->c_generation_id < c_seg_next->c_generation_id) {
1006 queue_insert_before(qhead, c_seg, c_seg_next, c_segment_t, c_age_list);
1007 break;
1008 }
1009 c_seg_next = (c_segment_t) queue_next(&c_seg_next->c_age_list);
1010
1011 if (queue_end(qhead, (queue_entry_t) c_seg_next)) {
1012 queue_enter(qhead, c_seg, c_segment_t, c_age_list);
1013 break;
1014 }
1015 }
1016 }
1017 }
1018
1019
1020 int try_minor_compaction_failed = 0;
1021 int try_minor_compaction_succeeded = 0;
1022
1023 void
1024 c_seg_try_minor_compaction_and_unlock(c_segment_t c_seg)
1025 {
1026 assert(c_seg->c_on_minorcompact_q);
1027 /*
1028 * c_seg is currently on the delayed minor compaction
1029 * queue and we have c_seg locked... if we can get the
1030 * c_list_lock w/o blocking (if we blocked we could deadlock
1031 * because the lock order is c_list_lock then c_seg's lock)
1032 * we'll pull it from the delayed list and free it directly
1033 */
1034 if (!lck_mtx_try_lock_spin_always(c_list_lock)) {
1035 /*
1036 * c_list_lock is held, we need to bail
1037 */
1038 try_minor_compaction_failed++;
1039
1040 lck_mtx_unlock_always(&c_seg->c_lock);
1041 } else {
1042 try_minor_compaction_succeeded++;
1043
1044 C_SEG_BUSY(c_seg);
1045 c_seg_do_minor_compaction_and_unlock(c_seg, TRUE, FALSE, FALSE);
1046 }
1047 }
1048
1049
1050 int
1051 c_seg_do_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy, boolean_t need_list_lock, boolean_t disallow_page_replacement)
1052 {
1053 int c_seg_freed;
1054
1055 assert(c_seg->c_busy);
1056 assert(!C_SEG_IS_ON_DISK_OR_SOQ(c_seg));
1057
1058 /*
1059 * check for the case that can occur when we are not swapping
1060 * and this segment has been major compacted in the past
1061 * and moved to the majorcompact q to remove it from further
1062 * consideration... if the occupancy falls too low we need
1063 * to put it back on the age_q so that it will be considered
1064 * in the next major compaction sweep... if we don't do this
1065 * we will eventually run into the c_segments_limit
1066 */
1067 if (c_seg->c_state == C_ON_MAJORCOMPACT_Q && C_SEG_SHOULD_MAJORCOMPACT_NOW(c_seg)) {
1068 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
1069 }
1070 if (!c_seg->c_on_minorcompact_q) {
1071 if (clear_busy == TRUE) {
1072 C_SEG_WAKEUP_DONE(c_seg);
1073 }
1074
1075 lck_mtx_unlock_always(&c_seg->c_lock);
1076
1077 return 0;
1078 }
1079 queue_remove(&c_minor_list_head, c_seg, c_segment_t, c_list);
1080 c_seg->c_on_minorcompact_q = 0;
1081 c_minor_count--;
1082
1083 lck_mtx_unlock_always(c_list_lock);
1084
1085 if (disallow_page_replacement == TRUE) {
1086 lck_mtx_unlock_always(&c_seg->c_lock);
1087
1088 PAGE_REPLACEMENT_DISALLOWED(TRUE);
1089
1090 lck_mtx_lock_spin_always(&c_seg->c_lock);
1091 }
1092 c_seg_freed = c_seg_minor_compaction_and_unlock(c_seg, clear_busy);
1093
1094 if (disallow_page_replacement == TRUE) {
1095 PAGE_REPLACEMENT_DISALLOWED(FALSE);
1096 }
1097
1098 if (need_list_lock == TRUE) {
1099 lck_mtx_lock_spin_always(c_list_lock);
1100 }
1101
1102 return c_seg_freed;
1103 }
1104
1105 void
1106 kdp_compressor_busy_find_owner(event64_t wait_event, thread_waitinfo_t *waitinfo)
1107 {
1108 c_segment_t c_seg = (c_segment_t) wait_event;
1109
1110 waitinfo->owner = thread_tid(c_seg->c_busy_for_thread);
1111 waitinfo->context = VM_KERNEL_UNSLIDE_OR_PERM(c_seg);
1112 }
1113
1114 #if DEVELOPMENT || DEBUG
1115 int
1116 do_cseg_wedge_thread(void)
1117 {
1118 struct c_segment c_seg;
1119 c_seg.c_busy_for_thread = current_thread();
1120
1121 debug_cseg_wait_event = (event_t) &c_seg;
1122
1123 thread_set_pending_block_hint(current_thread(), kThreadWaitCompressor);
1124 assert_wait((event_t) (&c_seg), THREAD_INTERRUPTIBLE);
1125
1126 thread_block(THREAD_CONTINUE_NULL);
1127
1128 return 0;
1129 }
1130
1131 int
1132 do_cseg_unwedge_thread(void)
1133 {
1134 thread_wakeup(debug_cseg_wait_event);
1135 debug_cseg_wait_event = NULL;
1136
1137 return 0;
1138 }
1139 #endif /* DEVELOPMENT || DEBUG */
1140
1141 void
1142 c_seg_wait_on_busy(c_segment_t c_seg)
1143 {
1144 c_seg->c_wanted = 1;
1145
1146 thread_set_pending_block_hint(current_thread(), kThreadWaitCompressor);
1147 assert_wait((event_t) (c_seg), THREAD_UNINT);
1148
1149 lck_mtx_unlock_always(&c_seg->c_lock);
1150 thread_block(THREAD_CONTINUE_NULL);
1151 }
1152
1153
1154 void
1155 c_seg_switch_state(c_segment_t c_seg, int new_state, boolean_t insert_head)
1156 {
1157 int old_state = c_seg->c_state;
1158
1159 #if !CONFIG_EMBEDDED
1160 #if DEVELOPMENT || DEBUG
1161 if (new_state != C_IS_FILLING) {
1162 LCK_MTX_ASSERT(&c_seg->c_lock, LCK_MTX_ASSERT_OWNED);
1163 }
1164 LCK_MTX_ASSERT(c_list_lock, LCK_MTX_ASSERT_OWNED);
1165 #endif
1166 #endif /* !CONFIG_EMBEDDED */
1167 switch (old_state) {
1168 case C_IS_EMPTY:
1169 assert(new_state == C_IS_FILLING || new_state == C_IS_FREE);
1170
1171 c_empty_count--;
1172 break;
1173
1174 case C_IS_FILLING:
1175 assert(new_state == C_ON_AGE_Q || new_state == C_ON_SWAPOUT_Q);
1176
1177 queue_remove(&c_filling_list_head, c_seg, c_segment_t, c_age_list);
1178 c_filling_count--;
1179 break;
1180
1181 case C_ON_AGE_Q:
1182 assert(new_state == C_ON_SWAPOUT_Q || new_state == C_ON_MAJORCOMPACT_Q ||
1183 new_state == C_IS_FREE);
1184
1185 queue_remove(&c_age_list_head, c_seg, c_segment_t, c_age_list);
1186 c_age_count--;
1187 break;
1188
1189 case C_ON_SWAPPEDIN_Q:
1190 assert(new_state == C_ON_AGE_Q || new_state == C_IS_FREE);
1191
1192 queue_remove(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
1193 c_swappedin_count--;
1194 break;
1195
1196 case C_ON_SWAPOUT_Q:
1197 assert(new_state == C_ON_AGE_Q || new_state == C_IS_FREE || new_state == C_IS_EMPTY || new_state == C_ON_SWAPIO_Q);
1198
1199 queue_remove(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
1200 thread_wakeup((event_t)&compaction_swapper_running);
1201 c_swapout_count--;
1202 break;
1203
1204 case C_ON_SWAPIO_Q:
1205 assert(new_state == C_ON_SWAPPEDOUT_Q || new_state == C_ON_SWAPPEDOUTSPARSE_Q || new_state == C_ON_AGE_Q);
1206
1207 queue_remove(&c_swapio_list_head, c_seg, c_segment_t, c_age_list);
1208 c_swapio_count--;
1209 break;
1210
1211 case C_ON_SWAPPEDOUT_Q:
1212 assert(new_state == C_ON_SWAPPEDIN_Q || new_state == C_ON_AGE_Q ||
1213 new_state == C_ON_SWAPPEDOUTSPARSE_Q ||
1214 new_state == C_ON_BAD_Q || new_state == C_IS_EMPTY || new_state == C_IS_FREE);
1215
1216 queue_remove(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
1217 c_swappedout_count--;
1218 break;
1219
1220 case C_ON_SWAPPEDOUTSPARSE_Q:
1221 assert(new_state == C_ON_SWAPPEDIN_Q || new_state == C_ON_AGE_Q ||
1222 new_state == C_ON_BAD_Q || new_state == C_IS_EMPTY || new_state == C_IS_FREE);
1223
1224 queue_remove(&c_swappedout_sparse_list_head, c_seg, c_segment_t, c_age_list);
1225 c_swappedout_sparse_count--;
1226 break;
1227
1228 case C_ON_MAJORCOMPACT_Q:
1229 assert(new_state == C_ON_AGE_Q || new_state == C_IS_FREE);
1230
1231 queue_remove(&c_major_list_head, c_seg, c_segment_t, c_age_list);
1232 c_major_count--;
1233 break;
1234
1235 case C_ON_BAD_Q:
1236 assert(new_state == C_IS_FREE);
1237
1238 queue_remove(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
1239 c_bad_count--;
1240 break;
1241
1242 default:
1243 panic("c_seg %p has bad c_state = %d\n", c_seg, old_state);
1244 }
1245
1246 switch (new_state) {
1247 case C_IS_FREE:
1248 assert(old_state != C_IS_FILLING);
1249
1250 break;
1251
1252 case C_IS_EMPTY:
1253 assert(old_state == C_ON_SWAPOUT_Q || old_state == C_ON_SWAPPEDOUT_Q || old_state == C_ON_SWAPPEDOUTSPARSE_Q);
1254
1255 c_empty_count++;
1256 break;
1257
1258 case C_IS_FILLING:
1259 assert(old_state == C_IS_EMPTY);
1260
1261 queue_enter(&c_filling_list_head, c_seg, c_segment_t, c_age_list);
1262 c_filling_count++;
1263 break;
1264
1265 case C_ON_AGE_Q:
1266 assert(old_state == C_IS_FILLING || old_state == C_ON_SWAPPEDIN_Q ||
1267 old_state == C_ON_SWAPOUT_Q || old_state == C_ON_SWAPIO_Q ||
1268 old_state == C_ON_MAJORCOMPACT_Q || old_state == C_ON_SWAPPEDOUT_Q || old_state == C_ON_SWAPPEDOUTSPARSE_Q);
1269
1270 if (old_state == C_IS_FILLING) {
1271 queue_enter(&c_age_list_head, c_seg, c_segment_t, c_age_list);
1272 } else {
1273 if (!queue_empty(&c_age_list_head)) {
1274 c_segment_t c_first;
1275
1276 c_first = (c_segment_t)queue_first(&c_age_list_head);
1277 c_seg->c_creation_ts = c_first->c_creation_ts;
1278 }
1279 queue_enter_first(&c_age_list_head, c_seg, c_segment_t, c_age_list);
1280 }
1281 c_age_count++;
1282 break;
1283
1284 case C_ON_SWAPPEDIN_Q:
1285 assert(old_state == C_ON_SWAPPEDOUT_Q || old_state == C_ON_SWAPPEDOUTSPARSE_Q);
1286
1287 if (insert_head == TRUE) {
1288 queue_enter_first(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
1289 } else {
1290 queue_enter(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
1291 }
1292 c_swappedin_count++;
1293 break;
1294
1295 case C_ON_SWAPOUT_Q:
1296 assert(old_state == C_ON_AGE_Q || old_state == C_IS_FILLING);
1297
1298 if (insert_head == TRUE) {
1299 queue_enter_first(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
1300 } else {
1301 queue_enter(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
1302 }
1303 c_swapout_count++;
1304 break;
1305
1306 case C_ON_SWAPIO_Q:
1307 assert(old_state == C_ON_SWAPOUT_Q);
1308
1309 if (insert_head == TRUE) {
1310 queue_enter_first(&c_swapio_list_head, c_seg, c_segment_t, c_age_list);
1311 } else {
1312 queue_enter(&c_swapio_list_head, c_seg, c_segment_t, c_age_list);
1313 }
1314 c_swapio_count++;
1315 break;
1316
1317 case C_ON_SWAPPEDOUT_Q:
1318 assert(old_state == C_ON_SWAPIO_Q);
1319
1320 if (insert_head == TRUE) {
1321 queue_enter_first(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
1322 } else {
1323 queue_enter(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
1324 }
1325 c_swappedout_count++;
1326 break;
1327
1328 case C_ON_SWAPPEDOUTSPARSE_Q:
1329 assert(old_state == C_ON_SWAPIO_Q || old_state == C_ON_SWAPPEDOUT_Q);
1330
1331 if (insert_head == TRUE) {
1332 queue_enter_first(&c_swappedout_sparse_list_head, c_seg, c_segment_t, c_age_list);
1333 } else {
1334 queue_enter(&c_swappedout_sparse_list_head, c_seg, c_segment_t, c_age_list);
1335 }
1336
1337 c_swappedout_sparse_count++;
1338 break;
1339
1340 case C_ON_MAJORCOMPACT_Q:
1341 assert(old_state == C_ON_AGE_Q);
1342
1343 if (insert_head == TRUE) {
1344 queue_enter_first(&c_major_list_head, c_seg, c_segment_t, c_age_list);
1345 } else {
1346 queue_enter(&c_major_list_head, c_seg, c_segment_t, c_age_list);
1347 }
1348 c_major_count++;
1349 break;
1350
1351 case C_ON_BAD_Q:
1352 assert(old_state == C_ON_SWAPPEDOUT_Q || old_state == C_ON_SWAPPEDOUTSPARSE_Q);
1353
1354 if (insert_head == TRUE) {
1355 queue_enter_first(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
1356 } else {
1357 queue_enter(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
1358 }
1359 c_bad_count++;
1360 break;
1361
1362 default:
1363 panic("c_seg %p requesting bad c_state = %d\n", c_seg, new_state);
1364 }
1365 c_seg->c_state = new_state;
1366 }
1367
1368
1369
1370 void
1371 c_seg_free(c_segment_t c_seg)
1372 {
1373 assert(c_seg->c_busy);
1374
1375 lck_mtx_unlock_always(&c_seg->c_lock);
1376 lck_mtx_lock_spin_always(c_list_lock);
1377 lck_mtx_lock_spin_always(&c_seg->c_lock);
1378
1379 c_seg_free_locked(c_seg);
1380 }
1381
1382
1383 void
1384 c_seg_free_locked(c_segment_t c_seg)
1385 {
1386 int segno;
1387 int pages_populated = 0;
1388 int32_t *c_buffer = NULL;
1389 uint64_t c_swap_handle = 0;
1390
1391 assert(c_seg->c_busy);
1392 assert(c_seg->c_slots_used == 0);
1393 assert(!c_seg->c_on_minorcompact_q);
1394 assert(!c_seg->c_busy_swapping);
1395
1396 if (c_seg->c_overage_swap == TRUE) {
1397 c_overage_swapped_count--;
1398 c_seg->c_overage_swap = FALSE;
1399 }
1400 if (!(C_SEG_IS_ONDISK(c_seg))) {
1401 c_buffer = c_seg->c_store.c_buffer;
1402 } else {
1403 c_swap_handle = c_seg->c_store.c_swap_handle;
1404 }
1405
1406 c_seg_switch_state(c_seg, C_IS_FREE, FALSE);
1407
1408 lck_mtx_unlock_always(c_list_lock);
1409
1410 if (c_buffer) {
1411 pages_populated = (round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE;
1412 c_seg->c_store.c_buffer = NULL;
1413 } else {
1414 c_seg->c_store.c_swap_handle = (uint64_t)-1;
1415 }
1416
1417 lck_mtx_unlock_always(&c_seg->c_lock);
1418
1419 if (c_buffer) {
1420 if (pages_populated) {
1421 kernel_memory_depopulate(compressor_map, (vm_offset_t) c_buffer, pages_populated * PAGE_SIZE, KMA_COMPRESSOR);
1422 }
1423 } else if (c_swap_handle) {
1424 /*
1425 * Free swap space on disk.
1426 */
1427 vm_swap_free(c_swap_handle);
1428 }
1429 lck_mtx_lock_spin_always(&c_seg->c_lock);
1430 /*
1431 * c_seg must remain busy until
1432 * after the call to vm_swap_free
1433 */
1434 C_SEG_WAKEUP_DONE(c_seg);
1435 lck_mtx_unlock_always(&c_seg->c_lock);
1436
1437 segno = c_seg->c_mysegno;
1438
1439 lck_mtx_lock_spin_always(c_list_lock);
1440 /*
1441 * because the c_buffer is now associated with the segno,
1442 * we can't put the segno back on the free list until
1443 * after we have depopulated the c_buffer range, or
1444 * we run the risk of depopulating a range that is
1445 * now being used in one of the compressor heads
1446 */
1447 c_segments[segno].c_segno = c_free_segno_head;
1448 c_free_segno_head = segno;
1449 c_segment_count--;
1450
1451 lck_mtx_unlock_always(c_list_lock);
1452
1453 lck_mtx_destroy(&c_seg->c_lock, &vm_compressor_lck_grp);
1454
1455 if (c_seg->c_slot_var_array_len) {
1456 kfree(c_seg->c_slot_var_array, sizeof(struct c_slot) * c_seg->c_slot_var_array_len);
1457 }
1458
1459 zfree(compressor_segment_zone, c_seg);
1460 }
1461
1462 #if DEVELOPMENT || DEBUG
1463 int c_seg_trim_page_count = 0;
1464 #endif
1465
1466 void
1467 c_seg_trim_tail(c_segment_t c_seg)
1468 {
1469 c_slot_t cs;
1470 uint32_t c_size;
1471 uint32_t c_offset;
1472 uint32_t c_rounded_size;
1473 uint16_t current_nextslot;
1474 uint32_t current_populated_offset;
1475
1476 if (c_seg->c_bytes_used == 0) {
1477 return;
1478 }
1479 current_nextslot = c_seg->c_nextslot;
1480 current_populated_offset = c_seg->c_populated_offset;
1481
1482 while (c_seg->c_nextslot) {
1483 cs = C_SEG_SLOT_FROM_INDEX(c_seg, (c_seg->c_nextslot - 1));
1484
1485 c_size = UNPACK_C_SIZE(cs);
1486
1487 if (c_size) {
1488 if (current_nextslot != c_seg->c_nextslot) {
1489 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
1490 c_offset = cs->c_offset + C_SEG_BYTES_TO_OFFSET(c_rounded_size);
1491
1492 c_seg->c_nextoffset = c_offset;
1493 c_seg->c_populated_offset = (c_offset + (C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1)) &
1494 ~(C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1);
1495
1496 if (c_seg->c_firstemptyslot > c_seg->c_nextslot) {
1497 c_seg->c_firstemptyslot = c_seg->c_nextslot;
1498 }
1499 #if DEVELOPMENT || DEBUG
1500 c_seg_trim_page_count += ((round_page_32(C_SEG_OFFSET_TO_BYTES(current_populated_offset)) -
1501 round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) /
1502 PAGE_SIZE);
1503 #endif
1504 }
1505 break;
1506 }
1507 c_seg->c_nextslot--;
1508 }
1509 assert(c_seg->c_nextslot);
1510 }
1511
1512
1513 int
1514 c_seg_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy)
1515 {
1516 c_slot_mapping_t slot_ptr;
1517 uint32_t c_offset = 0;
1518 uint32_t old_populated_offset;
1519 uint32_t c_rounded_size;
1520 uint32_t c_size;
1521 int c_indx = 0;
1522 int i;
1523 c_slot_t c_dst;
1524 c_slot_t c_src;
1525
1526 assert(c_seg->c_busy);
1527
1528 #if VALIDATE_C_SEGMENTS
1529 c_seg_validate(c_seg, FALSE);
1530 #endif
1531 if (c_seg->c_bytes_used == 0) {
1532 c_seg_free(c_seg);
1533 return 1;
1534 }
1535 lck_mtx_unlock_always(&c_seg->c_lock);
1536
1537 if (c_seg->c_firstemptyslot >= c_seg->c_nextslot || C_SEG_UNUSED_BYTES(c_seg) < PAGE_SIZE) {
1538 goto done;
1539 }
1540
1541 /* TODO: assert first emptyslot's c_size is actually 0 */
1542
1543 #if DEVELOPMENT || DEBUG
1544 C_SEG_MAKE_WRITEABLE(c_seg);
1545 #endif
1546
1547 #if VALIDATE_C_SEGMENTS
1548 c_seg->c_was_minor_compacted++;
1549 #endif
1550 c_indx = c_seg->c_firstemptyslot;
1551 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
1552
1553 old_populated_offset = c_seg->c_populated_offset;
1554 c_offset = c_dst->c_offset;
1555
1556 for (i = c_indx + 1; i < c_seg->c_nextslot && c_offset < c_seg->c_nextoffset; i++) {
1557 c_src = C_SEG_SLOT_FROM_INDEX(c_seg, i);
1558
1559 c_size = UNPACK_C_SIZE(c_src);
1560
1561 if (c_size == 0) {
1562 continue;
1563 }
1564
1565 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
1566 /* N.B.: This memcpy may be an overlapping copy */
1567 memcpy(&c_seg->c_store.c_buffer[c_offset], &c_seg->c_store.c_buffer[c_src->c_offset], c_rounded_size);
1568
1569 cslot_copy(c_dst, c_src);
1570 c_dst->c_offset = c_offset;
1571
1572 slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
1573 slot_ptr->s_cindx = c_indx;
1574
1575 c_offset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
1576 PACK_C_SIZE(c_src, 0);
1577 c_indx++;
1578
1579 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
1580 }
1581 c_seg->c_firstemptyslot = c_indx;
1582 c_seg->c_nextslot = c_indx;
1583 c_seg->c_nextoffset = c_offset;
1584 c_seg->c_populated_offset = (c_offset + (C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1)) & ~(C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1);
1585 c_seg->c_bytes_unused = 0;
1586
1587 #if VALIDATE_C_SEGMENTS
1588 c_seg_validate(c_seg, TRUE);
1589 #endif
1590 if (old_populated_offset > c_seg->c_populated_offset) {
1591 uint32_t gc_size;
1592 int32_t *gc_ptr;
1593
1594 gc_size = C_SEG_OFFSET_TO_BYTES(old_populated_offset - c_seg->c_populated_offset);
1595 gc_ptr = &c_seg->c_store.c_buffer[c_seg->c_populated_offset];
1596
1597 kernel_memory_depopulate(compressor_map, (vm_offset_t)gc_ptr, gc_size, KMA_COMPRESSOR);
1598 }
1599
1600 #if DEVELOPMENT || DEBUG
1601 C_SEG_WRITE_PROTECT(c_seg);
1602 #endif
1603
1604 done:
1605 if (clear_busy == TRUE) {
1606 lck_mtx_lock_spin_always(&c_seg->c_lock);
1607 C_SEG_WAKEUP_DONE(c_seg);
1608 lck_mtx_unlock_always(&c_seg->c_lock);
1609 }
1610 return 0;
1611 }
1612
1613
1614 static void
1615 c_seg_alloc_nextslot(c_segment_t c_seg)
1616 {
1617 struct c_slot *old_slot_array = NULL;
1618 struct c_slot *new_slot_array = NULL;
1619 int newlen;
1620 int oldlen;
1621
1622 if (c_seg->c_nextslot < c_seg_fixed_array_len) {
1623 return;
1624 }
1625
1626 if ((c_seg->c_nextslot - c_seg_fixed_array_len) >= c_seg->c_slot_var_array_len) {
1627 oldlen = c_seg->c_slot_var_array_len;
1628 old_slot_array = c_seg->c_slot_var_array;
1629
1630 if (oldlen == 0) {
1631 newlen = C_SEG_SLOT_VAR_ARRAY_MIN_LEN;
1632 } else {
1633 newlen = oldlen * 2;
1634 }
1635
1636 new_slot_array = (struct c_slot *)kalloc(sizeof(struct c_slot) * newlen);
1637
1638 lck_mtx_lock_spin_always(&c_seg->c_lock);
1639
1640 if (old_slot_array) {
1641 memcpy((char *)new_slot_array, (char *)old_slot_array, sizeof(struct c_slot) * oldlen);
1642 }
1643
1644 c_seg->c_slot_var_array_len = newlen;
1645 c_seg->c_slot_var_array = new_slot_array;
1646
1647 lck_mtx_unlock_always(&c_seg->c_lock);
1648
1649 if (old_slot_array) {
1650 kfree(old_slot_array, sizeof(struct c_slot) * oldlen);
1651 }
1652 }
1653 }
1654
1655
1656
1657 struct {
1658 uint64_t asked_permission;
1659 uint64_t compactions;
1660 uint64_t moved_slots;
1661 uint64_t moved_bytes;
1662 uint64_t wasted_space_in_swapouts;
1663 uint64_t count_of_swapouts;
1664 uint64_t count_of_freed_segs;
1665 } c_seg_major_compact_stats;
1666
1667
1668 #define C_MAJOR_COMPACTION_SIZE_APPROPRIATE ((C_SEG_BUFSIZE * 90) / 100)
1669
1670
1671 boolean_t
1672 c_seg_major_compact_ok(
1673 c_segment_t c_seg_dst,
1674 c_segment_t c_seg_src)
1675 {
1676 c_seg_major_compact_stats.asked_permission++;
1677
1678 if (c_seg_src->c_bytes_used >= C_MAJOR_COMPACTION_SIZE_APPROPRIATE &&
1679 c_seg_dst->c_bytes_used >= C_MAJOR_COMPACTION_SIZE_APPROPRIATE) {
1680 return FALSE;
1681 }
1682
1683 if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX_INDEX) {
1684 /*
1685 * destination segment is full... can't compact
1686 */
1687 return FALSE;
1688 }
1689
1690 return TRUE;
1691 }
1692
1693
1694 boolean_t
1695 c_seg_major_compact(
1696 c_segment_t c_seg_dst,
1697 c_segment_t c_seg_src)
1698 {
1699 c_slot_mapping_t slot_ptr;
1700 uint32_t c_rounded_size;
1701 uint32_t c_size;
1702 uint16_t dst_slot;
1703 int i;
1704 c_slot_t c_dst;
1705 c_slot_t c_src;
1706 boolean_t keep_compacting = TRUE;
1707
1708 /*
1709 * segments are not locked but they are both marked c_busy
1710 * which keeps c_decompress from working on them...
1711 * we can safely allocate new pages, move compressed data
1712 * from c_seg_src to c_seg_dst and update both c_segment's
1713 * state w/o holding the master lock
1714 */
1715 #if DEVELOPMENT || DEBUG
1716 C_SEG_MAKE_WRITEABLE(c_seg_dst);
1717 #endif
1718
1719 #if VALIDATE_C_SEGMENTS
1720 c_seg_dst->c_was_major_compacted++;
1721 c_seg_src->c_was_major_donor++;
1722 #endif
1723 c_seg_major_compact_stats.compactions++;
1724
1725 dst_slot = c_seg_dst->c_nextslot;
1726
1727 for (i = 0; i < c_seg_src->c_nextslot; i++) {
1728 c_src = C_SEG_SLOT_FROM_INDEX(c_seg_src, i);
1729
1730 c_size = UNPACK_C_SIZE(c_src);
1731
1732 if (c_size == 0) {
1733 /* BATCH: move what we have so far; */
1734 continue;
1735 }
1736
1737 if (C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset - c_seg_dst->c_nextoffset) < (unsigned) c_size) {
1738 int size_to_populate;
1739
1740 /* doesn't fit */
1741 size_to_populate = C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset);
1742
1743 if (size_to_populate == 0) {
1744 /* can't fit */
1745 keep_compacting = FALSE;
1746 break;
1747 }
1748 if (size_to_populate > C_SEG_MAX_POPULATE_SIZE) {
1749 size_to_populate = C_SEG_MAX_POPULATE_SIZE;
1750 }
1751
1752 kernel_memory_populate(compressor_map,
1753 (vm_offset_t) &c_seg_dst->c_store.c_buffer[c_seg_dst->c_populated_offset],
1754 size_to_populate,
1755 KMA_COMPRESSOR,
1756 VM_KERN_MEMORY_COMPRESSOR);
1757
1758 c_seg_dst->c_populated_offset += C_SEG_BYTES_TO_OFFSET(size_to_populate);
1759 assert(C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset) <= C_SEG_BUFSIZE);
1760 }
1761 c_seg_alloc_nextslot(c_seg_dst);
1762
1763 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, c_seg_dst->c_nextslot);
1764
1765 memcpy(&c_seg_dst->c_store.c_buffer[c_seg_dst->c_nextoffset], &c_seg_src->c_store.c_buffer[c_src->c_offset], c_size);
1766
1767 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
1768
1769 c_seg_major_compact_stats.moved_slots++;
1770 c_seg_major_compact_stats.moved_bytes += c_size;
1771
1772 cslot_copy(c_dst, c_src);
1773 c_dst->c_offset = c_seg_dst->c_nextoffset;
1774
1775 if (c_seg_dst->c_firstemptyslot == c_seg_dst->c_nextslot) {
1776 c_seg_dst->c_firstemptyslot++;
1777 }
1778 c_seg_dst->c_slots_used++;
1779 c_seg_dst->c_nextslot++;
1780 c_seg_dst->c_bytes_used += c_rounded_size;
1781 c_seg_dst->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
1782
1783 PACK_C_SIZE(c_src, 0);
1784
1785 c_seg_src->c_bytes_used -= c_rounded_size;
1786 c_seg_src->c_bytes_unused += c_rounded_size;
1787 c_seg_src->c_firstemptyslot = 0;
1788
1789 assert(c_seg_src->c_slots_used);
1790 c_seg_src->c_slots_used--;
1791
1792 if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX_INDEX) {
1793 /* dest segment is now full */
1794 keep_compacting = FALSE;
1795 break;
1796 }
1797 }
1798 #if DEVELOPMENT || DEBUG
1799 C_SEG_WRITE_PROTECT(c_seg_dst);
1800 #endif
1801 if (dst_slot < c_seg_dst->c_nextslot) {
1802 PAGE_REPLACEMENT_ALLOWED(TRUE);
1803 /*
1804 * we've now locked out c_decompress from
1805 * converting the slot passed into it into
1806 * a c_segment_t which allows us to use
1807 * the backptr to change which c_segment and
1808 * index the slot points to
1809 */
1810 while (dst_slot < c_seg_dst->c_nextslot) {
1811 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, dst_slot);
1812
1813 slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
1814 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
1815 slot_ptr->s_cseg = c_seg_dst->c_mysegno + 1;
1816 slot_ptr->s_cindx = dst_slot++;
1817 }
1818 PAGE_REPLACEMENT_ALLOWED(FALSE);
1819 }
1820 return keep_compacting;
1821 }
1822
1823
1824 uint64_t
1825 vm_compressor_compute_elapsed_msecs(clock_sec_t end_sec, clock_nsec_t end_nsec, clock_sec_t start_sec, clock_nsec_t start_nsec)
1826 {
1827 uint64_t end_msecs;
1828 uint64_t start_msecs;
1829
1830 end_msecs = (end_sec * 1000) + end_nsec / 1000000;
1831 start_msecs = (start_sec * 1000) + start_nsec / 1000000;
1832
1833 return end_msecs - start_msecs;
1834 }
1835
1836
1837
1838 uint32_t compressor_eval_period_in_msecs = 250;
1839 uint32_t compressor_sample_min_in_msecs = 500;
1840 uint32_t compressor_sample_max_in_msecs = 10000;
1841 uint32_t compressor_thrashing_threshold_per_10msecs = 50;
1842 uint32_t compressor_thrashing_min_per_10msecs = 20;
1843
1844 /* When true, reset sample data next chance we get. */
1845 static boolean_t compressor_need_sample_reset = FALSE;
1846
1847
1848 void
1849 compute_swapout_target_age(void)
1850 {
1851 clock_sec_t cur_ts_sec;
1852 clock_nsec_t cur_ts_nsec;
1853 uint32_t min_operations_needed_in_this_sample;
1854 uint64_t elapsed_msecs_in_eval;
1855 uint64_t elapsed_msecs_in_sample;
1856 boolean_t need_eval_reset = FALSE;
1857
1858 clock_get_system_nanotime(&cur_ts_sec, &cur_ts_nsec);
1859
1860 elapsed_msecs_in_sample = vm_compressor_compute_elapsed_msecs(cur_ts_sec, cur_ts_nsec, start_of_sample_period_sec, start_of_sample_period_nsec);
1861
1862 if (compressor_need_sample_reset ||
1863 elapsed_msecs_in_sample >= compressor_sample_max_in_msecs) {
1864 compressor_need_sample_reset = TRUE;
1865 need_eval_reset = TRUE;
1866 goto done;
1867 }
1868 elapsed_msecs_in_eval = vm_compressor_compute_elapsed_msecs(cur_ts_sec, cur_ts_nsec, start_of_eval_period_sec, start_of_eval_period_nsec);
1869
1870 if (elapsed_msecs_in_eval < compressor_eval_period_in_msecs) {
1871 goto done;
1872 }
1873 need_eval_reset = TRUE;
1874
1875 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_START, elapsed_msecs_in_eval, sample_period_compression_count, sample_period_decompression_count, 0, 0);
1876
1877 min_operations_needed_in_this_sample = (compressor_thrashing_min_per_10msecs * (uint32_t)elapsed_msecs_in_eval) / 10;
1878
1879 if ((sample_period_compression_count - last_eval_compression_count) < min_operations_needed_in_this_sample ||
1880 (sample_period_decompression_count - last_eval_decompression_count) < min_operations_needed_in_this_sample) {
1881 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, sample_period_compression_count - last_eval_compression_count,
1882 sample_period_decompression_count - last_eval_decompression_count, 0, 1, 0);
1883
1884 swapout_target_age = 0;
1885
1886 compressor_need_sample_reset = TRUE;
1887 need_eval_reset = TRUE;
1888 goto done;
1889 }
1890 last_eval_compression_count = sample_period_compression_count;
1891 last_eval_decompression_count = sample_period_decompression_count;
1892
1893 if (elapsed_msecs_in_sample < compressor_sample_min_in_msecs) {
1894 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, swapout_target_age, 0, 0, 5, 0);
1895 goto done;
1896 }
1897 if (sample_period_decompression_count > ((compressor_thrashing_threshold_per_10msecs * elapsed_msecs_in_sample) / 10)) {
1898 uint64_t running_total;
1899 uint64_t working_target;
1900 uint64_t aging_target;
1901 uint32_t oldest_age_of_csegs_sampled = 0;
1902 uint64_t working_set_approximation = 0;
1903
1904 swapout_target_age = 0;
1905
1906 working_target = (sample_period_decompression_count / 100) * 95; /* 95 percent */
1907 aging_target = (sample_period_decompression_count / 100) * 1; /* 1 percent */
1908 running_total = 0;
1909
1910 for (oldest_age_of_csegs_sampled = 0; oldest_age_of_csegs_sampled < DECOMPRESSION_SAMPLE_MAX_AGE; oldest_age_of_csegs_sampled++) {
1911 running_total += age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
1912
1913 working_set_approximation += oldest_age_of_csegs_sampled * age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
1914
1915 if (running_total >= working_target) {
1916 break;
1917 }
1918 }
1919 if (oldest_age_of_csegs_sampled < DECOMPRESSION_SAMPLE_MAX_AGE) {
1920 working_set_approximation = (working_set_approximation * 1000) / elapsed_msecs_in_sample;
1921
1922 if (working_set_approximation < VM_PAGE_COMPRESSOR_COUNT) {
1923 running_total = overage_decompressions_during_sample_period;
1924
1925 for (oldest_age_of_csegs_sampled = DECOMPRESSION_SAMPLE_MAX_AGE - 1; oldest_age_of_csegs_sampled; oldest_age_of_csegs_sampled--) {
1926 running_total += age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
1927
1928 if (running_total >= aging_target) {
1929 break;
1930 }
1931 }
1932 swapout_target_age = (uint32_t)cur_ts_sec - oldest_age_of_csegs_sampled;
1933
1934 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, swapout_target_age, working_set_approximation, VM_PAGE_COMPRESSOR_COUNT, 2, 0);
1935 } else {
1936 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, working_set_approximation, VM_PAGE_COMPRESSOR_COUNT, 0, 3, 0);
1937 }
1938 } else {
1939 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, working_target, running_total, 0, 4, 0);
1940 }
1941
1942 compressor_need_sample_reset = TRUE;
1943 need_eval_reset = TRUE;
1944 } else {
1945 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, sample_period_decompression_count, (compressor_thrashing_threshold_per_10msecs * elapsed_msecs_in_sample) / 10, 0, 6, 0);
1946 }
1947 done:
1948 if (compressor_need_sample_reset == TRUE) {
1949 bzero(age_of_decompressions_during_sample_period, sizeof(age_of_decompressions_during_sample_period));
1950 overage_decompressions_during_sample_period = 0;
1951
1952 start_of_sample_period_sec = cur_ts_sec;
1953 start_of_sample_period_nsec = cur_ts_nsec;
1954 sample_period_decompression_count = 0;
1955 sample_period_compression_count = 0;
1956 last_eval_decompression_count = 0;
1957 last_eval_compression_count = 0;
1958 compressor_need_sample_reset = FALSE;
1959 }
1960 if (need_eval_reset == TRUE) {
1961 start_of_eval_period_sec = cur_ts_sec;
1962 start_of_eval_period_nsec = cur_ts_nsec;
1963 }
1964 }
1965
1966
1967 int compaction_swapper_init_now = 0;
1968 int compaction_swapper_running = 0;
1969 int compaction_swapper_awakened = 0;
1970 int compaction_swapper_abort = 0;
1971
1972
1973 #if CONFIG_JETSAM
1974 boolean_t memorystatus_kill_on_VM_compressor_thrashing(boolean_t);
1975 boolean_t memorystatus_kill_on_VM_compressor_space_shortage(boolean_t);
1976 boolean_t memorystatus_kill_on_FC_thrashing(boolean_t);
1977 int compressor_thrashing_induced_jetsam = 0;
1978 int filecache_thrashing_induced_jetsam = 0;
1979 static boolean_t vm_compressor_thrashing_detected = FALSE;
1980 #endif /* CONFIG_JETSAM */
1981
1982 static boolean_t
1983 compressor_needs_to_swap(void)
1984 {
1985 boolean_t should_swap = FALSE;
1986
1987 if (vm_swapout_ripe_segments == TRUE && c_overage_swapped_count < c_overage_swapped_limit) {
1988 c_segment_t c_seg;
1989 clock_sec_t now;
1990 clock_sec_t age;
1991 clock_nsec_t nsec;
1992
1993 clock_get_system_nanotime(&now, &nsec);
1994 age = 0;
1995
1996 lck_mtx_lock_spin_always(c_list_lock);
1997
1998 if (!queue_empty(&c_age_list_head)) {
1999 c_seg = (c_segment_t) queue_first(&c_age_list_head);
2000
2001 age = now - c_seg->c_creation_ts;
2002 }
2003 lck_mtx_unlock_always(c_list_lock);
2004
2005 if (age >= vm_ripe_target_age) {
2006 return TRUE;
2007 }
2008 }
2009 if (VM_CONFIG_SWAP_IS_ACTIVE) {
2010 if (COMPRESSOR_NEEDS_TO_SWAP()) {
2011 return TRUE;
2012 }
2013 if (VM_PAGE_Q_THROTTLED(&vm_pageout_queue_external) && vm_page_anonymous_count < (vm_page_inactive_count / 20)) {
2014 return TRUE;
2015 }
2016 if (vm_page_free_count < (vm_page_free_reserved - (COMPRESSOR_FREE_RESERVED_LIMIT * 2))) {
2017 return TRUE;
2018 }
2019 }
2020 compute_swapout_target_age();
2021
2022 if (swapout_target_age) {
2023 c_segment_t c_seg;
2024
2025 lck_mtx_lock_spin_always(c_list_lock);
2026
2027 if (!queue_empty(&c_age_list_head)) {
2028 c_seg = (c_segment_t) queue_first(&c_age_list_head);
2029
2030 if (c_seg->c_creation_ts > swapout_target_age) {
2031 swapout_target_age = 0;
2032 }
2033 }
2034 lck_mtx_unlock_always(c_list_lock);
2035 }
2036 #if CONFIG_PHANTOM_CACHE
2037 if (vm_phantom_cache_check_pressure()) {
2038 should_swap = TRUE;
2039 }
2040 #endif
2041 if (swapout_target_age) {
2042 should_swap = TRUE;
2043 }
2044
2045 #if CONFIG_JETSAM
2046 if (should_swap || vm_compressor_low_on_space() == TRUE) {
2047 if (vm_compressor_thrashing_detected == FALSE) {
2048 vm_compressor_thrashing_detected = TRUE;
2049
2050 if (swapout_target_age || vm_compressor_low_on_space() == TRUE) {
2051 if (swapout_target_age) {
2052 /* The compressor is thrashing. */
2053 memorystatus_kill_on_VM_compressor_thrashing(TRUE /* async */);
2054 } else {
2055 /* The compressor is running low on space. */
2056 memorystatus_kill_on_VM_compressor_space_shortage(TRUE /* async */);
2057 }
2058 compressor_thrashing_induced_jetsam++;
2059 } else {
2060 memorystatus_kill_on_FC_thrashing(TRUE /* async */);
2061 filecache_thrashing_induced_jetsam++;
2062 }
2063 }
2064 /*
2065 * let the jetsam take precedence over
2066 * any major compactions we might have
2067 * been able to do... otherwise we run
2068 * the risk of doing major compactions
2069 * on segments we're about to free up
2070 * due to the jetsam activity.
2071 */
2072 should_swap = FALSE;
2073 }
2074
2075 #endif /* CONFIG_JETSAM */
2076
2077 if (should_swap == FALSE) {
2078 /*
2079 * vm_compressor_needs_to_major_compact returns true only if we're
2080 * about to run out of available compressor segments... in this
2081 * case, we absolutely need to run a major compaction even if
2082 * we've just kicked off a jetsam or we don't otherwise need to
2083 * swap... terminating objects releases
2084 * pages back to the uncompressed cache, but does not guarantee
2085 * that we will free up even a single compression segment
2086 */
2087 should_swap = vm_compressor_needs_to_major_compact();
2088 }
2089
2090 /*
2091 * returning TRUE when swap_supported == FALSE
2092 * will cause the major compaction engine to
2093 * run, but will not trigger any swapping...
2094 * segments that have been major compacted
2095 * will be moved to the majorcompact queue
2096 */
2097 return should_swap;
2098 }
2099
2100 #if CONFIG_JETSAM
2101 /*
2102 * This function is called from the jetsam thread after killing something to
2103 * mitigate thrashing.
2104 *
2105 * We need to restart our thrashing detection heuristics since memory pressure
2106 * has potentially changed significantly, and we don't want to detect on old
2107 * data from before the jetsam.
2108 */
2109 void
2110 vm_thrashing_jetsam_done(void)
2111 {
2112 vm_compressor_thrashing_detected = FALSE;
2113
2114 /* Were we compressor-thrashing or filecache-thrashing? */
2115 if (swapout_target_age) {
2116 swapout_target_age = 0;
2117 compressor_need_sample_reset = TRUE;
2118 }
2119 #if CONFIG_PHANTOM_CACHE
2120 else {
2121 vm_phantom_cache_restart_sample();
2122 }
2123 #endif
2124 }
2125 #endif /* CONFIG_JETSAM */
2126
2127 uint32_t vm_wake_compactor_swapper_calls = 0;
2128 uint32_t vm_run_compactor_already_running = 0;
2129 uint32_t vm_run_compactor_empty_minor_q = 0;
2130 uint32_t vm_run_compactor_did_compact = 0;
2131 uint32_t vm_run_compactor_waited = 0;
2132
2133 void
2134 vm_run_compactor(void)
2135 {
2136 if (c_segment_count == 0) {
2137 return;
2138 }
2139
2140 lck_mtx_lock_spin_always(c_list_lock);
2141
2142 if (c_minor_count == 0) {
2143 vm_run_compactor_empty_minor_q++;
2144
2145 lck_mtx_unlock_always(c_list_lock);
2146 return;
2147 }
2148 if (compaction_swapper_running) {
2149 if (vm_pageout_state.vm_restricted_to_single_processor == FALSE) {
2150 vm_run_compactor_already_running++;
2151
2152 lck_mtx_unlock_always(c_list_lock);
2153 return;
2154 }
2155 vm_run_compactor_waited++;
2156
2157 assert_wait((event_t)&compaction_swapper_running, THREAD_UNINT);
2158
2159 lck_mtx_unlock_always(c_list_lock);
2160
2161 thread_block(THREAD_CONTINUE_NULL);
2162
2163 return;
2164 }
2165 vm_run_compactor_did_compact++;
2166
2167 fastwake_warmup = FALSE;
2168 compaction_swapper_running = 1;
2169
2170 vm_compressor_do_delayed_compactions(FALSE);
2171
2172 compaction_swapper_running = 0;
2173
2174 lck_mtx_unlock_always(c_list_lock);
2175
2176 thread_wakeup((event_t)&compaction_swapper_running);
2177 }
2178
2179
2180 void
2181 vm_wake_compactor_swapper(void)
2182 {
2183 if (compaction_swapper_running || compaction_swapper_awakened || c_segment_count == 0) {
2184 return;
2185 }
2186
2187 if (c_minor_count || vm_compressor_needs_to_major_compact()) {
2188 lck_mtx_lock_spin_always(c_list_lock);
2189
2190 fastwake_warmup = FALSE;
2191
2192 if (compaction_swapper_running == 0 && compaction_swapper_awakened == 0) {
2193 vm_wake_compactor_swapper_calls++;
2194
2195 compaction_swapper_awakened = 1;
2196 thread_wakeup((event_t)&c_compressor_swap_trigger);
2197 }
2198 lck_mtx_unlock_always(c_list_lock);
2199 }
2200 }
2201
2202
2203 void
2204 vm_consider_swapping()
2205 {
2206 c_segment_t c_seg, c_seg_next;
2207 clock_sec_t now;
2208 clock_nsec_t nsec;
2209
2210 assert(VM_CONFIG_SWAP_IS_PRESENT);
2211
2212 lck_mtx_lock_spin_always(c_list_lock);
2213
2214 compaction_swapper_abort = 1;
2215
2216 while (compaction_swapper_running) {
2217 assert_wait((event_t)&compaction_swapper_running, THREAD_UNINT);
2218
2219 lck_mtx_unlock_always(c_list_lock);
2220
2221 thread_block(THREAD_CONTINUE_NULL);
2222
2223 lck_mtx_lock_spin_always(c_list_lock);
2224 }
2225 compaction_swapper_abort = 0;
2226 compaction_swapper_running = 1;
2227
2228 vm_swapout_ripe_segments = TRUE;
2229
2230 if (!queue_empty(&c_major_list_head)) {
2231 clock_get_system_nanotime(&now, &nsec);
2232
2233 c_seg = (c_segment_t)queue_first(&c_major_list_head);
2234
2235 while (!queue_end(&c_major_list_head, (queue_entry_t)c_seg)) {
2236 if (c_overage_swapped_count >= c_overage_swapped_limit) {
2237 break;
2238 }
2239
2240 c_seg_next = (c_segment_t) queue_next(&c_seg->c_age_list);
2241
2242 if ((now - c_seg->c_creation_ts) >= vm_ripe_target_age) {
2243 lck_mtx_lock_spin_always(&c_seg->c_lock);
2244
2245 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
2246
2247 lck_mtx_unlock_always(&c_seg->c_lock);
2248 }
2249 c_seg = c_seg_next;
2250 }
2251 }
2252 vm_compressor_compact_and_swap(FALSE);
2253
2254 compaction_swapper_running = 0;
2255
2256 vm_swapout_ripe_segments = FALSE;
2257
2258 lck_mtx_unlock_always(c_list_lock);
2259
2260 thread_wakeup((event_t)&compaction_swapper_running);
2261 }
2262
2263
2264 void
2265 vm_consider_waking_compactor_swapper(void)
2266 {
2267 boolean_t need_wakeup = FALSE;
2268
2269 if (c_segment_count == 0) {
2270 return;
2271 }
2272
2273 if (compaction_swapper_running || compaction_swapper_awakened) {
2274 return;
2275 }
2276
2277 if (!compaction_swapper_inited && !compaction_swapper_init_now) {
2278 compaction_swapper_init_now = 1;
2279 need_wakeup = TRUE;
2280 }
2281
2282 if (c_minor_count && (COMPRESSOR_NEEDS_TO_MINOR_COMPACT())) {
2283 need_wakeup = TRUE;
2284 } else if (compressor_needs_to_swap()) {
2285 need_wakeup = TRUE;
2286 } else if (c_minor_count) {
2287 uint64_t total_bytes;
2288
2289 total_bytes = compressor_object->resident_page_count * PAGE_SIZE_64;
2290
2291 if ((total_bytes - compressor_bytes_used) > total_bytes / 10) {
2292 need_wakeup = TRUE;
2293 }
2294 }
2295 if (need_wakeup == TRUE) {
2296 lck_mtx_lock_spin_always(c_list_lock);
2297
2298 fastwake_warmup = FALSE;
2299
2300 if (compaction_swapper_running == 0 && compaction_swapper_awakened == 0) {
2301 memoryshot(VM_WAKEUP_COMPACTOR_SWAPPER, DBG_FUNC_NONE);
2302
2303 compaction_swapper_awakened = 1;
2304 thread_wakeup((event_t)&c_compressor_swap_trigger);
2305 }
2306 lck_mtx_unlock_always(c_list_lock);
2307 }
2308 }
2309
2310
2311 #define C_SWAPOUT_LIMIT 4
2312 #define DELAYED_COMPACTIONS_PER_PASS 30
2313
2314 void
2315 vm_compressor_do_delayed_compactions(boolean_t flush_all)
2316 {
2317 c_segment_t c_seg;
2318 int number_compacted = 0;
2319 boolean_t needs_to_swap = FALSE;
2320
2321
2322 #if !CONFIG_EMBEDDED
2323 LCK_MTX_ASSERT(c_list_lock, LCK_MTX_ASSERT_OWNED);
2324 #endif /* !CONFIG_EMBEDDED */
2325
2326 while (!queue_empty(&c_minor_list_head) && needs_to_swap == FALSE) {
2327 c_seg = (c_segment_t)queue_first(&c_minor_list_head);
2328
2329 lck_mtx_lock_spin_always(&c_seg->c_lock);
2330
2331 if (c_seg->c_busy) {
2332 lck_mtx_unlock_always(c_list_lock);
2333 c_seg_wait_on_busy(c_seg);
2334 lck_mtx_lock_spin_always(c_list_lock);
2335
2336 continue;
2337 }
2338 C_SEG_BUSY(c_seg);
2339
2340 c_seg_do_minor_compaction_and_unlock(c_seg, TRUE, FALSE, TRUE);
2341
2342 if (VM_CONFIG_SWAP_IS_ACTIVE && (number_compacted++ > DELAYED_COMPACTIONS_PER_PASS)) {
2343 if ((flush_all == TRUE || compressor_needs_to_swap() == TRUE) && c_swapout_count < C_SWAPOUT_LIMIT) {
2344 needs_to_swap = TRUE;
2345 }
2346
2347 number_compacted = 0;
2348 }
2349 lck_mtx_lock_spin_always(c_list_lock);
2350 }
2351 }
2352
2353
2354 #define C_SEGMENT_SWAPPEDIN_AGE_LIMIT 10
2355
2356 static void
2357 vm_compressor_age_swapped_in_segments(boolean_t flush_all)
2358 {
2359 c_segment_t c_seg;
2360 clock_sec_t now;
2361 clock_nsec_t nsec;
2362
2363 clock_get_system_nanotime(&now, &nsec);
2364
2365 while (!queue_empty(&c_swappedin_list_head)) {
2366 c_seg = (c_segment_t)queue_first(&c_swappedin_list_head);
2367
2368 if (flush_all == FALSE && (now - c_seg->c_swappedin_ts) < C_SEGMENT_SWAPPEDIN_AGE_LIMIT) {
2369 break;
2370 }
2371
2372 lck_mtx_lock_spin_always(&c_seg->c_lock);
2373
2374 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
2375
2376 lck_mtx_unlock_always(&c_seg->c_lock);
2377 }
2378 }
2379
2380
2381 extern int vm_num_swap_files;
2382 extern int vm_num_pinned_swap_files;
2383 extern int vm_swappin_enabled;
2384
2385 extern unsigned int vm_swapfile_total_segs_used;
2386 extern unsigned int vm_swapfile_total_segs_alloced;
2387
2388
2389 void
2390 vm_compressor_flush(void)
2391 {
2392 uint64_t vm_swap_put_failures_at_start;
2393 wait_result_t wait_result = 0;
2394 AbsoluteTime startTime, endTime;
2395 clock_sec_t now_sec;
2396 clock_nsec_t now_nsec;
2397 uint64_t nsec;
2398
2399 HIBLOG("vm_compressor_flush - starting\n");
2400
2401 clock_get_uptime(&startTime);
2402
2403 lck_mtx_lock_spin_always(c_list_lock);
2404
2405 fastwake_warmup = FALSE;
2406 compaction_swapper_abort = 1;
2407
2408 while (compaction_swapper_running) {
2409 assert_wait((event_t)&compaction_swapper_running, THREAD_UNINT);
2410
2411 lck_mtx_unlock_always(c_list_lock);
2412
2413 thread_block(THREAD_CONTINUE_NULL);
2414
2415 lck_mtx_lock_spin_always(c_list_lock);
2416 }
2417 compaction_swapper_abort = 0;
2418 compaction_swapper_running = 1;
2419
2420 hibernate_flushing = TRUE;
2421 hibernate_no_swapspace = FALSE;
2422 c_generation_id_flush_barrier = c_generation_id + 1000;
2423
2424 clock_get_system_nanotime(&now_sec, &now_nsec);
2425 hibernate_flushing_deadline = now_sec + HIBERNATE_FLUSHING_SECS_TO_COMPLETE;
2426
2427 vm_swap_put_failures_at_start = vm_swap_put_failures;
2428
2429 vm_compressor_compact_and_swap(TRUE);
2430
2431 while (!queue_empty(&c_swapout_list_head)) {
2432 assert_wait_timeout((event_t) &compaction_swapper_running, THREAD_INTERRUPTIBLE, 5000, 1000 * NSEC_PER_USEC);
2433
2434 lck_mtx_unlock_always(c_list_lock);
2435
2436 wait_result = thread_block(THREAD_CONTINUE_NULL);
2437
2438 lck_mtx_lock_spin_always(c_list_lock);
2439
2440 if (wait_result == THREAD_TIMED_OUT) {
2441 break;
2442 }
2443 }
2444 hibernate_flushing = FALSE;
2445 compaction_swapper_running = 0;
2446
2447 if (vm_swap_put_failures > vm_swap_put_failures_at_start) {
2448 HIBLOG("vm_compressor_flush failed to clean %llu segments - vm_page_compressor_count(%d)\n",
2449 vm_swap_put_failures - vm_swap_put_failures_at_start, VM_PAGE_COMPRESSOR_COUNT);
2450 }
2451
2452 lck_mtx_unlock_always(c_list_lock);
2453
2454 thread_wakeup((event_t)&compaction_swapper_running);
2455
2456 clock_get_uptime(&endTime);
2457 SUB_ABSOLUTETIME(&endTime, &startTime);
2458 absolutetime_to_nanoseconds(endTime, &nsec);
2459
2460 HIBLOG("vm_compressor_flush completed - took %qd msecs - vm_num_swap_files = %d, vm_num_pinned_swap_files = %d, vm_swappin_enabled = %d\n",
2461 nsec / 1000000ULL, vm_num_swap_files, vm_num_pinned_swap_files, vm_swappin_enabled);
2462 }
2463
2464
2465 int compaction_swap_trigger_thread_awakened = 0;
2466
2467 static void
2468 vm_compressor_swap_trigger_thread(void)
2469 {
2470 current_thread()->options |= TH_OPT_VMPRIV;
2471
2472 /*
2473 * compaction_swapper_init_now is set when the first call to
2474 * vm_consider_waking_compactor_swapper is made from
2475 * vm_pageout_scan... since this function is called upon
2476 * thread creation, we want to make sure to delay adjusting
2477 * the tuneables until we are awakened via vm_pageout_scan
2478 * so that we are at a point where the vm_swapfile_open will
2479 * be operating on the correct directory (in case the default
2480 * of /var/vm/ is overridden by the dymanic_pager
2481 */
2482 if (compaction_swapper_init_now) {
2483 vm_compaction_swapper_do_init();
2484
2485 if (vm_pageout_state.vm_restricted_to_single_processor == TRUE) {
2486 thread_vm_bind_group_add();
2487 }
2488 thread_set_thread_name(current_thread(), "VM_cswap_trigger");
2489 compaction_swapper_init_now = 0;
2490 }
2491 lck_mtx_lock_spin_always(c_list_lock);
2492
2493 compaction_swap_trigger_thread_awakened++;
2494 compaction_swapper_awakened = 0;
2495
2496 if (compaction_swapper_running == 0) {
2497 compaction_swapper_running = 1;
2498
2499 vm_compressor_compact_and_swap(FALSE);
2500
2501 compaction_swapper_running = 0;
2502 }
2503 assert_wait((event_t)&c_compressor_swap_trigger, THREAD_UNINT);
2504
2505 if (compaction_swapper_running == 0) {
2506 thread_wakeup((event_t)&compaction_swapper_running);
2507 }
2508
2509 lck_mtx_unlock_always(c_list_lock);
2510
2511 thread_block((thread_continue_t)vm_compressor_swap_trigger_thread);
2512
2513 /* NOTREACHED */
2514 }
2515
2516
2517 void
2518 vm_compressor_record_warmup_start(void)
2519 {
2520 c_segment_t c_seg;
2521
2522 lck_mtx_lock_spin_always(c_list_lock);
2523
2524 if (first_c_segment_to_warm_generation_id == 0) {
2525 if (!queue_empty(&c_age_list_head)) {
2526 c_seg = (c_segment_t)queue_last(&c_age_list_head);
2527
2528 first_c_segment_to_warm_generation_id = c_seg->c_generation_id;
2529 } else {
2530 first_c_segment_to_warm_generation_id = 0;
2531 }
2532
2533 fastwake_recording_in_progress = TRUE;
2534 }
2535 lck_mtx_unlock_always(c_list_lock);
2536 }
2537
2538
2539 void
2540 vm_compressor_record_warmup_end(void)
2541 {
2542 c_segment_t c_seg;
2543
2544 lck_mtx_lock_spin_always(c_list_lock);
2545
2546 if (fastwake_recording_in_progress == TRUE) {
2547 if (!queue_empty(&c_age_list_head)) {
2548 c_seg = (c_segment_t)queue_last(&c_age_list_head);
2549
2550 last_c_segment_to_warm_generation_id = c_seg->c_generation_id;
2551 } else {
2552 last_c_segment_to_warm_generation_id = first_c_segment_to_warm_generation_id;
2553 }
2554
2555 fastwake_recording_in_progress = FALSE;
2556
2557 HIBLOG("vm_compressor_record_warmup (%qd - %qd)\n", first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id);
2558 }
2559 lck_mtx_unlock_always(c_list_lock);
2560 }
2561
2562
2563 #define DELAY_TRIM_ON_WAKE_SECS 25
2564
2565 void
2566 vm_compressor_delay_trim(void)
2567 {
2568 clock_sec_t sec;
2569 clock_nsec_t nsec;
2570
2571 clock_get_system_nanotime(&sec, &nsec);
2572 dont_trim_until_ts = sec + DELAY_TRIM_ON_WAKE_SECS;
2573 }
2574
2575
2576 void
2577 vm_compressor_do_warmup(void)
2578 {
2579 lck_mtx_lock_spin_always(c_list_lock);
2580
2581 if (first_c_segment_to_warm_generation_id == last_c_segment_to_warm_generation_id) {
2582 first_c_segment_to_warm_generation_id = last_c_segment_to_warm_generation_id = 0;
2583
2584 lck_mtx_unlock_always(c_list_lock);
2585 return;
2586 }
2587
2588 if (compaction_swapper_running == 0 && compaction_swapper_awakened == 0) {
2589 fastwake_warmup = TRUE;
2590
2591 compaction_swapper_awakened = 1;
2592 thread_wakeup((event_t)&c_compressor_swap_trigger);
2593 }
2594 lck_mtx_unlock_always(c_list_lock);
2595 }
2596
2597 void
2598 do_fastwake_warmup_all(void)
2599 {
2600 lck_mtx_lock_spin_always(c_list_lock);
2601
2602 if (queue_empty(&c_swappedout_list_head) && queue_empty(&c_swappedout_sparse_list_head)) {
2603 lck_mtx_unlock_always(c_list_lock);
2604 return;
2605 }
2606
2607 fastwake_warmup = TRUE;
2608
2609 do_fastwake_warmup(&c_swappedout_list_head, TRUE);
2610
2611 do_fastwake_warmup(&c_swappedout_sparse_list_head, TRUE);
2612
2613 fastwake_warmup = FALSE;
2614
2615 lck_mtx_unlock_always(c_list_lock);
2616 }
2617
2618 void
2619 do_fastwake_warmup(queue_head_t *c_queue, boolean_t consider_all_cseg)
2620 {
2621 c_segment_t c_seg = NULL;
2622 AbsoluteTime startTime, endTime;
2623 uint64_t nsec;
2624
2625
2626 HIBLOG("vm_compressor_fastwake_warmup (%qd - %qd) - starting\n", first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id);
2627
2628 clock_get_uptime(&startTime);
2629
2630 lck_mtx_unlock_always(c_list_lock);
2631
2632 proc_set_thread_policy(current_thread(),
2633 TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER2);
2634
2635 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2636
2637 lck_mtx_lock_spin_always(c_list_lock);
2638
2639 while (!queue_empty(c_queue) && fastwake_warmup == TRUE) {
2640 c_seg = (c_segment_t) queue_first(c_queue);
2641
2642 if (consider_all_cseg == FALSE) {
2643 if (c_seg->c_generation_id < first_c_segment_to_warm_generation_id ||
2644 c_seg->c_generation_id > last_c_segment_to_warm_generation_id) {
2645 break;
2646 }
2647
2648 if (vm_page_free_count < (AVAILABLE_MEMORY / 4)) {
2649 break;
2650 }
2651 }
2652
2653 lck_mtx_lock_spin_always(&c_seg->c_lock);
2654 lck_mtx_unlock_always(c_list_lock);
2655
2656 if (c_seg->c_busy) {
2657 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2658 c_seg_wait_on_busy(c_seg);
2659 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2660 } else {
2661 if (c_seg_swapin(c_seg, TRUE, FALSE) == 0) {
2662 lck_mtx_unlock_always(&c_seg->c_lock);
2663 }
2664 c_segment_warmup_count++;
2665
2666 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2667 vm_pageout_io_throttle();
2668 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2669 }
2670 lck_mtx_lock_spin_always(c_list_lock);
2671 }
2672 lck_mtx_unlock_always(c_list_lock);
2673
2674 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2675
2676 proc_set_thread_policy(current_thread(),
2677 TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER0);
2678
2679 clock_get_uptime(&endTime);
2680 SUB_ABSOLUTETIME(&endTime, &startTime);
2681 absolutetime_to_nanoseconds(endTime, &nsec);
2682
2683 HIBLOG("vm_compressor_fastwake_warmup completed - took %qd msecs\n", nsec / 1000000ULL);
2684
2685 lck_mtx_lock_spin_always(c_list_lock);
2686
2687 if (consider_all_cseg == FALSE) {
2688 first_c_segment_to_warm_generation_id = last_c_segment_to_warm_generation_id = 0;
2689 }
2690 }
2691
2692
2693 void
2694 vm_compressor_compact_and_swap(boolean_t flush_all)
2695 {
2696 c_segment_t c_seg, c_seg_next;
2697 boolean_t keep_compacting;
2698 clock_sec_t now;
2699 clock_nsec_t nsec;
2700
2701
2702 if (fastwake_warmup == TRUE) {
2703 uint64_t starting_warmup_count;
2704
2705 starting_warmup_count = c_segment_warmup_count;
2706
2707 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 11) | DBG_FUNC_START, c_segment_warmup_count,
2708 first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id, 0, 0);
2709 do_fastwake_warmup(&c_swappedout_list_head, FALSE);
2710 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 11) | DBG_FUNC_END, c_segment_warmup_count, c_segment_warmup_count - starting_warmup_count, 0, 0, 0);
2711
2712 fastwake_warmup = FALSE;
2713 }
2714
2715 /*
2716 * it's possible for the c_age_list_head to be empty if we
2717 * hit our limits for growing the compressor pool and we subsequently
2718 * hibernated... on the next hibernation we could see the queue as
2719 * empty and not proceeed even though we have a bunch of segments on
2720 * the swapped in queue that need to be dealt with.
2721 */
2722 vm_compressor_do_delayed_compactions(flush_all);
2723
2724 vm_compressor_age_swapped_in_segments(flush_all);
2725
2726 /*
2727 * we only need to grab the timestamp once per
2728 * invocation of this function since the
2729 * timescale we're interested in is measured
2730 * in days
2731 */
2732 clock_get_system_nanotime(&now, &nsec);
2733
2734 while (!queue_empty(&c_age_list_head) && compaction_swapper_abort == 0) {
2735 if (hibernate_flushing == TRUE) {
2736 clock_sec_t sec;
2737
2738 if (hibernate_should_abort()) {
2739 HIBLOG("vm_compressor_flush - hibernate_should_abort returned TRUE\n");
2740 break;
2741 }
2742 if (hibernate_no_swapspace == TRUE) {
2743 HIBLOG("vm_compressor_flush - out of swap space\n");
2744 break;
2745 }
2746 if (vm_swap_files_pinned() == FALSE) {
2747 HIBLOG("vm_compressor_flush - unpinned swap files\n");
2748 break;
2749 }
2750 if (hibernate_in_progress_with_pinned_swap == TRUE &&
2751 (vm_swapfile_total_segs_alloced == vm_swapfile_total_segs_used)) {
2752 HIBLOG("vm_compressor_flush - out of pinned swap space\n");
2753 break;
2754 }
2755 clock_get_system_nanotime(&sec, &nsec);
2756
2757 if (sec > hibernate_flushing_deadline) {
2758 HIBLOG("vm_compressor_flush - failed to finish before deadline\n");
2759 break;
2760 }
2761 }
2762 if (c_swapout_count >= C_SWAPOUT_LIMIT) {
2763 assert_wait_timeout((event_t) &compaction_swapper_running, THREAD_INTERRUPTIBLE, 100, 1000 * NSEC_PER_USEC);
2764
2765 lck_mtx_unlock_always(c_list_lock);
2766
2767 thread_block(THREAD_CONTINUE_NULL);
2768
2769 lck_mtx_lock_spin_always(c_list_lock);
2770 }
2771 /*
2772 * Minor compactions
2773 */
2774 vm_compressor_do_delayed_compactions(flush_all);
2775
2776 vm_compressor_age_swapped_in_segments(flush_all);
2777
2778 if (c_swapout_count >= C_SWAPOUT_LIMIT) {
2779 /*
2780 * we timed out on the above thread_block
2781 * let's loop around and try again
2782 * the timeout allows us to continue
2783 * to do minor compactions to make
2784 * more memory available
2785 */
2786 continue;
2787 }
2788
2789 /*
2790 * Swap out segments?
2791 */
2792 if (flush_all == FALSE) {
2793 boolean_t needs_to_swap;
2794
2795 lck_mtx_unlock_always(c_list_lock);
2796
2797 needs_to_swap = compressor_needs_to_swap();
2798
2799 #if !CONFIG_EMBEDDED
2800 if (needs_to_swap == TRUE && vm_swap_low_on_space()) {
2801 vm_compressor_take_paging_space_action();
2802 }
2803 #endif /* !CONFIG_EMBEDDED */
2804
2805 lck_mtx_lock_spin_always(c_list_lock);
2806
2807 if (needs_to_swap == FALSE) {
2808 break;
2809 }
2810 }
2811 if (queue_empty(&c_age_list_head)) {
2812 break;
2813 }
2814 c_seg = (c_segment_t) queue_first(&c_age_list_head);
2815
2816 assert(c_seg->c_state == C_ON_AGE_Q);
2817
2818 if (flush_all == TRUE && c_seg->c_generation_id > c_generation_id_flush_barrier) {
2819 break;
2820 }
2821
2822 lck_mtx_lock_spin_always(&c_seg->c_lock);
2823
2824 if (c_seg->c_busy) {
2825 lck_mtx_unlock_always(c_list_lock);
2826 c_seg_wait_on_busy(c_seg);
2827 lck_mtx_lock_spin_always(c_list_lock);
2828
2829 continue;
2830 }
2831 C_SEG_BUSY(c_seg);
2832
2833 if (c_seg_do_minor_compaction_and_unlock(c_seg, FALSE, TRUE, TRUE)) {
2834 /*
2835 * found an empty c_segment and freed it
2836 * so go grab the next guy in the queue
2837 */
2838 c_seg_major_compact_stats.count_of_freed_segs++;
2839 continue;
2840 }
2841 /*
2842 * Major compaction
2843 */
2844 keep_compacting = TRUE;
2845
2846 while (keep_compacting == TRUE) {
2847 assert(c_seg->c_busy);
2848
2849 /* look for another segment to consolidate */
2850
2851 c_seg_next = (c_segment_t) queue_next(&c_seg->c_age_list);
2852
2853 if (queue_end(&c_age_list_head, (queue_entry_t)c_seg_next)) {
2854 break;
2855 }
2856
2857 assert(c_seg_next->c_state == C_ON_AGE_Q);
2858
2859 if (c_seg_major_compact_ok(c_seg, c_seg_next) == FALSE) {
2860 break;
2861 }
2862
2863 lck_mtx_lock_spin_always(&c_seg_next->c_lock);
2864
2865 if (c_seg_next->c_busy) {
2866 lck_mtx_unlock_always(c_list_lock);
2867 c_seg_wait_on_busy(c_seg_next);
2868 lck_mtx_lock_spin_always(c_list_lock);
2869
2870 continue;
2871 }
2872 /* grab that segment */
2873 C_SEG_BUSY(c_seg_next);
2874
2875 if (c_seg_do_minor_compaction_and_unlock(c_seg_next, FALSE, TRUE, TRUE)) {
2876 /*
2877 * found an empty c_segment and freed it
2878 * so we can't continue to use c_seg_next
2879 */
2880 c_seg_major_compact_stats.count_of_freed_segs++;
2881 continue;
2882 }
2883
2884 /* unlock the list ... */
2885 lck_mtx_unlock_always(c_list_lock);
2886
2887 /* do the major compaction */
2888
2889 keep_compacting = c_seg_major_compact(c_seg, c_seg_next);
2890
2891 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2892
2893 lck_mtx_lock_spin_always(&c_seg_next->c_lock);
2894 /*
2895 * run a minor compaction on the donor segment
2896 * since we pulled at least some of it's
2897 * data into our target... if we've emptied
2898 * it, now is a good time to free it which
2899 * c_seg_minor_compaction_and_unlock also takes care of
2900 *
2901 * by passing TRUE, we ask for c_busy to be cleared
2902 * and c_wanted to be taken care of
2903 */
2904 if (c_seg_minor_compaction_and_unlock(c_seg_next, TRUE)) {
2905 c_seg_major_compact_stats.count_of_freed_segs++;
2906 }
2907
2908 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2909
2910 /* relock the list */
2911 lck_mtx_lock_spin_always(c_list_lock);
2912 } /* major compaction */
2913
2914 lck_mtx_lock_spin_always(&c_seg->c_lock);
2915
2916 assert(c_seg->c_busy);
2917 assert(!c_seg->c_on_minorcompact_q);
2918
2919 if (VM_CONFIG_SWAP_IS_ACTIVE) {
2920 /*
2921 * This mode of putting a generic c_seg on the swapout list is
2922 * only supported when we have general swapping enabled
2923 */
2924 c_seg_switch_state(c_seg, C_ON_SWAPOUT_Q, FALSE);
2925 } else {
2926 if ((vm_swapout_ripe_segments == TRUE && c_overage_swapped_count < c_overage_swapped_limit)) {
2927 assert(VM_CONFIG_SWAP_IS_PRESENT);
2928 /*
2929 * we are running compressor sweeps with swap-behind
2930 * make sure the c_seg has aged enough before swapping it
2931 * out...
2932 */
2933 if ((now - c_seg->c_creation_ts) >= vm_ripe_target_age) {
2934 c_seg->c_overage_swap = TRUE;
2935 c_overage_swapped_count++;
2936 c_seg_switch_state(c_seg, C_ON_SWAPOUT_Q, FALSE);
2937 }
2938 }
2939 }
2940 if (c_seg->c_state == C_ON_AGE_Q) {
2941 /*
2942 * this c_seg didn't get moved to the swapout queue
2943 * so we need to move it out of the way...
2944 * we just did a major compaction on it so put it
2945 * on that queue
2946 */
2947 c_seg_switch_state(c_seg, C_ON_MAJORCOMPACT_Q, FALSE);
2948 } else {
2949 c_seg_major_compact_stats.wasted_space_in_swapouts += C_SEG_BUFSIZE - c_seg->c_bytes_used;
2950 c_seg_major_compact_stats.count_of_swapouts++;
2951 }
2952 C_SEG_WAKEUP_DONE(c_seg);
2953
2954 lck_mtx_unlock_always(&c_seg->c_lock);
2955
2956 if (c_swapout_count) {
2957 lck_mtx_unlock_always(c_list_lock);
2958
2959 thread_wakeup((event_t)&c_swapout_list_head);
2960
2961 lck_mtx_lock_spin_always(c_list_lock);
2962 }
2963 }
2964 }
2965
2966
2967 static c_segment_t
2968 c_seg_allocate(c_segment_t *current_chead)
2969 {
2970 c_segment_t c_seg;
2971 int min_needed;
2972 int size_to_populate;
2973
2974 #if !CONFIG_EMBEDDED
2975 if (vm_compressor_low_on_space()) {
2976 vm_compressor_take_paging_space_action();
2977 }
2978 #endif /* !CONFIG_EMBEDDED */
2979
2980 if ((c_seg = *current_chead) == NULL) {
2981 uint32_t c_segno;
2982
2983 lck_mtx_lock_spin_always(c_list_lock);
2984
2985 while (c_segments_busy == TRUE) {
2986 assert_wait((event_t) (&c_segments_busy), THREAD_UNINT);
2987
2988 lck_mtx_unlock_always(c_list_lock);
2989
2990 thread_block(THREAD_CONTINUE_NULL);
2991
2992 lck_mtx_lock_spin_always(c_list_lock);
2993 }
2994 if (c_free_segno_head == (uint32_t)-1) {
2995 uint32_t c_segments_available_new;
2996
2997 if (c_segments_available >= c_segments_limit || c_segment_pages_compressed >= c_segment_pages_compressed_limit) {
2998 lck_mtx_unlock_always(c_list_lock);
2999
3000 return NULL;
3001 }
3002 c_segments_busy = TRUE;
3003 lck_mtx_unlock_always(c_list_lock);
3004
3005 kernel_memory_populate(compressor_map, (vm_offset_t)c_segments_next_page,
3006 PAGE_SIZE, KMA_KOBJECT, VM_KERN_MEMORY_COMPRESSOR);
3007 c_segments_next_page += PAGE_SIZE;
3008
3009 c_segments_available_new = c_segments_available + C_SEGMENTS_PER_PAGE;
3010
3011 if (c_segments_available_new > c_segments_limit) {
3012 c_segments_available_new = c_segments_limit;
3013 }
3014
3015 for (c_segno = c_segments_available + 1; c_segno < c_segments_available_new; c_segno++) {
3016 c_segments[c_segno - 1].c_segno = c_segno;
3017 }
3018
3019 lck_mtx_lock_spin_always(c_list_lock);
3020
3021 c_segments[c_segno - 1].c_segno = c_free_segno_head;
3022 c_free_segno_head = c_segments_available;
3023 c_segments_available = c_segments_available_new;
3024
3025 c_segments_busy = FALSE;
3026 thread_wakeup((event_t) (&c_segments_busy));
3027 }
3028 c_segno = c_free_segno_head;
3029 assert(c_segno >= 0 && c_segno < c_segments_limit);
3030
3031 c_free_segno_head = (uint32_t)c_segments[c_segno].c_segno;
3032
3033 /*
3034 * do the rest of the bookkeeping now while we're still behind
3035 * the list lock and grab our generation id now into a local
3036 * so that we can install it once we have the c_seg allocated
3037 */
3038 c_segment_count++;
3039 if (c_segment_count > c_segment_count_max) {
3040 c_segment_count_max = c_segment_count;
3041 }
3042
3043 lck_mtx_unlock_always(c_list_lock);
3044
3045 c_seg = (c_segment_t)zalloc(compressor_segment_zone);
3046 bzero((char *)c_seg, sizeof(struct c_segment));
3047
3048 c_seg->c_store.c_buffer = (int32_t *)C_SEG_BUFFER_ADDRESS(c_segno);
3049
3050 lck_mtx_init(&c_seg->c_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
3051
3052 c_seg->c_state = C_IS_EMPTY;
3053 c_seg->c_firstemptyslot = C_SLOT_MAX_INDEX;
3054 c_seg->c_mysegno = c_segno;
3055
3056 lck_mtx_lock_spin_always(c_list_lock);
3057 c_empty_count++;
3058 c_seg_switch_state(c_seg, C_IS_FILLING, FALSE);
3059 c_segments[c_segno].c_seg = c_seg;
3060 assert(c_segments[c_segno].c_segno > c_segments_available);
3061 lck_mtx_unlock_always(c_list_lock);
3062
3063 *current_chead = c_seg;
3064
3065 #if DEVELOPMENT || DEBUG
3066 C_SEG_MAKE_WRITEABLE(c_seg);
3067 #endif
3068 }
3069 c_seg_alloc_nextslot(c_seg);
3070
3071 size_to_populate = C_SEG_ALLOCSIZE - C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset);
3072
3073 if (size_to_populate) {
3074 min_needed = PAGE_SIZE + (C_SEG_ALLOCSIZE - C_SEG_BUFSIZE);
3075
3076 if (C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset - c_seg->c_nextoffset) < (unsigned) min_needed) {
3077 if (size_to_populate > C_SEG_MAX_POPULATE_SIZE) {
3078 size_to_populate = C_SEG_MAX_POPULATE_SIZE;
3079 }
3080
3081 OSAddAtomic64(size_to_populate / PAGE_SIZE, &vm_pageout_vminfo.vm_compressor_pages_grabbed);
3082
3083 kernel_memory_populate(compressor_map,
3084 (vm_offset_t) &c_seg->c_store.c_buffer[c_seg->c_populated_offset],
3085 size_to_populate,
3086 KMA_COMPRESSOR,
3087 VM_KERN_MEMORY_COMPRESSOR);
3088 } else {
3089 size_to_populate = 0;
3090 }
3091 }
3092 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3093
3094 lck_mtx_lock_spin_always(&c_seg->c_lock);
3095
3096 if (size_to_populate) {
3097 c_seg->c_populated_offset += C_SEG_BYTES_TO_OFFSET(size_to_populate);
3098 }
3099
3100 return c_seg;
3101 }
3102
3103 #if DEVELOPMENT || DEBUG
3104 #if CONFIG_FREEZE
3105 extern boolean_t memorystatus_freeze_to_memory;
3106 #endif /* CONFIG_FREEZE */
3107 #endif /* DEVELOPMENT || DEBUG */
3108
3109 static void
3110 c_current_seg_filled(c_segment_t c_seg, c_segment_t *current_chead)
3111 {
3112 uint32_t unused_bytes;
3113 uint32_t offset_to_depopulate;
3114 int new_state = C_ON_AGE_Q;
3115 clock_sec_t sec;
3116 clock_nsec_t nsec;
3117 boolean_t head_insert = FALSE;
3118
3119 unused_bytes = trunc_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset - c_seg->c_nextoffset));
3120
3121 if (unused_bytes) {
3122 offset_to_depopulate = C_SEG_BYTES_TO_OFFSET(round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_nextoffset)));
3123
3124 /*
3125 * release the extra physical page(s) at the end of the segment
3126 */
3127 lck_mtx_unlock_always(&c_seg->c_lock);
3128
3129 kernel_memory_depopulate(
3130 compressor_map,
3131 (vm_offset_t) &c_seg->c_store.c_buffer[offset_to_depopulate],
3132 unused_bytes,
3133 KMA_COMPRESSOR);
3134
3135 lck_mtx_lock_spin_always(&c_seg->c_lock);
3136
3137 c_seg->c_populated_offset = offset_to_depopulate;
3138 }
3139 assert(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset) <= C_SEG_BUFSIZE);
3140
3141 #if DEVELOPMENT || DEBUG
3142 {
3143 boolean_t c_seg_was_busy = FALSE;
3144
3145 if (!c_seg->c_busy) {
3146 C_SEG_BUSY(c_seg);
3147 } else {
3148 c_seg_was_busy = TRUE;
3149 }
3150
3151 lck_mtx_unlock_always(&c_seg->c_lock);
3152
3153 C_SEG_WRITE_PROTECT(c_seg);
3154
3155 lck_mtx_lock_spin_always(&c_seg->c_lock);
3156
3157 if (c_seg_was_busy == FALSE) {
3158 C_SEG_WAKEUP_DONE(c_seg);
3159 }
3160 }
3161 #endif
3162
3163 #if CONFIG_FREEZE
3164 if (current_chead == (c_segment_t*)&freezer_chead &&
3165 VM_CONFIG_SWAP_IS_PRESENT &&
3166 VM_CONFIG_FREEZER_SWAP_IS_ACTIVE
3167 #if DEVELOPMENT || DEBUG
3168 && !memorystatus_freeze_to_memory
3169 #endif /* DEVELOPMENT || DEBUG */
3170 ) {
3171 new_state = C_ON_SWAPOUT_Q;
3172 }
3173 #endif /* CONFIG_FREEZE */
3174
3175 if (vm_darkwake_mode == TRUE) {
3176 new_state = C_ON_SWAPOUT_Q;
3177 head_insert = TRUE;
3178 }
3179
3180 clock_get_system_nanotime(&sec, &nsec);
3181 c_seg->c_creation_ts = (uint32_t)sec;
3182
3183 lck_mtx_lock_spin_always(c_list_lock);
3184
3185 c_seg->c_generation_id = c_generation_id++;
3186 c_seg_switch_state(c_seg, new_state, head_insert);
3187
3188 #if CONFIG_FREEZE
3189 if (c_seg->c_state == C_ON_SWAPOUT_Q) {
3190 /*
3191 * darkwake and freezer can't co-exist together
3192 * We'll need to fix this accounting as a start.
3193 */
3194 assert(vm_darkwake_mode == FALSE);
3195 c_freezer_swapout_page_count += (C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset)) / PAGE_SIZE_64;
3196 }
3197 #endif /* CONFIG_FREEZE */
3198
3199 if (c_seg->c_state == C_ON_AGE_Q && C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE) {
3200 c_seg_need_delayed_compaction(c_seg, TRUE);
3201 }
3202
3203 lck_mtx_unlock_always(c_list_lock);
3204
3205 if (c_seg->c_state == C_ON_SWAPOUT_Q) {
3206 thread_wakeup((event_t)&c_swapout_list_head);
3207 }
3208
3209 *current_chead = NULL;
3210 }
3211
3212
3213 /*
3214 * returns with c_seg locked
3215 */
3216 void
3217 c_seg_swapin_requeue(c_segment_t c_seg, boolean_t has_data, boolean_t minor_compact_ok, boolean_t age_on_swapin_q)
3218 {
3219 clock_sec_t sec;
3220 clock_nsec_t nsec;
3221
3222 clock_get_system_nanotime(&sec, &nsec);
3223
3224 lck_mtx_lock_spin_always(c_list_lock);
3225 lck_mtx_lock_spin_always(&c_seg->c_lock);
3226
3227 assert(c_seg->c_busy_swapping);
3228 assert(c_seg->c_busy);
3229
3230 c_seg->c_busy_swapping = 0;
3231
3232 if (c_seg->c_overage_swap == TRUE) {
3233 c_overage_swapped_count--;
3234 c_seg->c_overage_swap = FALSE;
3235 }
3236 if (has_data == TRUE) {
3237 if (age_on_swapin_q == TRUE) {
3238 c_seg_switch_state(c_seg, C_ON_SWAPPEDIN_Q, FALSE);
3239 } else {
3240 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
3241 }
3242
3243 if (minor_compact_ok == TRUE && !c_seg->c_on_minorcompact_q && C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE) {
3244 c_seg_need_delayed_compaction(c_seg, TRUE);
3245 }
3246 } else {
3247 c_seg->c_store.c_buffer = (int32_t*) NULL;
3248 c_seg->c_populated_offset = C_SEG_BYTES_TO_OFFSET(0);
3249
3250 c_seg_switch_state(c_seg, C_ON_BAD_Q, FALSE);
3251 }
3252 c_seg->c_swappedin_ts = (uint32_t)sec;
3253
3254 lck_mtx_unlock_always(c_list_lock);
3255 }
3256
3257
3258
3259 /*
3260 * c_seg has to be locked and is returned locked if the c_seg isn't freed
3261 * PAGE_REPLACMENT_DISALLOWED has to be TRUE on entry and is returned TRUE
3262 * c_seg_swapin returns 1 if the c_seg was freed, 0 otherwise
3263 */
3264
3265 int
3266 c_seg_swapin(c_segment_t c_seg, boolean_t force_minor_compaction, boolean_t age_on_swapin_q)
3267 {
3268 vm_offset_t addr = 0;
3269 uint32_t io_size = 0;
3270 uint64_t f_offset;
3271
3272 assert(C_SEG_IS_ONDISK(c_seg));
3273
3274 #if !CHECKSUM_THE_SWAP
3275 c_seg_trim_tail(c_seg);
3276 #endif
3277 io_size = round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset));
3278 f_offset = c_seg->c_store.c_swap_handle;
3279
3280 C_SEG_BUSY(c_seg);
3281 c_seg->c_busy_swapping = 1;
3282
3283 /*
3284 * This thread is likely going to block for I/O.
3285 * Make sure it is ready to run when the I/O completes because
3286 * it needs to clear the busy bit on the c_seg so that other
3287 * waiting threads can make progress too. To do that, boost
3288 * the rwlock_count so that the priority is boosted.
3289 */
3290 set_thread_rwlock_boost();
3291 lck_mtx_unlock_always(&c_seg->c_lock);
3292
3293 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3294
3295 addr = (vm_offset_t)C_SEG_BUFFER_ADDRESS(c_seg->c_mysegno);
3296 c_seg->c_store.c_buffer = (int32_t*) addr;
3297
3298 kernel_memory_populate(compressor_map, addr, io_size, KMA_COMPRESSOR, VM_KERN_MEMORY_COMPRESSOR);
3299
3300 if (vm_swap_get(c_seg, f_offset, io_size) != KERN_SUCCESS) {
3301 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3302
3303 kernel_memory_depopulate(compressor_map, addr, io_size, KMA_COMPRESSOR);
3304
3305 c_seg_swapin_requeue(c_seg, FALSE, TRUE, age_on_swapin_q);
3306 } else {
3307 #if ENCRYPTED_SWAP
3308 vm_swap_decrypt(c_seg);
3309 #endif /* ENCRYPTED_SWAP */
3310
3311 #if CHECKSUM_THE_SWAP
3312 if (c_seg->cseg_swap_size != io_size) {
3313 panic("swapin size doesn't match swapout size");
3314 }
3315
3316 if (c_seg->cseg_hash != vmc_hash((char*) c_seg->c_store.c_buffer, (int)io_size)) {
3317 panic("c_seg_swapin - Swap hash mismatch\n");
3318 }
3319 #endif /* CHECKSUM_THE_SWAP */
3320
3321 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3322
3323 c_seg_swapin_requeue(c_seg, TRUE, force_minor_compaction == TRUE ? FALSE : TRUE, age_on_swapin_q);
3324
3325 OSAddAtomic64(c_seg->c_bytes_used, &compressor_bytes_used);
3326
3327 if (force_minor_compaction == TRUE) {
3328 if (c_seg_minor_compaction_and_unlock(c_seg, FALSE)) {
3329 /*
3330 * c_seg was completely empty so it was freed,
3331 * so be careful not to reference it again
3332 *
3333 * Drop the rwlock_count so that the thread priority
3334 * is returned back to where it is supposed to be.
3335 */
3336 clear_thread_rwlock_boost();
3337 return 1;
3338 }
3339
3340 lck_mtx_lock_spin_always(&c_seg->c_lock);
3341 }
3342 }
3343 C_SEG_WAKEUP_DONE(c_seg);
3344
3345 /*
3346 * Drop the rwlock_count so that the thread priority
3347 * is returned back to where it is supposed to be.
3348 */
3349 clear_thread_rwlock_boost();
3350
3351 return 0;
3352 }
3353
3354
3355 static void
3356 c_segment_sv_hash_drop_ref(int hash_indx)
3357 {
3358 struct c_sv_hash_entry o_sv_he, n_sv_he;
3359
3360 while (1) {
3361 o_sv_he.he_record = c_segment_sv_hash_table[hash_indx].he_record;
3362
3363 n_sv_he.he_ref = o_sv_he.he_ref - 1;
3364 n_sv_he.he_data = o_sv_he.he_data;
3365
3366 if (OSCompareAndSwap64((UInt64)o_sv_he.he_record, (UInt64)n_sv_he.he_record, (UInt64 *) &c_segment_sv_hash_table[hash_indx].he_record) == TRUE) {
3367 if (n_sv_he.he_ref == 0) {
3368 OSAddAtomic(-1, &c_segment_svp_in_hash);
3369 }
3370 break;
3371 }
3372 }
3373 }
3374
3375
3376 static int
3377 c_segment_sv_hash_insert(uint32_t data)
3378 {
3379 int hash_sindx;
3380 int misses;
3381 struct c_sv_hash_entry o_sv_he, n_sv_he;
3382 boolean_t got_ref = FALSE;
3383
3384 if (data == 0) {
3385 OSAddAtomic(1, &c_segment_svp_zero_compressions);
3386 } else {
3387 OSAddAtomic(1, &c_segment_svp_nonzero_compressions);
3388 }
3389
3390 hash_sindx = data & C_SV_HASH_MASK;
3391
3392 for (misses = 0; misses < C_SV_HASH_MAX_MISS; misses++) {
3393 o_sv_he.he_record = c_segment_sv_hash_table[hash_sindx].he_record;
3394
3395 while (o_sv_he.he_data == data || o_sv_he.he_ref == 0) {
3396 n_sv_he.he_ref = o_sv_he.he_ref + 1;
3397 n_sv_he.he_data = data;
3398
3399 if (OSCompareAndSwap64((UInt64)o_sv_he.he_record, (UInt64)n_sv_he.he_record, (UInt64 *) &c_segment_sv_hash_table[hash_sindx].he_record) == TRUE) {
3400 if (n_sv_he.he_ref == 1) {
3401 OSAddAtomic(1, &c_segment_svp_in_hash);
3402 }
3403 got_ref = TRUE;
3404 break;
3405 }
3406 o_sv_he.he_record = c_segment_sv_hash_table[hash_sindx].he_record;
3407 }
3408 if (got_ref == TRUE) {
3409 break;
3410 }
3411 hash_sindx++;
3412
3413 if (hash_sindx == C_SV_HASH_SIZE) {
3414 hash_sindx = 0;
3415 }
3416 }
3417 if (got_ref == FALSE) {
3418 return -1;
3419 }
3420
3421 return hash_sindx;
3422 }
3423
3424
3425 #if RECORD_THE_COMPRESSED_DATA
3426
3427 static void
3428 c_compressed_record_data(char *src, int c_size)
3429 {
3430 if ((c_compressed_record_cptr + c_size + 4) >= c_compressed_record_ebuf) {
3431 panic("c_compressed_record_cptr >= c_compressed_record_ebuf");
3432 }
3433
3434 *(int *)((void *)c_compressed_record_cptr) = c_size;
3435
3436 c_compressed_record_cptr += 4;
3437
3438 memcpy(c_compressed_record_cptr, src, c_size);
3439 c_compressed_record_cptr += c_size;
3440 }
3441 #endif
3442
3443
3444 static int
3445 c_compress_page(char *src, c_slot_mapping_t slot_ptr, c_segment_t *current_chead, char *scratch_buf)
3446 {
3447 int c_size;
3448 int c_rounded_size = 0;
3449 int max_csize;
3450 c_slot_t cs;
3451 c_segment_t c_seg;
3452
3453 KERNEL_DEBUG(0xe0400000 | DBG_FUNC_START, *current_chead, 0, 0, 0, 0);
3454 retry:
3455 if ((c_seg = c_seg_allocate(current_chead)) == NULL) {
3456 return 1;
3457 }
3458 /*
3459 * returns with c_seg lock held
3460 * and PAGE_REPLACEMENT_DISALLOWED(TRUE)...
3461 * c_nextslot has been allocated and
3462 * c_store.c_buffer populated
3463 */
3464 assert(c_seg->c_state == C_IS_FILLING);
3465
3466 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_seg->c_nextslot);
3467
3468 cs->c_packed_ptr = C_SLOT_PACK_PTR(slot_ptr);
3469 assert(slot_ptr == (c_slot_mapping_t)C_SLOT_UNPACK_PTR(cs));
3470
3471 cs->c_offset = c_seg->c_nextoffset;
3472
3473 max_csize = C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES((int32_t)cs->c_offset);
3474
3475 if (max_csize > PAGE_SIZE) {
3476 max_csize = PAGE_SIZE;
3477 }
3478
3479 #if CHECKSUM_THE_DATA
3480 cs->c_hash_data = vmc_hash(src, PAGE_SIZE);
3481 #endif
3482 boolean_t incomp_copy = FALSE;
3483 int max_csize_adj = (max_csize - 4);
3484
3485 if (vm_compressor_algorithm() != VM_COMPRESSOR_DEFAULT_CODEC) {
3486 #if defined(__arm__) || defined(__arm64__)
3487 uint16_t ccodec = CINVALID;
3488
3489 if (max_csize >= C_SEG_OFFSET_ALIGNMENT_BOUNDARY) {
3490 c_size = metacompressor((const uint8_t *) src,
3491 (uint8_t *) &c_seg->c_store.c_buffer[cs->c_offset],
3492 max_csize_adj, &ccodec,
3493 scratch_buf, &incomp_copy);
3494 #if C_SEG_OFFSET_ALIGNMENT_BOUNDARY > 4
3495 if (c_size > max_csize_adj) {
3496 c_size = -1;
3497 }
3498 #endif
3499 } else {
3500 c_size = -1;
3501 }
3502 assert(ccodec == CCWK || ccodec == CCLZ4);
3503 cs->c_codec = ccodec;
3504 #endif
3505 } else {
3506 #if defined(__arm__) || defined(__arm64__)
3507 cs->c_codec = CCWK;
3508 #endif
3509 #if defined(__arm64__)
3510 __unreachable_ok_push
3511 if (PAGE_SIZE == 4096) {
3512 c_size = WKdm_compress_4k((WK_word *)(uintptr_t)src, (WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
3513 (WK_word *)(uintptr_t)scratch_buf, max_csize_adj);
3514 } else {
3515 c_size = WKdm_compress_16k((WK_word *)(uintptr_t)src, (WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
3516 (WK_word *)(uintptr_t)scratch_buf, max_csize_adj);
3517 }
3518 __unreachable_ok_pop
3519 #else
3520 c_size = WKdm_compress_new((const WK_word *)(uintptr_t)src, (WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
3521 (WK_word *)(uintptr_t)scratch_buf, max_csize_adj);
3522 #endif
3523 }
3524 assertf(((c_size <= max_csize_adj) && (c_size >= -1)),
3525 "c_size invalid (%d, %d), cur compressions: %d", c_size, max_csize_adj, c_segment_pages_compressed);
3526
3527 if (c_size == -1) {
3528 if (max_csize < PAGE_SIZE) {
3529 c_current_seg_filled(c_seg, current_chead);
3530 assert(*current_chead == NULL);
3531
3532 lck_mtx_unlock_always(&c_seg->c_lock);
3533 /* TODO: it may be worth requiring codecs to distinguish
3534 * between incompressible inputs and failures due to
3535 * budget exhaustion.
3536 */
3537 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3538 goto retry;
3539 }
3540 c_size = PAGE_SIZE;
3541
3542 if (incomp_copy == FALSE) {
3543 memcpy(&c_seg->c_store.c_buffer[cs->c_offset], src, c_size);
3544 }
3545
3546 OSAddAtomic(1, &c_segment_noncompressible_pages);
3547 } else if (c_size == 0) {
3548 int hash_index;
3549
3550 /*
3551 * special case - this is a page completely full of a single 32 bit value
3552 */
3553 hash_index = c_segment_sv_hash_insert(*(uint32_t *)(uintptr_t)src);
3554
3555 if (hash_index != -1) {
3556 slot_ptr->s_cindx = hash_index;
3557 slot_ptr->s_cseg = C_SV_CSEG_ID;
3558
3559 OSAddAtomic(1, &c_segment_svp_hash_succeeded);
3560 #if RECORD_THE_COMPRESSED_DATA
3561 c_compressed_record_data(src, 4);
3562 #endif
3563 goto sv_compression;
3564 }
3565 c_size = 4;
3566
3567 memcpy(&c_seg->c_store.c_buffer[cs->c_offset], src, c_size);
3568
3569 OSAddAtomic(1, &c_segment_svp_hash_failed);
3570 }
3571
3572 #if RECORD_THE_COMPRESSED_DATA
3573 c_compressed_record_data((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size);
3574 #endif
3575 #if CHECKSUM_THE_COMPRESSED_DATA
3576 cs->c_hash_compressed_data = vmc_hash((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size);
3577 #endif
3578 #if POPCOUNT_THE_COMPRESSED_DATA
3579 cs->c_pop_cdata = vmc_pop((uintptr_t) &c_seg->c_store.c_buffer[cs->c_offset], c_size);
3580 #endif
3581 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
3582
3583 PACK_C_SIZE(cs, c_size);
3584 c_seg->c_bytes_used += c_rounded_size;
3585 c_seg->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
3586 c_seg->c_slots_used++;
3587
3588 slot_ptr->s_cindx = c_seg->c_nextslot++;
3589 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
3590 slot_ptr->s_cseg = c_seg->c_mysegno + 1;
3591
3592 sv_compression:
3593 if (c_seg->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg->c_nextslot >= C_SLOT_MAX_INDEX) {
3594 c_current_seg_filled(c_seg, current_chead);
3595 assert(*current_chead == NULL);
3596 }
3597 lck_mtx_unlock_always(&c_seg->c_lock);
3598
3599 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3600
3601 #if RECORD_THE_COMPRESSED_DATA
3602 if ((c_compressed_record_cptr - c_compressed_record_sbuf) >= C_SEG_ALLOCSIZE) {
3603 c_compressed_record_write(c_compressed_record_sbuf, (int)(c_compressed_record_cptr - c_compressed_record_sbuf));
3604 c_compressed_record_cptr = c_compressed_record_sbuf;
3605 }
3606 #endif
3607 if (c_size) {
3608 OSAddAtomic64(c_size, &c_segment_compressed_bytes);
3609 OSAddAtomic64(c_rounded_size, &compressor_bytes_used);
3610 }
3611 OSAddAtomic64(PAGE_SIZE, &c_segment_input_bytes);
3612
3613 OSAddAtomic(1, &c_segment_pages_compressed);
3614 OSAddAtomic(1, &sample_period_compression_count);
3615
3616 KERNEL_DEBUG(0xe0400000 | DBG_FUNC_END, *current_chead, c_size, c_segment_input_bytes, c_segment_compressed_bytes, 0);
3617
3618 return 0;
3619 }
3620
3621 static inline void
3622 sv_decompress(int32_t *ddst, int32_t pattern)
3623 {
3624 // assert(__builtin_constant_p(PAGE_SIZE) != 0);
3625 #if defined(__x86_64__)
3626 memset_word(ddst, pattern, PAGE_SIZE / sizeof(int32_t));
3627 #elif defined(__arm64__)
3628 assert((PAGE_SIZE % 128) == 0);
3629 if (pattern == 0) {
3630 fill32_dczva((addr64_t)ddst, PAGE_SIZE);
3631 } else {
3632 fill32_nt((addr64_t)ddst, PAGE_SIZE, pattern);
3633 }
3634 #else
3635 size_t i;
3636
3637 /* Unroll the pattern fill loop 4x to encourage the
3638 * compiler to emit NEON stores, cf.
3639 * <rdar://problem/25839866> Loop autovectorization
3640 * anomalies.
3641 */
3642 /* * We use separate loops for each PAGE_SIZE
3643 * to allow the autovectorizer to engage, as PAGE_SIZE
3644 * may not be a constant.
3645 */
3646
3647 __unreachable_ok_push
3648 if (PAGE_SIZE == 4096) {
3649 for (i = 0; i < (4096U / sizeof(int32_t)); i += 4) {
3650 *ddst++ = pattern;
3651 *ddst++ = pattern;
3652 *ddst++ = pattern;
3653 *ddst++ = pattern;
3654 }
3655 } else {
3656 assert(PAGE_SIZE == 16384);
3657 for (i = 0; i < (int)(16384U / sizeof(int32_t)); i += 4) {
3658 *ddst++ = pattern;
3659 *ddst++ = pattern;
3660 *ddst++ = pattern;
3661 *ddst++ = pattern;
3662 }
3663 }
3664 __unreachable_ok_pop
3665 #endif
3666 }
3667
3668 static int
3669 c_decompress_page(char *dst, volatile c_slot_mapping_t slot_ptr, int flags, int *zeroslot)
3670 {
3671 c_slot_t cs;
3672 c_segment_t c_seg;
3673 uint32_t c_segno;
3674 int c_indx;
3675 int c_rounded_size;
3676 uint32_t c_size;
3677 int retval = 0;
3678 boolean_t need_unlock = TRUE;
3679 boolean_t consider_defragmenting = FALSE;
3680 boolean_t kdp_mode = FALSE;
3681
3682 if (__improbable(flags & C_KDP)) {
3683 if (not_in_kdp) {
3684 panic("C_KDP passed to decompress page from outside of debugger context");
3685 }
3686
3687 assert((flags & C_KEEP) == C_KEEP);
3688 assert((flags & C_DONT_BLOCK) == C_DONT_BLOCK);
3689
3690 if ((flags & (C_DONT_BLOCK | C_KEEP)) != (C_DONT_BLOCK | C_KEEP)) {
3691 return -2;
3692 }
3693
3694 kdp_mode = TRUE;
3695 *zeroslot = 0;
3696 }
3697
3698 ReTry:
3699 if (__probable(!kdp_mode)) {
3700 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3701 } else {
3702 if (kdp_lck_rw_lock_is_acquired_exclusive(&c_master_lock)) {
3703 return -2;
3704 }
3705 }
3706
3707 #if HIBERNATION
3708 /*
3709 * if hibernation is enabled, it indicates (via a call
3710 * to 'vm_decompressor_lock' that no further
3711 * decompressions are allowed once it reaches
3712 * the point of flushing all of the currently dirty
3713 * anonymous memory through the compressor and out
3714 * to disk... in this state we allow freeing of compressed
3715 * pages and must honor the C_DONT_BLOCK case
3716 */
3717 if (__improbable(dst && decompressions_blocked == TRUE)) {
3718 if (flags & C_DONT_BLOCK) {
3719 if (__probable(!kdp_mode)) {
3720 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3721 }
3722
3723 *zeroslot = 0;
3724 return -2;
3725 }
3726 /*
3727 * it's safe to atomically assert and block behind the
3728 * lock held in shared mode because "decompressions_blocked" is
3729 * only set and cleared and the thread_wakeup done when the lock
3730 * is held exclusively
3731 */
3732 assert_wait((event_t)&decompressions_blocked, THREAD_UNINT);
3733
3734 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3735
3736 thread_block(THREAD_CONTINUE_NULL);
3737
3738 goto ReTry;
3739 }
3740 #endif
3741 /* s_cseg is actually "segno+1" */
3742 c_segno = slot_ptr->s_cseg - 1;
3743
3744 if (__improbable(c_segno >= c_segments_available)) {
3745 panic("c_decompress_page: c_segno %d >= c_segments_available %d, slot_ptr(%p), slot_data(%x)",
3746 c_segno, c_segments_available, slot_ptr, *(int *)((void *)slot_ptr));
3747 }
3748
3749 if (__improbable(c_segments[c_segno].c_segno < c_segments_available)) {
3750 panic("c_decompress_page: c_segno %d is free, slot_ptr(%p), slot_data(%x)",
3751 c_segno, slot_ptr, *(int *)((void *)slot_ptr));
3752 }
3753
3754 c_seg = c_segments[c_segno].c_seg;
3755
3756 if (__probable(!kdp_mode)) {
3757 lck_mtx_lock_spin_always(&c_seg->c_lock);
3758 } else {
3759 if (kdp_lck_mtx_lock_spin_is_acquired(&c_seg->c_lock)) {
3760 return -2;
3761 }
3762 }
3763
3764 assert(c_seg->c_state != C_IS_EMPTY && c_seg->c_state != C_IS_FREE);
3765
3766 if (dst == NULL && c_seg->c_busy_swapping) {
3767 assert(c_seg->c_busy);
3768
3769 goto bypass_busy_check;
3770 }
3771 if (flags & C_DONT_BLOCK) {
3772 if (c_seg->c_busy || (C_SEG_IS_ONDISK(c_seg) && dst)) {
3773 *zeroslot = 0;
3774
3775 retval = -2;
3776 goto done;
3777 }
3778 }
3779 if (c_seg->c_busy) {
3780 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3781
3782 c_seg_wait_on_busy(c_seg);
3783
3784 goto ReTry;
3785 }
3786 bypass_busy_check:
3787
3788 c_indx = slot_ptr->s_cindx;
3789
3790 if (__improbable(c_indx >= c_seg->c_nextslot)) {
3791 panic("c_decompress_page: c_indx %d >= c_nextslot %d, c_seg(%p), slot_ptr(%p), slot_data(%x)",
3792 c_indx, c_seg->c_nextslot, c_seg, slot_ptr, *(int *)((void *)slot_ptr));
3793 }
3794
3795 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
3796
3797 c_size = UNPACK_C_SIZE(cs);
3798
3799 if (__improbable(c_size == 0)) {
3800 panic("c_decompress_page: c_size == 0, c_seg(%p), slot_ptr(%p), slot_data(%x)",
3801 c_seg, slot_ptr, *(int *)((void *)slot_ptr));
3802 }
3803
3804 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
3805
3806 if (dst) {
3807 uint32_t age_of_cseg;
3808 clock_sec_t cur_ts_sec;
3809 clock_nsec_t cur_ts_nsec;
3810
3811 if (C_SEG_IS_ONDISK(c_seg)) {
3812 assert(kdp_mode == FALSE);
3813 retval = c_seg_swapin(c_seg, FALSE, TRUE);
3814 assert(retval == 0);
3815
3816 retval = 1;
3817 }
3818 if (c_seg->c_state == C_ON_BAD_Q) {
3819 assert(c_seg->c_store.c_buffer == NULL);
3820 *zeroslot = 0;
3821
3822 retval = -1;
3823 goto done;
3824 }
3825
3826 #if POPCOUNT_THE_COMPRESSED_DATA
3827 unsigned csvpop;
3828 uintptr_t csvaddr = (uintptr_t) &c_seg->c_store.c_buffer[cs->c_offset];
3829 if (cs->c_pop_cdata != (csvpop = vmc_pop(csvaddr, c_size))) {
3830 panic("Compressed data popcount doesn't match original, bit distance: %d %p (phys: %p) %p %p 0x%x 0x%x 0x%x 0x%x", (csvpop - cs->c_pop_cdata), (void *)csvaddr, (void *) kvtophys(csvaddr), c_seg, cs, cs->c_offset, c_size, csvpop, cs->c_pop_cdata);
3831 }
3832 #endif
3833
3834 #if CHECKSUM_THE_COMPRESSED_DATA
3835 unsigned csvhash;
3836 if (cs->c_hash_compressed_data != (csvhash = vmc_hash((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size))) {
3837 panic("Compressed data doesn't match original %p %p %u %u %u", c_seg, cs, c_size, cs->c_hash_compressed_data, csvhash);
3838 }
3839 #endif
3840 if (c_rounded_size == PAGE_SIZE) {
3841 /*
3842 * page wasn't compressible... just copy it out
3843 */
3844 memcpy(dst, &c_seg->c_store.c_buffer[cs->c_offset], PAGE_SIZE);
3845 } else if (c_size == 4) {
3846 int32_t data;
3847 int32_t *dptr;
3848
3849 /*
3850 * page was populated with a single value
3851 * that didn't fit into our fast hash
3852 * so we packed it in as a single non-compressed value
3853 * that we need to populate the page with
3854 */
3855 dptr = (int32_t *)(uintptr_t)dst;
3856 data = *(int32_t *)(&c_seg->c_store.c_buffer[cs->c_offset]);
3857 sv_decompress(dptr, data);
3858 } else {
3859 uint32_t my_cpu_no;
3860 char *scratch_buf;
3861
3862 if (__probable(!kdp_mode)) {
3863 /*
3864 * we're behind the c_seg lock held in spin mode
3865 * which means pre-emption is disabled... therefore
3866 * the following sequence is atomic and safe
3867 */
3868 my_cpu_no = cpu_number();
3869
3870 assert(my_cpu_no < compressor_cpus);
3871
3872 scratch_buf = &compressor_scratch_bufs[my_cpu_no * vm_compressor_get_decode_scratch_size()];
3873 } else {
3874 scratch_buf = kdp_compressor_scratch_buf;
3875 }
3876
3877 if (vm_compressor_algorithm() != VM_COMPRESSOR_DEFAULT_CODEC) {
3878 #if defined(__arm__) || defined(__arm64__)
3879 uint16_t c_codec = cs->c_codec;
3880 metadecompressor((const uint8_t *) &c_seg->c_store.c_buffer[cs->c_offset],
3881 (uint8_t *)dst, c_size, c_codec, (void *)scratch_buf);
3882 #endif
3883 } else {
3884 #if defined(__arm64__)
3885 __unreachable_ok_push
3886 if (PAGE_SIZE == 4096) {
3887 WKdm_decompress_4k((WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
3888 (WK_word *)(uintptr_t)dst, (WK_word *)(uintptr_t)scratch_buf, c_size);
3889 } else {
3890 WKdm_decompress_16k((WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
3891 (WK_word *)(uintptr_t)dst, (WK_word *)(uintptr_t)scratch_buf, c_size);
3892 }
3893 __unreachable_ok_pop
3894 #else
3895 WKdm_decompress_new((WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
3896 (WK_word *)(uintptr_t)dst, (WK_word *)(uintptr_t)scratch_buf, c_size);
3897 #endif
3898 }
3899 }
3900
3901 #if CHECKSUM_THE_DATA
3902 if (cs->c_hash_data != vmc_hash(dst, PAGE_SIZE)) {
3903 #if defined(__arm__) || defined(__arm64__)
3904 int32_t *dinput = &c_seg->c_store.c_buffer[cs->c_offset];
3905 panic("decompressed data doesn't match original cs: %p, hash: 0x%x, offset: %d, c_size: %d, c_rounded_size: %d, codec: %d, header: 0x%x 0x%x 0x%x", cs, cs->c_hash_data, cs->c_offset, c_size, c_rounded_size, cs->c_codec, *dinput, *(dinput + 1), *(dinput + 2));
3906 #else
3907 panic("decompressed data doesn't match original cs: %p, hash: %d, offset: 0x%x, c_size: %d", cs, cs->c_hash_data, cs->c_offset, c_size);
3908 #endif
3909 }
3910 #endif
3911 if (c_seg->c_swappedin_ts == 0 && !kdp_mode) {
3912 clock_get_system_nanotime(&cur_ts_sec, &cur_ts_nsec);
3913
3914 age_of_cseg = (uint32_t)cur_ts_sec - c_seg->c_creation_ts;
3915 if (age_of_cseg < DECOMPRESSION_SAMPLE_MAX_AGE) {
3916 OSAddAtomic(1, &age_of_decompressions_during_sample_period[age_of_cseg]);
3917 } else {
3918 OSAddAtomic(1, &overage_decompressions_during_sample_period);
3919 }
3920
3921 OSAddAtomic(1, &sample_period_decompression_count);
3922 }
3923 }
3924 if (flags & C_KEEP) {
3925 *zeroslot = 0;
3926 goto done;
3927 }
3928 assert(kdp_mode == FALSE);
3929
3930 c_seg->c_bytes_unused += c_rounded_size;
3931 c_seg->c_bytes_used -= c_rounded_size;
3932
3933 assert(c_seg->c_slots_used);
3934 c_seg->c_slots_used--;
3935
3936 PACK_C_SIZE(cs, 0);
3937
3938 if (c_indx < c_seg->c_firstemptyslot) {
3939 c_seg->c_firstemptyslot = c_indx;
3940 }
3941
3942 OSAddAtomic(-1, &c_segment_pages_compressed);
3943
3944 if (c_seg->c_state != C_ON_BAD_Q && !(C_SEG_IS_ONDISK(c_seg))) {
3945 /*
3946 * C_SEG_IS_ONDISK == TRUE can occur when we're doing a
3947 * free of a compressed page (i.e. dst == NULL)
3948 */
3949 OSAddAtomic64(-c_rounded_size, &compressor_bytes_used);
3950 }
3951 if (c_seg->c_busy_swapping) {
3952 /*
3953 * bypass case for c_busy_swapping...
3954 * let the swapin/swapout paths deal with putting
3955 * the c_seg on the minor compaction queue if needed
3956 */
3957 assert(c_seg->c_busy);
3958 goto done;
3959 }
3960 assert(!c_seg->c_busy);
3961
3962 if (c_seg->c_state != C_IS_FILLING) {
3963 if (c_seg->c_bytes_used == 0) {
3964 if (!(C_SEG_IS_ONDISK(c_seg))) {
3965 int pages_populated;
3966
3967 pages_populated = (round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE;
3968 c_seg->c_populated_offset = C_SEG_BYTES_TO_OFFSET(0);
3969
3970 if (pages_populated) {
3971 assert(c_seg->c_state != C_ON_BAD_Q);
3972 assert(c_seg->c_store.c_buffer != NULL);
3973
3974 C_SEG_BUSY(c_seg);
3975 lck_mtx_unlock_always(&c_seg->c_lock);
3976
3977 kernel_memory_depopulate(compressor_map, (vm_offset_t) c_seg->c_store.c_buffer, pages_populated * PAGE_SIZE, KMA_COMPRESSOR);
3978
3979 lck_mtx_lock_spin_always(&c_seg->c_lock);
3980 C_SEG_WAKEUP_DONE(c_seg);
3981 }
3982 if (!c_seg->c_on_minorcompact_q && c_seg->c_state != C_ON_SWAPOUT_Q && c_seg->c_state != C_ON_SWAPIO_Q) {
3983 c_seg_need_delayed_compaction(c_seg, FALSE);
3984 }
3985 } else {
3986 if (c_seg->c_state != C_ON_SWAPPEDOUTSPARSE_Q) {
3987 c_seg_move_to_sparse_list(c_seg);
3988 consider_defragmenting = TRUE;
3989 }
3990 }
3991 } else if (c_seg->c_on_minorcompact_q) {
3992 assert(c_seg->c_state != C_ON_BAD_Q);
3993 assert(!C_SEG_IS_ON_DISK_OR_SOQ(c_seg));
3994
3995 if (C_SEG_SHOULD_MINORCOMPACT_NOW(c_seg)) {
3996 c_seg_try_minor_compaction_and_unlock(c_seg);
3997 need_unlock = FALSE;
3998 }
3999 } else if (!(C_SEG_IS_ONDISK(c_seg))) {
4000 if (c_seg->c_state != C_ON_BAD_Q && c_seg->c_state != C_ON_SWAPOUT_Q && c_seg->c_state != C_ON_SWAPIO_Q &&
4001 C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE) {
4002 c_seg_need_delayed_compaction(c_seg, FALSE);
4003 }
4004 } else if (c_seg->c_state != C_ON_SWAPPEDOUTSPARSE_Q && C_SEG_ONDISK_IS_SPARSE(c_seg)) {
4005 c_seg_move_to_sparse_list(c_seg);
4006 consider_defragmenting = TRUE;
4007 }
4008 }
4009 done:
4010 if (__improbable(kdp_mode)) {
4011 return retval;
4012 }
4013
4014 if (need_unlock == TRUE) {
4015 lck_mtx_unlock_always(&c_seg->c_lock);
4016 }
4017
4018 PAGE_REPLACEMENT_DISALLOWED(FALSE);
4019
4020 if (consider_defragmenting == TRUE) {
4021 vm_swap_consider_defragmenting(VM_SWAP_FLAGS_NONE);
4022 }
4023
4024 #if CONFIG_EMBEDDED
4025 if ((c_minor_count && COMPRESSOR_NEEDS_TO_MINOR_COMPACT()) || vm_compressor_needs_to_major_compact()) {
4026 vm_wake_compactor_swapper();
4027 }
4028 #endif
4029
4030 return retval;
4031 }
4032
4033
4034 int
4035 vm_compressor_get(ppnum_t pn, int *slot, int flags)
4036 {
4037 c_slot_mapping_t slot_ptr;
4038 char *dst;
4039 int zeroslot = 1;
4040 int retval;
4041
4042 dst = pmap_map_compressor_page(pn);
4043 slot_ptr = (c_slot_mapping_t)slot;
4044
4045 assert(dst != NULL);
4046
4047 if (slot_ptr->s_cseg == C_SV_CSEG_ID) {
4048 int32_t data;
4049 int32_t *dptr;
4050
4051 /*
4052 * page was populated with a single value
4053 * that found a home in our hash table
4054 * grab that value from the hash and populate the page
4055 * that we need to populate the page with
4056 */
4057 dptr = (int32_t *)(uintptr_t)dst;
4058 data = c_segment_sv_hash_table[slot_ptr->s_cindx].he_data;
4059 sv_decompress(dptr, data);
4060 if (!(flags & C_KEEP)) {
4061 c_segment_sv_hash_drop_ref(slot_ptr->s_cindx);
4062
4063 OSAddAtomic(-1, &c_segment_pages_compressed);
4064 *slot = 0;
4065 }
4066 if (data) {
4067 OSAddAtomic(1, &c_segment_svp_nonzero_decompressions);
4068 } else {
4069 OSAddAtomic(1, &c_segment_svp_zero_decompressions);
4070 }
4071
4072 pmap_unmap_compressor_page(pn, dst);
4073 return 0;
4074 }
4075
4076 retval = c_decompress_page(dst, slot_ptr, flags, &zeroslot);
4077
4078 /*
4079 * zeroslot will be set to 0 by c_decompress_page if (flags & C_KEEP)
4080 * or (flags & C_DONT_BLOCK) and we found 'c_busy' or 'C_SEG_IS_ONDISK' to be TRUE
4081 */
4082 if (zeroslot) {
4083 *slot = 0;
4084 }
4085
4086 pmap_unmap_compressor_page(pn, dst);
4087
4088 /*
4089 * returns 0 if we successfully decompressed a page from a segment already in memory
4090 * returns 1 if we had to first swap in the segment, before successfully decompressing the page
4091 * returns -1 if we encountered an error swapping in the segment - decompression failed
4092 * returns -2 if (flags & C_DONT_BLOCK) and we found 'c_busy' or 'C_SEG_IS_ONDISK' to be true
4093 */
4094 return retval;
4095 }
4096
4097
4098 int
4099 vm_compressor_free(int *slot, int flags)
4100 {
4101 c_slot_mapping_t slot_ptr;
4102 int zeroslot = 1;
4103 int retval;
4104
4105 assert(flags == 0 || flags == C_DONT_BLOCK);
4106
4107 slot_ptr = (c_slot_mapping_t)slot;
4108
4109 if (slot_ptr->s_cseg == C_SV_CSEG_ID) {
4110 c_segment_sv_hash_drop_ref(slot_ptr->s_cindx);
4111 OSAddAtomic(-1, &c_segment_pages_compressed);
4112
4113 *slot = 0;
4114 return 0;
4115 }
4116 retval = c_decompress_page(NULL, slot_ptr, flags, &zeroslot);
4117 /*
4118 * returns 0 if we successfully freed the specified compressed page
4119 * returns -2 if (flags & C_DONT_BLOCK) and we found 'c_busy' set
4120 */
4121
4122 if (retval == 0) {
4123 *slot = 0;
4124 } else {
4125 assert(retval == -2);
4126 }
4127
4128 return retval;
4129 }
4130
4131
4132 int
4133 vm_compressor_put(ppnum_t pn, int *slot, void **current_chead, char *scratch_buf)
4134 {
4135 char *src;
4136 int retval;
4137
4138 src = pmap_map_compressor_page(pn);
4139 assert(src != NULL);
4140
4141 retval = c_compress_page(src, (c_slot_mapping_t)slot, (c_segment_t *)current_chead, scratch_buf);
4142 pmap_unmap_compressor_page(pn, src);
4143
4144 return retval;
4145 }
4146
4147 void
4148 vm_compressor_transfer(
4149 int *dst_slot_p,
4150 int *src_slot_p)
4151 {
4152 c_slot_mapping_t dst_slot, src_slot;
4153 c_segment_t c_seg;
4154 int c_indx;
4155 c_slot_t cs;
4156
4157 src_slot = (c_slot_mapping_t) src_slot_p;
4158
4159 if (src_slot->s_cseg == C_SV_CSEG_ID) {
4160 *dst_slot_p = *src_slot_p;
4161 *src_slot_p = 0;
4162 return;
4163 }
4164 dst_slot = (c_slot_mapping_t) dst_slot_p;
4165 Retry:
4166 PAGE_REPLACEMENT_DISALLOWED(TRUE);
4167 /* get segment for src_slot */
4168 c_seg = c_segments[src_slot->s_cseg - 1].c_seg;
4169 /* lock segment */
4170 lck_mtx_lock_spin_always(&c_seg->c_lock);
4171 /* wait if it's busy */
4172 if (c_seg->c_busy && !c_seg->c_busy_swapping) {
4173 PAGE_REPLACEMENT_DISALLOWED(FALSE);
4174 c_seg_wait_on_busy(c_seg);
4175 goto Retry;
4176 }
4177 /* find the c_slot */
4178 c_indx = src_slot->s_cindx;
4179 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
4180 /* point the c_slot back to dst_slot instead of src_slot */
4181 cs->c_packed_ptr = C_SLOT_PACK_PTR(dst_slot);
4182 /* transfer */
4183 *dst_slot_p = *src_slot_p;
4184 *src_slot_p = 0;
4185 lck_mtx_unlock_always(&c_seg->c_lock);
4186 PAGE_REPLACEMENT_DISALLOWED(FALSE);
4187 }
4188
4189 #if CONFIG_FREEZE
4190
4191 int freezer_finished_filling = 0;
4192
4193 void
4194 vm_compressor_finished_filling(
4195 void **current_chead)
4196 {
4197 c_segment_t c_seg;
4198
4199 if ((c_seg = *(c_segment_t *)current_chead) == NULL) {
4200 return;
4201 }
4202
4203 assert(c_seg->c_state == C_IS_FILLING);
4204
4205 lck_mtx_lock_spin_always(&c_seg->c_lock);
4206
4207 c_current_seg_filled(c_seg, (c_segment_t *)current_chead);
4208
4209 lck_mtx_unlock_always(&c_seg->c_lock);
4210
4211 freezer_finished_filling++;
4212 }
4213
4214
4215 /*
4216 * This routine is used to transfer the compressed chunks from
4217 * the c_seg/cindx pointed to by slot_p into a new c_seg headed
4218 * by the current_chead and a new cindx within that c_seg.
4219 *
4220 * Currently, this routine is only used by the "freezer backed by
4221 * compressor with swap" mode to create a series of c_segs that
4222 * only contain compressed data belonging to one task. So, we
4223 * move a task's previously compressed data into a set of new
4224 * c_segs which will also hold the task's yet to be compressed data.
4225 */
4226
4227 kern_return_t
4228 vm_compressor_relocate(
4229 void **current_chead,
4230 int *slot_p)
4231 {
4232 c_slot_mapping_t slot_ptr;
4233 c_slot_mapping_t src_slot;
4234 uint32_t c_rounded_size;
4235 uint32_t c_size;
4236 uint16_t dst_slot;
4237 c_slot_t c_dst;
4238 c_slot_t c_src;
4239 int c_indx;
4240 c_segment_t c_seg_dst = NULL;
4241 c_segment_t c_seg_src = NULL;
4242 kern_return_t kr = KERN_SUCCESS;
4243
4244
4245 src_slot = (c_slot_mapping_t) slot_p;
4246
4247 if (src_slot->s_cseg == C_SV_CSEG_ID) {
4248 /*
4249 * no need to relocate... this is a page full of a single
4250 * value which is hashed to a single entry not contained
4251 * in a c_segment_t
4252 */
4253 return kr;
4254 }
4255
4256 Relookup_dst:
4257 c_seg_dst = c_seg_allocate((c_segment_t *)current_chead);
4258 /*
4259 * returns with c_seg lock held
4260 * and PAGE_REPLACEMENT_DISALLOWED(TRUE)...
4261 * c_nextslot has been allocated and
4262 * c_store.c_buffer populated
4263 */
4264 if (c_seg_dst == NULL) {
4265 /*
4266 * Out of compression segments?
4267 */
4268 kr = KERN_RESOURCE_SHORTAGE;
4269 goto out;
4270 }
4271
4272 assert(c_seg_dst->c_busy == 0);
4273
4274 C_SEG_BUSY(c_seg_dst);
4275
4276 dst_slot = c_seg_dst->c_nextslot;
4277
4278 lck_mtx_unlock_always(&c_seg_dst->c_lock);
4279
4280 Relookup_src:
4281 c_seg_src = c_segments[src_slot->s_cseg - 1].c_seg;
4282
4283 assert(c_seg_dst != c_seg_src);
4284
4285 lck_mtx_lock_spin_always(&c_seg_src->c_lock);
4286
4287 if (C_SEG_IS_ONDISK(c_seg_src)) {
4288 /*
4289 * A "thaw" can mark a process as eligible for
4290 * another freeze cycle without bringing any of
4291 * its swapped out c_segs back from disk (because
4292 * that is done on-demand).
4293 *
4294 * If the src c_seg we find for our pre-compressed
4295 * data is already on-disk, then we are dealing
4296 * with an app's data that is already packed and
4297 * swapped out. Don't do anything.
4298 */
4299
4300 PAGE_REPLACEMENT_DISALLOWED(FALSE);
4301
4302 lck_mtx_unlock_always(&c_seg_src->c_lock);
4303
4304 c_seg_src = NULL;
4305
4306 goto out;
4307 }
4308
4309 if (c_seg_src->c_busy) {
4310 PAGE_REPLACEMENT_DISALLOWED(FALSE);
4311 c_seg_wait_on_busy(c_seg_src);
4312
4313 c_seg_src = NULL;
4314
4315 PAGE_REPLACEMENT_DISALLOWED(TRUE);
4316
4317 goto Relookup_src;
4318 }
4319
4320 C_SEG_BUSY(c_seg_src);
4321
4322 lck_mtx_unlock_always(&c_seg_src->c_lock);
4323
4324 PAGE_REPLACEMENT_DISALLOWED(FALSE);
4325
4326 /* find the c_slot */
4327 c_indx = src_slot->s_cindx;
4328
4329 c_src = C_SEG_SLOT_FROM_INDEX(c_seg_src, c_indx);
4330
4331 c_size = UNPACK_C_SIZE(c_src);
4332
4333 assert(c_size);
4334
4335 if (c_size > (uint32_t)(C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES((int32_t)c_seg_dst->c_nextoffset))) {
4336 /*
4337 * This segment is full. We need a new one.
4338 */
4339
4340 PAGE_REPLACEMENT_DISALLOWED(TRUE);
4341
4342 lck_mtx_lock_spin_always(&c_seg_src->c_lock);
4343 C_SEG_WAKEUP_DONE(c_seg_src);
4344 lck_mtx_unlock_always(&c_seg_src->c_lock);
4345
4346 c_seg_src = NULL;
4347
4348 lck_mtx_lock_spin_always(&c_seg_dst->c_lock);
4349
4350 assert(c_seg_dst->c_busy);
4351 assert(c_seg_dst->c_state == C_IS_FILLING);
4352 assert(!c_seg_dst->c_on_minorcompact_q);
4353
4354 c_current_seg_filled(c_seg_dst, (c_segment_t *)current_chead);
4355 assert(*current_chead == NULL);
4356
4357 C_SEG_WAKEUP_DONE(c_seg_dst);
4358
4359 lck_mtx_unlock_always(&c_seg_dst->c_lock);
4360
4361 c_seg_dst = NULL;
4362
4363 PAGE_REPLACEMENT_DISALLOWED(FALSE);
4364
4365 goto Relookup_dst;
4366 }
4367
4368 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, c_seg_dst->c_nextslot);
4369
4370 memcpy(&c_seg_dst->c_store.c_buffer[c_seg_dst->c_nextoffset], &c_seg_src->c_store.c_buffer[c_src->c_offset], c_size);
4371 //is platform alignment actually necessary since wkdm aligns its output?
4372 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
4373
4374 cslot_copy(c_dst, c_src);
4375 c_dst->c_offset = c_seg_dst->c_nextoffset;
4376
4377 if (c_seg_dst->c_firstemptyslot == c_seg_dst->c_nextslot) {
4378 c_seg_dst->c_firstemptyslot++;
4379 }
4380
4381 c_seg_dst->c_slots_used++;
4382 c_seg_dst->c_nextslot++;
4383 c_seg_dst->c_bytes_used += c_rounded_size;
4384 c_seg_dst->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
4385
4386
4387 PACK_C_SIZE(c_src, 0);
4388
4389 c_seg_src->c_bytes_used -= c_rounded_size;
4390 c_seg_src->c_bytes_unused += c_rounded_size;
4391
4392 assert(c_seg_src->c_slots_used);
4393 c_seg_src->c_slots_used--;
4394
4395 if (c_indx < c_seg_src->c_firstemptyslot) {
4396 c_seg_src->c_firstemptyslot = c_indx;
4397 }
4398
4399 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, dst_slot);
4400
4401 PAGE_REPLACEMENT_ALLOWED(TRUE);
4402 slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
4403 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
4404 slot_ptr->s_cseg = c_seg_dst->c_mysegno + 1;
4405 slot_ptr->s_cindx = dst_slot;
4406
4407 PAGE_REPLACEMENT_ALLOWED(FALSE);
4408
4409 out:
4410 if (c_seg_src) {
4411 lck_mtx_lock_spin_always(&c_seg_src->c_lock);
4412
4413 C_SEG_WAKEUP_DONE(c_seg_src);
4414
4415 if (c_seg_src->c_bytes_used == 0 && c_seg_src->c_state != C_IS_FILLING) {
4416 if (!c_seg_src->c_on_minorcompact_q) {
4417 c_seg_need_delayed_compaction(c_seg_src, FALSE);
4418 }
4419 }
4420
4421 lck_mtx_unlock_always(&c_seg_src->c_lock);
4422 }
4423
4424 if (c_seg_dst) {
4425 PAGE_REPLACEMENT_DISALLOWED(TRUE);
4426
4427 lck_mtx_lock_spin_always(&c_seg_dst->c_lock);
4428
4429 if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX_INDEX) {
4430 /*
4431 * Nearing or exceeded maximum slot and offset capacity.
4432 */
4433 assert(c_seg_dst->c_busy);
4434 assert(c_seg_dst->c_state == C_IS_FILLING);
4435 assert(!c_seg_dst->c_on_minorcompact_q);
4436
4437 c_current_seg_filled(c_seg_dst, (c_segment_t *)current_chead);
4438 assert(*current_chead == NULL);
4439 }
4440
4441 C_SEG_WAKEUP_DONE(c_seg_dst);
4442
4443 lck_mtx_unlock_always(&c_seg_dst->c_lock);
4444
4445 c_seg_dst = NULL;
4446
4447 PAGE_REPLACEMENT_DISALLOWED(FALSE);
4448 }
4449
4450 return kr;
4451 }
4452 #endif /* CONFIG_FREEZE */
mirror of XNU