]> git.saurik.com Git - apple/xnu.git/blame - osfmk/vm/vm_compressor.c
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-3789.41.3.tar.gz
[apple/xnu.git] / osfmk / vm / vm_compressor.c
CommitLineData
39236c6e
A		 1/*
2 * Copyright (c) 2000-2013 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <vm/vm_compressor.h>
fe8ab488
A		 30
31#if CONFIG_PHANTOM_CACHE
32#include <vm/vm_phantom_cache.h>
33#endif
34
39236c6e
A		 35#include <vm/vm_map.h>
36#include <vm/vm_pageout.h>
37#include <vm/memory_object.h>
39037602
A		 38#include <vm/vm_compressor_algorithms.h>
39#include <vm/vm_fault.h>
39236c6e
A		 40#include <mach/mach_host.h> /* for host_info() */
41#include <kern/ledger.h>
39037602 42#include <kern/policy_internal.h>
39236c6e 43
3e170ce0
A		 44#include <i386/misc_protos.h>
45
39236c6e
A		 46#include <IOKit/IOHibernatePrivate.h>
47
48/*
49 * vm_compressor_mode has a heirarchy of control to set its value.
50 * boot-args are checked first, then device-tree, and finally
51 * the default value that is defined below. See vm_fault_init() for
52 * the boot-arg & device-tree code.
53 */
54
39236c6e
A		 55int vm_compressor_mode = VM_PAGER_COMPRESSOR_WITH_SWAP;
56int vm_scale = 16;
57
58
04b8595b 59int vm_compressor_is_active = 0;
39236c6e 60int vm_compression_limit = 0;
3e170ce0 61int vm_compressor_available = 0;
39236c6e 62
39236c6e
A		 63extern void vm_pageout_io_throttle(void);
64
65#if CHECKSUM_THE_DATA || CHECKSUM_THE_SWAP || CHECKSUM_THE_COMPRESSED_DATA
66extern unsigned int hash_string(char *cp, int len);
67#endif
68
3e170ce0
A		 69#define UNPACK_C_SIZE(cs) ((cs->c_size == (PAGE_SIZE-1)) ? PAGE_SIZE : cs->c_size)
70#define PACK_C_SIZE(cs, size) (cs->c_size = ((size == PAGE_SIZE) ? PAGE_SIZE - 1 : size))
fe8ab488 71
39236c6e 72
3e170ce0
A		 73struct c_sv_hash_entry {
74 union {
75 struct {
76 uint32_t c_sv_he_ref;
77 uint32_t c_sv_he_data;
78 } c_sv_he;
79 uint64_t c_sv_he_record;
80
81 } c_sv_he_un;
39236c6e
A		 82};
83
3e170ce0
A		 84#define he_ref c_sv_he_un.c_sv_he.c_sv_he_ref
85#define he_data c_sv_he_un.c_sv_he.c_sv_he_data
86#define he_record c_sv_he_un.c_sv_he_record
87
88#define C_SV_HASH_MAX_MISS 32
89#define C_SV_HASH_SIZE ((1 << 10))
90#define C_SV_HASH_MASK ((1 << 10) - 1)
91#define C_SV_CSEG_ID ((1 << 22) - 1)
39236c6e
A		 92
93
94struct c_slot_mapping {
95 uint32_t s_cseg:22, /* segment number + 1 */
96 s_cindx:10; /* index in the segment */
97};
3e170ce0 98#define C_SLOT_MAX_INDEX (1 << 10)
39236c6e
A		 99
100typedef struct c_slot_mapping *c_slot_mapping_t;
101
102
103union c_segu {
104 c_segment_t c_seg;
39037602 105 uintptr_t c_segno;
39236c6e
A		 106};
107
108
109
39037602
A		 110#define C_SLOT_PACK_PTR(ptr) (((uintptr_t)ptr - (uintptr_t) KERNEL_PMAP_HEAP_RANGE_START) >> 2)
111#define C_SLOT_UNPACK_PTR(cslot) ((uintptr_t)(cslot->c_packed_ptr << 2) + (uintptr_t) KERNEL_PMAP_HEAP_RANGE_START)
39236c6e
A		 112
113
114uint32_t c_segment_count = 0;
3e170ce0 115uint32_t c_segment_count_max = 0;
39236c6e
A		 116
117uint64_t c_generation_id = 0;
118uint64_t c_generation_id_flush_barrier;
119
120
121#define HIBERNATE_FLUSHING_SECS_TO_COMPLETE 120
122
123boolean_t hibernate_no_swapspace = FALSE;
124clock_sec_t hibernate_flushing_deadline = 0;
125
126
3e170ce0
A		 127#if RECORD_THE_COMPRESSED_DATA
128char *c_compressed_record_sbuf;
129char *c_compressed_record_ebuf;
130char *c_compressed_record_cptr;
39236c6e
A		 131#endif
132
3e170ce0 133
39236c6e
A		 134queue_head_t c_age_list_head;
135queue_head_t c_swapout_list_head;
136queue_head_t c_swappedin_list_head;
137queue_head_t c_swappedout_list_head;
138queue_head_t c_swappedout_sparse_list_head;
3e170ce0
A		 139queue_head_t c_major_list_head;
140queue_head_t c_filling_list_head;
141queue_head_t c_bad_list_head;
39236c6e
A		 142
143uint32_t c_age_count = 0;
144uint32_t c_swapout_count = 0;
145uint32_t c_swappedin_count = 0;
146uint32_t c_swappedout_count = 0;
147uint32_t c_swappedout_sparse_count = 0;
3e170ce0
A		 148uint32_t c_major_count = 0;
149uint32_t c_filling_count = 0;
150uint32_t c_empty_count = 0;
151uint32_t c_bad_count = 0;
152
39236c6e
A		 153
154queue_head_t c_minor_list_head;
155uint32_t c_minor_count = 0;
156
3e170ce0
A		 157int c_overage_swapped_count = 0;
158int c_overage_swapped_limit = 0;
159
160int c_seg_fixed_array_len;
39236c6e 161union c_segu *c_segments;
3e170ce0
A		 162vm_offset_t c_buffers;
163vm_size_t c_buffers_size;
39236c6e
A		 164caddr_t c_segments_next_page;
165boolean_t c_segments_busy;
166uint32_t c_segments_available;
167uint32_t c_segments_limit;
fe8ab488 168uint32_t c_segments_nearing_limit;
3e170ce0
A		 169
170uint32_t c_segment_svp_in_hash;
171uint32_t c_segment_svp_hash_succeeded;
172uint32_t c_segment_svp_hash_failed;
173uint32_t c_segment_svp_zero_compressions;
174uint32_t c_segment_svp_nonzero_compressions;
175uint32_t c_segment_svp_zero_decompressions;
176uint32_t c_segment_svp_nonzero_decompressions;
177
178uint32_t c_segment_noncompressible_pages;
179
39236c6e
A		 180uint32_t c_segment_pages_compressed;
181uint32_t c_segment_pages_compressed_limit;
fe8ab488 182uint32_t c_segment_pages_compressed_nearing_limit;
39236c6e
A		 183uint32_t c_free_segno_head = (uint32_t)-1;
184
185uint32_t vm_compressor_minorcompact_threshold_divisor = 10;
186uint32_t vm_compressor_majorcompact_threshold_divisor = 10;
187uint32_t vm_compressor_unthrottle_threshold_divisor = 10;
188uint32_t vm_compressor_catchup_threshold_divisor = 10;
189
190#define C_SEGMENTS_PER_PAGE (PAGE_SIZE / sizeof(union c_segu))
191
192
193lck_grp_attr_t vm_compressor_lck_grp_attr;
194lck_attr_t vm_compressor_lck_attr;
195lck_grp_t vm_compressor_lck_grp;
39236c6e 196lck_mtx_t *c_list_lock;
39236c6e 197lck_rw_t c_master_lock;
fe8ab488 198boolean_t decompressions_blocked = FALSE;
39236c6e
A		 199
200zone_t compressor_segment_zone;
201int c_compressor_swap_trigger = 0;
202
203uint32_t compressor_cpus;
204char *compressor_scratch_bufs;
3e170ce0
A		 205char *kdp_compressor_scratch_buf;
206char *kdp_compressor_decompressed_page;
207addr64_t kdp_compressor_decompressed_page_paddr;
208ppnum_t kdp_compressor_decompressed_page_ppnum;
39236c6e
A		 209
210clock_sec_t start_of_sample_period_sec = 0;
211clock_nsec_t start_of_sample_period_nsec = 0;
212clock_sec_t start_of_eval_period_sec = 0;
213clock_nsec_t start_of_eval_period_nsec = 0;
214uint32_t sample_period_decompression_count = 0;
215uint32_t sample_period_compression_count = 0;
216uint32_t last_eval_decompression_count = 0;
217uint32_t last_eval_compression_count = 0;
218
219#define DECOMPRESSION_SAMPLE_MAX_AGE (60 * 30)
220
3e170ce0
A		 221boolean_t vm_swapout_ripe_segments = FALSE;
222uint32_t vm_ripe_target_age = (60 * 60 * 48);
223
39236c6e
A		 224uint32_t swapout_target_age = 0;
225uint32_t age_of_decompressions_during_sample_period[DECOMPRESSION_SAMPLE_MAX_AGE];
226uint32_t overage_decompressions_during_sample_period = 0;
227
d190cdc3 228void do_fastwake_warmup(queue_head_t *, boolean_t);
39236c6e
A		 229boolean_t fastwake_warmup = FALSE;
230boolean_t fastwake_recording_in_progress = FALSE;
231clock_sec_t dont_trim_until_ts = 0;
232
233uint64_t c_segment_warmup_count;
234uint64_t first_c_segment_to_warm_generation_id = 0;
235uint64_t last_c_segment_to_warm_generation_id = 0;
236boolean_t hibernate_flushing = FALSE;
237
fe8ab488
A		 238int64_t c_segment_input_bytes __attribute__((aligned(8))) = 0;
239int64_t c_segment_compressed_bytes __attribute__((aligned(8))) = 0;
240int64_t compressor_bytes_used __attribute__((aligned(8))) = 0;
3e170ce0
A		 241
242
243struct c_sv_hash_entry c_segment_sv_hash_table[C_SV_HASH_SIZE] __attribute__ ((aligned (8)));
244
39236c6e
A		 245
246static boolean_t compressor_needs_to_swap(void);
247static void vm_compressor_swap_trigger_thread(void);
248static void vm_compressor_do_delayed_compactions(boolean_t);
249static void vm_compressor_compact_and_swap(boolean_t);
250static void vm_compressor_age_swapped_in_segments(boolean_t);
39236c6e 251
3e170ce0
A		 252static void vm_compressor_take_paging_space_action(void);
253
39236c6e
A		 254boolean_t vm_compressor_low_on_space(void);
255
256void compute_swapout_target_age(void);
257
258boolean_t c_seg_major_compact(c_segment_t, c_segment_t);
259boolean_t c_seg_major_compact_ok(c_segment_t, c_segment_t);
260
261int c_seg_minor_compaction_and_unlock(c_segment_t, boolean_t);
262int c_seg_do_minor_compaction_and_unlock(c_segment_t, boolean_t, boolean_t, boolean_t);
263void c_seg_try_minor_compaction_and_unlock(c_segment_t c_seg);
39236c6e
A		 264
265void c_seg_move_to_sparse_list(c_segment_t);
266void c_seg_insert_into_q(queue_head_t *, c_segment_t);
267
39236c6e 268uint64_t vm_available_memory(void);
fe8ab488 269uint64_t vm_compressor_pages_compressed(void);
39236c6e 270
39037602
A		 271/*
272 * indicate the need to do a major compaction if
273 * the overall set of in-use compression segments
274 * becomes sparse... on systems that support pressure
275 * driven swapping, this will also cause swapouts to
276 * be initiated.
277 */
278static inline boolean_t vm_compressor_needs_to_major_compact()
279{
280 uint32_t incore_seg_count;
281
282 incore_seg_count = c_segment_count - c_swappedout_count - c_swappedout_sparse_count;
283
284 if ((c_segment_count >= (c_segments_nearing_limit / 8)) &&
285 ((incore_seg_count * C_SEG_MAX_PAGES) - VM_PAGE_COMPRESSOR_COUNT) >
286 ((incore_seg_count / 8) * C_SEG_MAX_PAGES))
287 return (1);
288 return (0);
289}
290
39236c6e
A		 291
292uint64_t
293vm_available_memory(void)
294{
295 return (((uint64_t)AVAILABLE_NON_COMPRESSED_MEMORY) * PAGE_SIZE_64);
296}
297
298
fe8ab488
A		 299uint64_t
300vm_compressor_pages_compressed(void)
301{
302 return (c_segment_pages_compressed * PAGE_SIZE_64);
303}
304
305
39236c6e
A		 306boolean_t
307vm_compressor_low_on_space(void)
308{
fe8ab488
A		 309 if ((c_segment_pages_compressed > c_segment_pages_compressed_nearing_limit) ||
310 (c_segment_count > c_segments_nearing_limit))
39236c6e
A		 311 return (TRUE);
312
313 return (FALSE);
314}
315
316
317int
fe8ab488 318vm_wants_task_throttled(task_t task)
39236c6e 319{
fe8ab488
A		 320 if (task == kernel_task)
321 return (0);
322
39037602 323 if (VM_CONFIG_SWAP_IS_ACTIVE) {
fe8ab488
A		 324 if ((vm_compressor_low_on_space() || HARD_THROTTLE_LIMIT_REACHED()) &&
325 (unsigned int)pmap_compressed(task->map->pmap) > (c_segment_pages_compressed / 4))
39236c6e 326 return (1);
39236c6e
A		 327 }
328 return (0);
329}
330
331
3e170ce0
A		 332
333static uint32_t no_paging_space_action_in_progress = 0;
334extern void memorystatus_send_low_swap_note(void);
335
336static void
337vm_compressor_take_paging_space_action(void)
338{
339 if (no_paging_space_action_in_progress == 0) {
340
341 if (OSCompareAndSwap(0, 1, (UInt32 *)&no_paging_space_action_in_progress)) {
342
343 if (no_paging_space_action()) {
344 memorystatus_send_low_swap_note();
345 }
346
347 no_paging_space_action_in_progress = 0;
348 }
349 }
350}
351
352
39236c6e
A		 353void
354vm_compressor_init_locks(void)
355{
356 lck_grp_attr_setdefault(&vm_compressor_lck_grp_attr);
357 lck_grp_init(&vm_compressor_lck_grp, "vm_compressor", &vm_compressor_lck_grp_attr);
358 lck_attr_setdefault(&vm_compressor_lck_attr);
359
360 lck_rw_init(&c_master_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
39236c6e
A		 361}
362
363
364void
365vm_decompressor_lock(void)
366{
fe8ab488
A		 367 PAGE_REPLACEMENT_ALLOWED(TRUE);
368
369 decompressions_blocked = TRUE;
370
371 PAGE_REPLACEMENT_ALLOWED(FALSE);
39236c6e
A		 372}
373
374void
375vm_decompressor_unlock(void)
376{
fe8ab488
A		 377 PAGE_REPLACEMENT_ALLOWED(TRUE);
378
379 decompressions_blocked = FALSE;
380
381 PAGE_REPLACEMENT_ALLOWED(FALSE);
39236c6e 382
fe8ab488 383 thread_wakeup((event_t)&decompressions_blocked);
39236c6e
A		 384}
385
39037602
A		 386static inline void cslot_copy(c_slot_t cdst, c_slot_t csrc) {
387#if CHECKSUM_THE_DATA
388 cdst->c_hash_data = csrc->c_hash_data;
389#endif
390#if CHECKSUM_THE_COMPRESSED_DATA
391 cdst->c_hash_compressed_data = csrc->c_hash_compressed_data;
392#endif
393 cdst->c_size = csrc->c_size;
394 cdst->c_packed_ptr = csrc->c_packed_ptr;
395}
39236c6e 396
39037602 397vm_map_t compressor_map;
39236c6e
A		 398
399void
400vm_compressor_init(void)
401{
402 thread_t thread;
fe8ab488
A		 403 struct c_slot cs_dummy;
404 c_slot_t cs = &cs_dummy;
3e170ce0
A		 405 int c_segment_min_size;
406 int c_segment_padded_size;
39037602
A		 407 kern_return_t retval = KERN_SUCCESS;
408 vm_offset_t start_addr = 0;
409 vm_size_t c_segments_arr_size = 0, compressor_submap_size = 0;
410#if RECORD_THE_COMPRESSED_DATA
411 vm_size_t c_compressed_record_sbuf_size = 0;
412#endif /* RECORD_THE_COMPRESSED_DATA */
fe8ab488
A		 413
414 /*
415 * ensure that any pointer that gets created from
416 * the vm_page zone can be packed properly
417 */
418 cs->c_packed_ptr = C_SLOT_PACK_PTR(zone_map_min_address);
419
420 if (C_SLOT_UNPACK_PTR(cs) != (uintptr_t)zone_map_min_address)
421 panic("C_SLOT_UNPACK_PTR failed on zone_map_min_address - %p", (void *)zone_map_min_address);
422
423 cs->c_packed_ptr = C_SLOT_PACK_PTR(zone_map_max_address);
424
425 if (C_SLOT_UNPACK_PTR(cs) != (uintptr_t)zone_map_max_address)
426 panic("C_SLOT_UNPACK_PTR failed on zone_map_max_address - %p", (void *)zone_map_max_address);
427
39236c6e
A		 428
429 assert((C_SEGMENTS_PER_PAGE * sizeof(union c_segu)) == PAGE_SIZE);
430
431 PE_parse_boot_argn("vm_compression_limit", &vm_compression_limit, sizeof (vm_compression_limit));
432
433 if (max_mem <= (3ULL * 1024ULL * 1024ULL * 1024ULL)) {
434 vm_compressor_minorcompact_threshold_divisor = 11;
435 vm_compressor_majorcompact_threshold_divisor = 13;
436 vm_compressor_unthrottle_threshold_divisor = 20;
437 vm_compressor_catchup_threshold_divisor = 35;
438 } else {
439 vm_compressor_minorcompact_threshold_divisor = 20;
440 vm_compressor_majorcompact_threshold_divisor = 25;
441 vm_compressor_unthrottle_threshold_divisor = 35;
442 vm_compressor_catchup_threshold_divisor = 50;
443 }
444 /*
445 * vm_page_init_lck_grp is now responsible for calling vm_compressor_init_locks
446 * c_master_lock needs to be available early so that "vm_page_find_contiguous" can
447 * use PAGE_REPLACEMENT_ALLOWED to coordinate with the compressor.
448 */
449
39236c6e 450 c_list_lock = lck_mtx_alloc_init(&vm_compressor_lck_grp, &vm_compressor_lck_attr);
3e170ce0 451
39236c6e 452 queue_init(&c_bad_list_head);
39236c6e
A		 453 queue_init(&c_age_list_head);
454 queue_init(&c_minor_list_head);
3e170ce0
A		 455 queue_init(&c_major_list_head);
456 queue_init(&c_filling_list_head);
39236c6e
A		 457 queue_init(&c_swapout_list_head);
458 queue_init(&c_swappedin_list_head);
459 queue_init(&c_swappedout_list_head);
460 queue_init(&c_swappedout_sparse_list_head);
461
3e170ce0
A		 462 c_segment_min_size = sizeof(struct c_segment) + (C_SEG_SLOT_VAR_ARRAY_MIN_LEN * sizeof(struct c_slot));
463
464 for (c_segment_padded_size = 128; c_segment_padded_size < c_segment_min_size; c_segment_padded_size = c_segment_padded_size << 1);
465
466 compressor_segment_zone = zinit(c_segment_padded_size, 128000 * c_segment_padded_size, PAGE_SIZE, "compressor_segment");
39236c6e
A		 467 zone_change(compressor_segment_zone, Z_CALLERACCT, FALSE);
468 zone_change(compressor_segment_zone, Z_NOENCRYPT, TRUE);
469
3e170ce0 470 c_seg_fixed_array_len = (c_segment_padded_size - sizeof(struct c_segment)) / sizeof(struct c_slot);
39236c6e
A		 471
472 c_free_segno_head = -1;
473 c_segments_available = 0;
474
475 if (vm_compression_limit == 0) {
476 c_segment_pages_compressed_limit = (uint32_t)((max_mem / PAGE_SIZE)) * vm_scale;
477
478#define OLD_SWAP_LIMIT (1024 * 1024 * 16)
479#define MAX_SWAP_LIMIT (1024 * 1024 * 128)
480
481 if (c_segment_pages_compressed_limit > (OLD_SWAP_LIMIT))
482 c_segment_pages_compressed_limit = OLD_SWAP_LIMIT;
483
484 if (c_segment_pages_compressed_limit < (uint32_t)(max_mem / PAGE_SIZE_64))
485 c_segment_pages_compressed_limit = (uint32_t)(max_mem / PAGE_SIZE_64);
486 } else {
487 if (vm_compression_limit < MAX_SWAP_LIMIT)
488 c_segment_pages_compressed_limit = vm_compression_limit;
489 else
490 c_segment_pages_compressed_limit = MAX_SWAP_LIMIT;
491 }
492 if ((c_segments_limit = c_segment_pages_compressed_limit / (C_SEG_BUFSIZE / PAGE_SIZE)) > C_SEG_MAX_LIMIT)
493 c_segments_limit = C_SEG_MAX_LIMIT;
494
fe8ab488
A		 495 c_segment_pages_compressed_nearing_limit = (c_segment_pages_compressed_limit * 98) / 100;
496 c_segments_nearing_limit = (c_segments_limit * 98) / 100;
497
39236c6e
A		 498 c_segments_busy = FALSE;
499
39037602
A		 500 /*
501 * Submap needs space for:
502 * - c_segments
503 * - c_buffers
504 * - swap reclaimations -- C_SEG_BUFSIZE
505 */
506 c_segments_arr_size = vm_map_round_page((sizeof(union c_segu) * c_segments_limit),VM_MAP_PAGE_MASK(kernel_map));
507 c_buffers_size = vm_map_round_page(((vm_size_t)C_SEG_ALLOCSIZE * (vm_size_t)c_segments_limit), VM_MAP_PAGE_MASK(kernel_map));
508
509 compressor_submap_size = c_segments_arr_size + c_buffers_size + C_SEG_BUFSIZE;
510
511#if RECORD_THE_COMPRESSED_DATA
512 c_compressed_record_sbuf_size = (vm_size_t)C_SEG_ALLOCSIZE + (PAGE_SIZE * 2);
513 compressor_submap_size += c_compressed_record_sbuf_size;
514#endif /* RECORD_THE_COMPRESSED_DATA */
515
516 retval = kmem_suballoc(kernel_map, &start_addr, compressor_submap_size,
517 FALSE, VM_FLAGS_ANYWHERE | VM_FLAGS_PERMANENT | VM_MAKE_TAG(0),
518 &compressor_map);
519
520 if (retval != KERN_SUCCESS)
521 panic("vm_compressor_init: kmem_suballoc failed");
522
523 if (kernel_memory_allocate(compressor_map, (vm_offset_t *)(&c_segments), (sizeof(union c_segu) * c_segments_limit), 0, KMA_KOBJECT | KMA_VAONLY | KMA_PERMANENT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS)
3e170ce0 524 panic("vm_compressor_init: kernel_memory_allocate failed - c_segments\n");
39037602 525 if (kernel_memory_allocate(compressor_map, &c_buffers, c_buffers_size, 0, KMA_COMPRESSOR | KMA_VAONLY | KMA_PERMANENT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS)
3e170ce0 526 panic("vm_compressor_init: kernel_memory_allocate failed - c_buffers\n");
39236c6e
A		 527
528 c_segments_next_page = (caddr_t)c_segments;
39037602 529 vm_compressor_algorithm_init();
39236c6e
A		 530
531 {
532 host_basic_info_data_t hinfo;
533 mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
534
535#define BSD_HOST 1
536 host_info((host_t)BSD_HOST, HOST_BASIC_INFO, (host_info_t)&hinfo, &count);
537
538 compressor_cpus = hinfo.max_cpus;
39037602 539 compressor_scratch_bufs = kalloc_tag(compressor_cpus * vm_compressor_get_decode_scratch_size(), VM_KERN_MEMORY_COMPRESSOR);
39236c6e 540
39037602 541 kdp_compressor_scratch_buf = kalloc_tag(vm_compressor_get_decode_scratch_size(), VM_KERN_MEMORY_COMPRESSOR);
3e170ce0
A		 542 kdp_compressor_decompressed_page = kalloc_tag(PAGE_SIZE, VM_KERN_MEMORY_COMPRESSOR);
543 kdp_compressor_decompressed_page_paddr = kvtophys((vm_offset_t)kdp_compressor_decompressed_page);
544 kdp_compressor_decompressed_page_ppnum = (ppnum_t) atop(kdp_compressor_decompressed_page_paddr);
39236c6e 545 }
39037602
A		 546#if CONFIG_FREEZE
547 freezer_compressor_scratch_buf = kalloc_tag(vm_compressor_get_encode_scratch_size(), VM_KERN_MEMORY_COMPRESSOR);
3e170ce0
A		 548#endif
549
550#if RECORD_THE_COMPRESSED_DATA
39037602 551 if (kernel_memory_allocate(compressor_map, (vm_offset_t *)&c_compressed_record_sbuf, c_compressed_record_sbuf_size, 0, KMA_KOBJECT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS)
3e170ce0
A		 552 panic("vm_compressor_init: kernel_memory_allocate failed - c_compressed_record_sbuf\n");
553
554 c_compressed_record_cptr = c_compressed_record_sbuf;
39037602 555 c_compressed_record_ebuf = c_compressed_record_sbuf + c_compressed_record_sbuf_size;
3e170ce0 556#endif
39236c6e
A		 557
558 if (kernel_thread_start_priority((thread_continue_t)vm_compressor_swap_trigger_thread, NULL,
559 BASEPRI_PREEMPT - 1, &thread) != KERN_SUCCESS) {
560 panic("vm_compressor_swap_trigger_thread: create failed");
561 }
39236c6e
A		 562 thread_deallocate(thread);
563
39236c6e
A		 564 if (vm_pageout_internal_start() != KERN_SUCCESS) {
565 panic("vm_compressor_init: Failed to start the internal pageout thread.\n");
566 }
39037602 567 if (VM_CONFIG_SWAP_IS_PRESENT)
fe8ab488 568 vm_compressor_swap_init();
fe8ab488 569
39037602 570 if (VM_CONFIG_COMPRESSOR_IS_ACTIVE)
04b8595b
A		 571 vm_compressor_is_active = 1;
572
39236c6e
A		 573#if CONFIG_FREEZE
574 memorystatus_freeze_enabled = TRUE;
575#endif /* CONFIG_FREEZE */
576
3e170ce0 577 vm_compressor_available = 1;
39236c6e
A		 578
579 vm_page_reactivate_all_throttled();
580}
581
582
583#if VALIDATE_C_SEGMENTS
584
585static void
586c_seg_validate(c_segment_t c_seg, boolean_t must_be_compact)
587{
588 int c_indx;
589 int32_t bytes_used;
590 int32_t bytes_unused;
591 uint32_t c_rounded_size;
592 uint32_t c_size;
593 c_slot_t cs;
594
595 if (c_seg->c_firstemptyslot < c_seg->c_nextslot) {
596 c_indx = c_seg->c_firstemptyslot;
597 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
598
599 if (cs == NULL)
600 panic("c_seg_validate: no slot backing c_firstemptyslot");
601
602 if (cs->c_size)
603 panic("c_seg_validate: c_firstemptyslot has non-zero size (%d)\n", cs->c_size);
604 }
605 bytes_used = 0;
606 bytes_unused = 0;
607
608 for (c_indx = 0; c_indx < c_seg->c_nextslot; c_indx++) {
609
610 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
611
612 c_size = UNPACK_C_SIZE(cs);
613
614 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
615
616 bytes_used += c_rounded_size;
617
618#if CHECKSUM_THE_COMPRESSED_DATA
619 if (c_size && cs->c_hash_compressed_data != hash_string((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size))
620 panic("compressed data doesn't match original");
621#endif
622 }
623
624 if (bytes_used != c_seg->c_bytes_used)
625 panic("c_seg_validate: bytes_used mismatch - found %d, segment has %d\n", bytes_used, c_seg->c_bytes_used);
626
627 if (c_seg->c_bytes_used > C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset))
628 panic("c_seg_validate: c_bytes_used > c_nextoffset - c_nextoffset = %d, c_bytes_used = %d\n",
629 (int32_t)C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset), c_seg->c_bytes_used);
630
631 if (must_be_compact) {
632 if (c_seg->c_bytes_used != C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset))
633 panic("c_seg_validate: c_bytes_used doesn't match c_nextoffset - c_nextoffset = %d, c_bytes_used = %d\n",
634 (int32_t)C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset), c_seg->c_bytes_used);
635 }
636}
637
638#endif
639
640
641void
39037602 642c_seg_need_delayed_compaction(c_segment_t c_seg, boolean_t c_list_lock_held)
39236c6e
A		 643{
644 boolean_t clear_busy = FALSE;
645
39037602
A		 646 if (c_list_lock_held == FALSE) {
647 if ( !lck_mtx_try_lock_spin_always(c_list_lock)) {
648 C_SEG_BUSY(c_seg);
39236c6e 649
39037602
A		 650 lck_mtx_unlock_always(&c_seg->c_lock);
651 lck_mtx_lock_spin_always(c_list_lock);
652 lck_mtx_lock_spin_always(&c_seg->c_lock);
39236c6e 653
39037602
A		 654 clear_busy = TRUE;
655 }
39236c6e 656 }
3e170ce0
A		 657 assert(c_seg->c_state != C_IS_FILLING);
658
659 if (!c_seg->c_on_minorcompact_q && !(C_SEG_IS_ONDISK(c_seg))) {
39236c6e
A		 660 queue_enter(&c_minor_list_head, c_seg, c_segment_t, c_list);
661 c_seg->c_on_minorcompact_q = 1;
662 c_minor_count++;
663 }
39037602
A		 664 if (c_list_lock_held == FALSE)
665 lck_mtx_unlock_always(c_list_lock);
39236c6e
A		 666
667 if (clear_busy == TRUE)
668 C_SEG_WAKEUP_DONE(c_seg);
669}
670
671
672unsigned int c_seg_moved_to_sparse_list = 0;
673
674void
675c_seg_move_to_sparse_list(c_segment_t c_seg)
676{
677 boolean_t clear_busy = FALSE;
678
679 if ( !lck_mtx_try_lock_spin_always(c_list_lock)) {
fe8ab488 680 C_SEG_BUSY(c_seg);
39236c6e
A		 681
682 lck_mtx_unlock_always(&c_seg->c_lock);
683 lck_mtx_lock_spin_always(c_list_lock);
684 lck_mtx_lock_spin_always(&c_seg->c_lock);
685
686 clear_busy = TRUE;
687 }
3e170ce0 688 c_seg_switch_state(c_seg, C_ON_SWAPPEDOUTSPARSE_Q, FALSE);
39236c6e
A		 689
690 c_seg_moved_to_sparse_list++;
691
692 lck_mtx_unlock_always(c_list_lock);
693
694 if (clear_busy == TRUE)
695 C_SEG_WAKEUP_DONE(c_seg);
696}
697
698
699void
700c_seg_insert_into_q(queue_head_t *qhead, c_segment_t c_seg)
701{
702 c_segment_t c_seg_next;
703
704 if (queue_empty(qhead)) {
705 queue_enter(qhead, c_seg, c_segment_t, c_age_list);
706 } else {
707 c_seg_next = (c_segment_t)queue_first(qhead);
708
709 while (TRUE) {
710
711 if (c_seg->c_generation_id < c_seg_next->c_generation_id) {
712 queue_insert_before(qhead, c_seg, c_seg_next, c_segment_t, c_age_list);
713 break;
714 }
715 c_seg_next = (c_segment_t) queue_next(&c_seg_next->c_age_list);
716
717 if (queue_end(qhead, (queue_entry_t) c_seg_next)) {
718 queue_enter(qhead, c_seg, c_segment_t, c_age_list);
719 break;
720 }
721 }
722 }
723}
724
725
726int try_minor_compaction_failed = 0;
727int try_minor_compaction_succeeded = 0;
728
729void
730c_seg_try_minor_compaction_and_unlock(c_segment_t c_seg)
731{
732
733 assert(c_seg->c_on_minorcompact_q);
734 /*
735 * c_seg is currently on the delayed minor compaction
736 * queue and we have c_seg locked... if we can get the
737 * c_list_lock w/o blocking (if we blocked we could deadlock
738 * because the lock order is c_list_lock then c_seg's lock)
739 * we'll pull it from the delayed list and free it directly
740 */
741 if ( !lck_mtx_try_lock_spin_always(c_list_lock)) {
742 /*
743 * c_list_lock is held, we need to bail
744 */
745 try_minor_compaction_failed++;
746
747 lck_mtx_unlock_always(&c_seg->c_lock);
748 } else {
749 try_minor_compaction_succeeded++;
750
fe8ab488 751 C_SEG_BUSY(c_seg);
39236c6e
A		 752 c_seg_do_minor_compaction_and_unlock(c_seg, TRUE, FALSE, FALSE);
753 }
754}
755
756
757int
758c_seg_do_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy, boolean_t need_list_lock, boolean_t disallow_page_replacement)
759{
760 int c_seg_freed;
761
762 assert(c_seg->c_busy);
763
3e170ce0
A		 764 /*
765 * check for the case that can occur when we are not swapping
766 * and this segment has been major compacted in the past
767 * and moved to the majorcompact q to remove it from further
768 * consideration... if the occupancy falls too low we need
769 * to put it back on the age_q so that it will be considered
770 * in the next major compaction sweep... if we don't do this
771 * we will eventually run into the c_segments_limit
772 */
773 if (c_seg->c_state == C_ON_MAJORCOMPACT_Q && C_SEG_SHOULD_MAJORCOMPACT(c_seg)) {
774
775 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
776 }
39236c6e
A		 777 if (!c_seg->c_on_minorcompact_q) {
778 if (clear_busy == TRUE)
779 C_SEG_WAKEUP_DONE(c_seg);
780
781 lck_mtx_unlock_always(&c_seg->c_lock);
782
783 return (0);
784 }
785 queue_remove(&c_minor_list_head, c_seg, c_segment_t, c_list);
786 c_seg->c_on_minorcompact_q = 0;
787 c_minor_count--;
788
789 lck_mtx_unlock_always(c_list_lock);
790
791 if (disallow_page_replacement == TRUE) {
792 lck_mtx_unlock_always(&c_seg->c_lock);
793
794 PAGE_REPLACEMENT_DISALLOWED(TRUE);
795
796 lck_mtx_lock_spin_always(&c_seg->c_lock);
797 }
798 c_seg_freed = c_seg_minor_compaction_and_unlock(c_seg, clear_busy);
799
800 if (disallow_page_replacement == TRUE)
801 PAGE_REPLACEMENT_DISALLOWED(FALSE);
802
803 if (need_list_lock == TRUE)
804 lck_mtx_lock_spin_always(c_list_lock);
805
806 return (c_seg_freed);
807}
808
809
810void
811c_seg_wait_on_busy(c_segment_t c_seg)
812{
813 c_seg->c_wanted = 1;
814 assert_wait((event_t) (c_seg), THREAD_UNINT);
815
816 lck_mtx_unlock_always(&c_seg->c_lock);
817 thread_block(THREAD_CONTINUE_NULL);
818}
819
820
3e170ce0
A		 821void
822c_seg_switch_state(c_segment_t c_seg, int new_state, boolean_t insert_head)
823{
824 int old_state = c_seg->c_state;
39236c6e 825
3e170ce0
A		 826#if __i386__ || __x86_64__
827 if (new_state != C_IS_FILLING)
39037602
A		 828 LCK_MTX_ASSERT(&c_seg->c_lock, LCK_MTX_ASSERT_OWNED);
829 LCK_MTX_ASSERT(c_list_lock, LCK_MTX_ASSERT_OWNED);
3e170ce0
A		 830#endif
831 switch (old_state) {
39236c6e 832
3e170ce0
A		 833 case C_IS_EMPTY:
834 assert(new_state == C_IS_FILLING || new_state == C_IS_FREE);
39236c6e 835
3e170ce0
A		 836 c_empty_count--;
837 break;
838
839 case C_IS_FILLING:
840 assert(new_state == C_ON_AGE_Q || new_state == C_ON_SWAPOUT_Q);
841
842 queue_remove(&c_filling_list_head, c_seg, c_segment_t, c_age_list);
843 c_filling_count--;
844 break;
845
846 case C_ON_AGE_Q:
847 assert(new_state == C_ON_SWAPOUT_Q || new_state == C_ON_MAJORCOMPACT_Q ||
848 new_state == C_IS_FREE);
849
850 queue_remove(&c_age_list_head, c_seg, c_segment_t, c_age_list);
851 c_age_count--;
852 break;
853
854 case C_ON_SWAPPEDIN_Q:
855 assert(new_state == C_ON_AGE_Q || new_state == C_IS_FREE);
856
857 queue_remove(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
858 c_swappedin_count--;
859 break;
860
861 case C_ON_SWAPOUT_Q:
862 assert(new_state == C_ON_SWAPPEDOUT_Q || new_state == C_ON_SWAPPEDOUTSPARSE_Q ||
863 new_state == C_ON_AGE_Q || new_state == C_IS_FREE || new_state == C_IS_EMPTY);
864
865 queue_remove(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
866 thread_wakeup((event_t)&compaction_swapper_running);
867 c_swapout_count--;
868 break;
869
870 case C_ON_SWAPPEDOUT_Q:
39037602
A		 871 assert(new_state == C_ON_SWAPPEDIN_Q || new_state == C_ON_AGE_Q ||
872 new_state == C_ON_SWAPPEDOUTSPARSE_Q ||
3e170ce0
A		 873 new_state == C_ON_BAD_Q || new_state == C_IS_EMPTY || new_state == C_IS_FREE);
874
875 queue_remove(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
876 c_swappedout_count--;
877 break;
878
879 case C_ON_SWAPPEDOUTSPARSE_Q:
39037602 880 assert(new_state == C_ON_SWAPPEDIN_Q || new_state == C_ON_AGE_Q ||
3e170ce0
A		 881 new_state == C_ON_BAD_Q || new_state == C_IS_EMPTY || new_state == C_IS_FREE);
882
883 queue_remove(&c_swappedout_sparse_list_head, c_seg, c_segment_t, c_age_list);
884 c_swappedout_sparse_count--;
885 break;
886
887 case C_ON_MAJORCOMPACT_Q:
888 assert(new_state == C_ON_AGE_Q || new_state == C_IS_FREE);
889
890 queue_remove(&c_major_list_head, c_seg, c_segment_t, c_age_list);
891 c_major_count--;
892 break;
893
894 case C_ON_BAD_Q:
895 assert(new_state == C_IS_FREE);
896
897 queue_remove(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
898 c_bad_count--;
899 break;
900
901 default:
902 panic("c_seg %p has bad c_state = %d\n", c_seg, old_state);
39236c6e 903 }
39236c6e 904
3e170ce0
A		 905 switch(new_state) {
906 case C_IS_FREE:
907 assert(old_state != C_IS_FILLING);
908
909 break;
39236c6e 910
3e170ce0
A		 911 case C_IS_EMPTY:
912 assert(old_state == C_ON_SWAPOUT_Q || old_state == C_ON_SWAPPEDOUT_Q || old_state == C_ON_SWAPPEDOUTSPARSE_Q);
39236c6e 913
3e170ce0
A		 914 c_empty_count++;
915 break;
916
917 case C_IS_FILLING:
918 assert(old_state == C_IS_EMPTY);
919
920 queue_enter(&c_filling_list_head, c_seg, c_segment_t, c_age_list);
921 c_filling_count++;
922 break;
923
924 case C_ON_AGE_Q:
39037602
A		 925 assert(old_state == C_IS_FILLING || old_state == C_ON_SWAPPEDIN_Q || old_state == C_ON_SWAPOUT_Q ||
926 old_state == C_ON_MAJORCOMPACT_Q || old_state == C_ON_SWAPPEDOUT_Q || old_state == C_ON_SWAPPEDOUTSPARSE_Q);
3e170ce0
A		 927
928 if (old_state == C_IS_FILLING)
929 queue_enter(&c_age_list_head, c_seg, c_segment_t, c_age_list);
39037602
A		 930 else {
931 if (!queue_empty(&c_age_list_head)) {
932 c_segment_t c_first;
933
934 c_first = (c_segment_t)queue_first(&c_age_list_head);
935 c_seg->c_creation_ts = c_first->c_creation_ts;
936 }
937 queue_enter_first(&c_age_list_head, c_seg, c_segment_t, c_age_list);
938 }
3e170ce0
A		 939 c_age_count++;
940 break;
941
942 case C_ON_SWAPPEDIN_Q:
943 assert(c_seg->c_state == C_ON_SWAPPEDOUT_Q || c_seg->c_state == C_ON_SWAPPEDOUTSPARSE_Q);
944
945 if (insert_head == TRUE)
946 queue_enter_first(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
947 else
948 queue_enter(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
949 c_swappedin_count++;
950 break;
951
952 case C_ON_SWAPOUT_Q:
953 assert(old_state == C_ON_AGE_Q || old_state == C_IS_FILLING);
954
955 if (insert_head == TRUE)
956 queue_enter_first(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
957 else
958 queue_enter(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
959 c_swapout_count++;
960 break;
961
962 case C_ON_SWAPPEDOUT_Q:
963 assert(c_seg->c_state == C_ON_SWAPOUT_Q);
964
965 if (insert_head == TRUE)
966 queue_enter_first(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
967 else
968 queue_enter(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
969 c_swappedout_count++;
970 break;
971
972 case C_ON_SWAPPEDOUTSPARSE_Q:
973 assert(c_seg->c_state == C_ON_SWAPOUT_Q || c_seg->c_state == C_ON_SWAPPEDOUT_Q);
974
39037602
A		 975 if (insert_head == TRUE)
976 queue_enter_first(&c_swappedout_sparse_list_head, c_seg, c_segment_t, c_age_list);
977 else
978 queue_enter(&c_swappedout_sparse_list_head, c_seg, c_segment_t, c_age_list);
979
3e170ce0
A		 980 c_swappedout_sparse_count++;
981 break;
982
983 case C_ON_MAJORCOMPACT_Q:
984 assert(c_seg->c_state == C_ON_AGE_Q);
985
986 if (insert_head == TRUE)
987 queue_enter_first(&c_major_list_head, c_seg, c_segment_t, c_age_list);
988 else
989 queue_enter(&c_major_list_head, c_seg, c_segment_t, c_age_list);
990 c_major_count++;
991 break;
992
993 case C_ON_BAD_Q:
994 assert(c_seg->c_state == C_ON_SWAPPEDOUT_Q || c_seg->c_state == C_ON_SWAPPEDOUTSPARSE_Q);
995
996 if (insert_head == TRUE)
997 queue_enter_first(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
998 else
999 queue_enter(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
1000 c_bad_count++;
1001 break;
1002
1003 default:
1004 panic("c_seg %p requesting bad c_state = %d\n", c_seg, new_state);
1005 }
1006 c_seg->c_state = new_state;
39236c6e
A		 1007}
1008
1009
3e170ce0 1010
39236c6e
A		 1011void
1012c_seg_free(c_segment_t c_seg)
1013{
fe8ab488 1014 assert(c_seg->c_busy);
39236c6e
A		 1015
1016 lck_mtx_unlock_always(&c_seg->c_lock);
1017 lck_mtx_lock_spin_always(c_list_lock);
1018 lck_mtx_lock_spin_always(&c_seg->c_lock);
1019
1020 c_seg_free_locked(c_seg);
1021}
1022
1023
1024void
1025c_seg_free_locked(c_segment_t c_seg)
1026{
3e170ce0 1027 int segno;
04b8595b 1028 int pages_populated = 0;
39236c6e 1029 int32_t *c_buffer = NULL;
04b8595b 1030 uint64_t c_swap_handle = 0;
39236c6e 1031
3e170ce0 1032 assert(c_seg->c_busy);
39236c6e 1033 assert(!c_seg->c_on_minorcompact_q);
3e170ce0 1034 assert(!c_seg->c_busy_swapping);
39236c6e 1035
3e170ce0
A		 1036 if (c_seg->c_overage_swap == TRUE) {
1037 c_overage_swapped_count--;
1038 c_seg->c_overage_swap = FALSE;
1039 }
1040 if ( !(C_SEG_IS_ONDISK(c_seg)))
1041 c_buffer = c_seg->c_store.c_buffer;
1042 else
1043 c_swap_handle = c_seg->c_store.c_swap_handle;
39236c6e 1044
3e170ce0 1045 c_seg_switch_state(c_seg, C_IS_FREE, FALSE);
39236c6e 1046
3e170ce0 1047 lck_mtx_unlock_always(c_list_lock);
39236c6e 1048
3e170ce0 1049 if (c_buffer) {
39236c6e 1050 pages_populated = (round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE;
39236c6e 1051 c_seg->c_store.c_buffer = NULL;
3e170ce0 1052 } else
39236c6e 1053 c_seg->c_store.c_swap_handle = (uint64_t)-1;
3e170ce0 1054
39236c6e
A		 1055 lck_mtx_unlock_always(&c_seg->c_lock);
1056
1057 if (c_buffer) {
8a3053a0 1058 if (pages_populated)
39037602 1059 kernel_memory_depopulate(compressor_map, (vm_offset_t) c_buffer, pages_populated * PAGE_SIZE, KMA_COMPRESSOR);
39236c6e 1060
3e170ce0
A		 1061 } else if (c_swap_handle) {
1062 /*
1063 * Free swap space on disk.
1064 */
39236c6e 1065 vm_swap_free(c_swap_handle);
3e170ce0
A		 1066 }
1067 lck_mtx_lock_spin_always(&c_seg->c_lock);
1068
1069 C_SEG_WAKEUP_DONE(c_seg);
1070 lck_mtx_unlock_always(&c_seg->c_lock);
1071
1072 segno = c_seg->c_mysegno;
39236c6e 1073
3e170ce0
A		 1074 lck_mtx_lock_spin_always(c_list_lock);
1075 /*
1076 * because the c_buffer is now associated with the segno,
1077 * we can't put the segno back on the free list until
1078 * after we have depopulated the c_buffer range, or
1079 * we run the risk of depopulating a range that is
1080 * now being used in one of the compressor heads
1081 */
1082 c_segments[segno].c_segno = c_free_segno_head;
1083 c_free_segno_head = segno;
1084 c_segment_count--;
1085
1086 lck_mtx_unlock_always(c_list_lock);
39236c6e 1087
39236c6e 1088 lck_mtx_destroy(&c_seg->c_lock, &vm_compressor_lck_grp);
39236c6e 1089
3e170ce0
A		 1090 if (c_seg->c_slot_var_array_len)
1091 kfree(c_seg->c_slot_var_array, sizeof(struct c_slot) * c_seg->c_slot_var_array_len);
39236c6e 1092
39236c6e
A		 1093 zfree(compressor_segment_zone, c_seg);
1094}
1095
1096
1097int c_seg_trim_page_count = 0;
1098
1099void
1100c_seg_trim_tail(c_segment_t c_seg)
1101{
1102 c_slot_t cs;
1103 uint32_t c_size;
1104 uint32_t c_offset;
1105 uint32_t c_rounded_size;
1106 uint16_t current_nextslot;
1107 uint32_t current_populated_offset;
1108
1109 if (c_seg->c_bytes_used == 0)
1110 return;
1111 current_nextslot = c_seg->c_nextslot;
1112 current_populated_offset = c_seg->c_populated_offset;
1113
1114 while (c_seg->c_nextslot) {
1115
1116 cs = C_SEG_SLOT_FROM_INDEX(c_seg, (c_seg->c_nextslot - 1));
1117
1118 c_size = UNPACK_C_SIZE(cs);
1119
1120 if (c_size) {
1121 if (current_nextslot != c_seg->c_nextslot) {
1122 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
1123 c_offset = cs->c_offset + C_SEG_BYTES_TO_OFFSET(c_rounded_size);
1124
1125 c_seg->c_nextoffset = c_offset;
1126 c_seg->c_populated_offset = (c_offset + (C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1)) & ~(C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1);
1127
1128 if (c_seg->c_firstemptyslot > c_seg->c_nextslot)
1129 c_seg->c_firstemptyslot = c_seg->c_nextslot;
1130
1131 c_seg_trim_page_count += ((round_page_32(C_SEG_OFFSET_TO_BYTES(current_populated_offset)) -
1132 round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE);
1133 }
1134 break;
1135 }
1136 c_seg->c_nextslot--;
1137 }
1138 assert(c_seg->c_nextslot);
1139}
1140
1141
1142int
1143c_seg_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy)
1144{
1145 c_slot_mapping_t slot_ptr;
1146 uint32_t c_offset = 0;
1147 uint32_t old_populated_offset;
1148 uint32_t c_rounded_size;
1149 uint32_t c_size;
1150 int c_indx = 0;
1151 int i;
1152 c_slot_t c_dst;
1153 c_slot_t c_src;
39236c6e
A		 1154
1155 assert(c_seg->c_busy);
1156
1157#if VALIDATE_C_SEGMENTS
1158 c_seg_validate(c_seg, FALSE);
1159#endif
1160 if (c_seg->c_bytes_used == 0) {
1161 c_seg_free(c_seg);
1162 return (1);
1163 }
39037602
A		 1164 lck_mtx_unlock_always(&c_seg->c_lock);
1165
39236c6e
A		 1166 if (c_seg->c_firstemptyslot >= c_seg->c_nextslot || C_SEG_UNUSED_BYTES(c_seg) < PAGE_SIZE)
1167 goto done;
1168
39037602
A		 1169#if DEVELOPMENT || DEBUG
1170 C_SEG_MAKE_WRITEABLE(c_seg);
1171#endif
1172
39236c6e
A		 1173#if VALIDATE_C_SEGMENTS
1174 c_seg->c_was_minor_compacted++;
1175#endif
1176 c_indx = c_seg->c_firstemptyslot;
1177 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
1178
1179 old_populated_offset = c_seg->c_populated_offset;
1180 c_offset = c_dst->c_offset;
1181
1182 for (i = c_indx + 1; i < c_seg->c_nextslot && c_offset < c_seg->c_nextoffset; i++) {
1183
1184 c_src = C_SEG_SLOT_FROM_INDEX(c_seg, i);
1185
1186 c_size = UNPACK_C_SIZE(c_src);
1187
1188 if (c_size == 0)
1189 continue;
1190
39037602 1191 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
39236c6e 1192
39037602
A		 1193 memcpy(&c_seg->c_store.c_buffer[c_offset], &c_seg->c_store.c_buffer[c_src->c_offset], c_rounded_size);
1194
1195 cslot_copy(c_dst, c_src);
39236c6e
A		 1196 c_dst->c_offset = c_offset;
1197
1198 slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
1199 slot_ptr->s_cindx = c_indx;
1200
39236c6e
A		 1201 c_offset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
1202 PACK_C_SIZE(c_src, 0);
1203 c_indx++;
1204
1205 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
1206 }
1207 c_seg->c_firstemptyslot = c_indx;
1208 c_seg->c_nextslot = c_indx;
1209 c_seg->c_nextoffset = c_offset;
1210 c_seg->c_populated_offset = (c_offset + (C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1)) & ~(C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1);
1211 c_seg->c_bytes_unused = 0;
1212
1213#if VALIDATE_C_SEGMENTS
1214 c_seg_validate(c_seg, TRUE);
1215#endif
39236c6e
A		 1216 if (old_populated_offset > c_seg->c_populated_offset) {
1217 uint32_t gc_size;
1218 int32_t *gc_ptr;
1219
1220 gc_size = C_SEG_OFFSET_TO_BYTES(old_populated_offset - c_seg->c_populated_offset);
1221 gc_ptr = &c_seg->c_store.c_buffer[c_seg->c_populated_offset];
1222
39037602
A		 1223 kernel_memory_depopulate(compressor_map, (vm_offset_t)gc_ptr, gc_size, KMA_COMPRESSOR);
1224 }
39236c6e 1225
39037602
A		 1226#if DEVELOPMENT || DEBUG
1227 C_SEG_WRITE_PROTECT(c_seg);
1228#endif
39236c6e 1229
39236c6e 1230done:
39037602
A		 1231 if (clear_busy == TRUE) {
1232 lck_mtx_lock_spin_always(&c_seg->c_lock);
1233 C_SEG_WAKEUP_DONE(c_seg);
39236c6e
A		 1234 lck_mtx_unlock_always(&c_seg->c_lock);
1235 }
1236 return (0);
1237}
1238
1239
3e170ce0
A		 1240static void
1241c_seg_alloc_nextslot(c_segment_t c_seg)
1242{
1243 struct c_slot *old_slot_array = NULL;
1244 struct c_slot *new_slot_array = NULL;
1245 int newlen;
1246 int oldlen;
1247
1248 if (c_seg->c_nextslot < c_seg_fixed_array_len)
1249 return;
1250
1251 if ((c_seg->c_nextslot - c_seg_fixed_array_len) >= c_seg->c_slot_var_array_len) {
1252
1253 oldlen = c_seg->c_slot_var_array_len;
1254 old_slot_array = c_seg->c_slot_var_array;
1255
1256 if (oldlen == 0)
1257 newlen = C_SEG_SLOT_VAR_ARRAY_MIN_LEN;
1258 else
1259 newlen = oldlen * 2;
1260
1261 new_slot_array = (struct c_slot *)kalloc(sizeof(struct c_slot) * newlen);
1262
1263 lck_mtx_lock_spin_always(&c_seg->c_lock);
1264
1265 if (old_slot_array)
1266 memcpy((char *)new_slot_array, (char *)old_slot_array, sizeof(struct c_slot) * oldlen);
1267
1268 c_seg->c_slot_var_array_len = newlen;
1269 c_seg->c_slot_var_array = new_slot_array;
1270
1271 lck_mtx_unlock_always(&c_seg->c_lock);
1272
1273 if (old_slot_array)
1274 kfree(old_slot_array, sizeof(struct c_slot) * oldlen);
1275 }
1276}
1277
1278
39236c6e
A		 1279
1280struct {
1281 uint64_t asked_permission;
1282 uint64_t compactions;
1283 uint64_t moved_slots;
1284 uint64_t moved_bytes;
1285 uint64_t wasted_space_in_swapouts;
1286 uint64_t count_of_swapouts;
3e170ce0 1287 uint64_t count_of_freed_segs;
39236c6e
A		 1288} c_seg_major_compact_stats;
1289
1290
04b8595b 1291#define C_MAJOR_COMPACTION_SIZE_APPROPRIATE ((C_SEG_BUFSIZE * 90) / 100)
39236c6e
A		 1292
1293
1294boolean_t
1295c_seg_major_compact_ok(
1296 c_segment_t c_seg_dst,
1297 c_segment_t c_seg_src)
1298{
1299
1300 c_seg_major_compact_stats.asked_permission++;
1301
39236c6e
A		 1302 if (c_seg_src->c_bytes_used >= C_MAJOR_COMPACTION_SIZE_APPROPRIATE &&
1303 c_seg_dst->c_bytes_used >= C_MAJOR_COMPACTION_SIZE_APPROPRIATE)
1304 return (FALSE);
1305
3e170ce0 1306 if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX_INDEX) {
39236c6e
A		 1307 /*
1308 * destination segment is full... can't compact
1309 */
1310 return (FALSE);
1311 }
1312
1313 return (TRUE);
1314}
1315
1316
1317boolean_t
1318c_seg_major_compact(
1319 c_segment_t c_seg_dst,
1320 c_segment_t c_seg_src)
1321{
1322 c_slot_mapping_t slot_ptr;
1323 uint32_t c_rounded_size;
1324 uint32_t c_size;
1325 uint16_t dst_slot;
1326 int i;
1327 c_slot_t c_dst;
1328 c_slot_t c_src;
39236c6e
A		 1329 boolean_t keep_compacting = TRUE;
1330
1331 /*
1332 * segments are not locked but they are both marked c_busy
1333 * which keeps c_decompress from working on them...
1334 * we can safely allocate new pages, move compressed data
1335 * from c_seg_src to c_seg_dst and update both c_segment's
1336 * state w/o holding the master lock
1337 */
39037602
A		 1338#if DEVELOPMENT || DEBUG
1339 C_SEG_MAKE_WRITEABLE(c_seg_dst);
1340#endif
39236c6e
A		 1341
1342#if VALIDATE_C_SEGMENTS
1343 c_seg_dst->c_was_major_compacted++;
1344 c_seg_src->c_was_major_donor++;
1345#endif
1346 c_seg_major_compact_stats.compactions++;
1347
1348 dst_slot = c_seg_dst->c_nextslot;
1349
1350 for (i = 0; i < c_seg_src->c_nextslot; i++) {
1351
1352 c_src = C_SEG_SLOT_FROM_INDEX(c_seg_src, i);
1353
1354 c_size = UNPACK_C_SIZE(c_src);
1355
1356 if (c_size == 0) {
1357 /* BATCH: move what we have so far; */
1358 continue;
1359 }
1360
1361 if (C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset - c_seg_dst->c_nextoffset) < (unsigned) c_size) {
3e170ce0
A		 1362 int size_to_populate;
1363
39236c6e 1364 /* doesn't fit */
3e170ce0
A		 1365 size_to_populate = C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset);
1366
1367 if (size_to_populate == 0) {
39236c6e
A		 1368 /* can't fit */
1369 keep_compacting = FALSE;
1370 break;
1371 }
3e170ce0
A		 1372 if (size_to_populate > C_SEG_MAX_POPULATE_SIZE)
1373 size_to_populate = C_SEG_MAX_POPULATE_SIZE;
1374
39037602 1375 kernel_memory_populate(compressor_map,
39236c6e 1376 (vm_offset_t) &c_seg_dst->c_store.c_buffer[c_seg_dst->c_populated_offset],
3e170ce0
A		 1377 size_to_populate,
1378 KMA_COMPRESSOR,
1379 VM_KERN_MEMORY_COMPRESSOR);
39236c6e 1380
3e170ce0 1381 c_seg_dst->c_populated_offset += C_SEG_BYTES_TO_OFFSET(size_to_populate);
39236c6e
A		 1382 assert(C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset) <= C_SEG_BUFSIZE);
1383 }
3e170ce0 1384 c_seg_alloc_nextslot(c_seg_dst);
39236c6e 1385
39236c6e
A		 1386 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, c_seg_dst->c_nextslot);
1387
1388 memcpy(&c_seg_dst->c_store.c_buffer[c_seg_dst->c_nextoffset], &c_seg_src->c_store.c_buffer[c_src->c_offset], c_size);
1389
1390 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
1391
1392 c_seg_major_compact_stats.moved_slots++;
1393 c_seg_major_compact_stats.moved_bytes += c_size;
1394
39037602 1395 cslot_copy(c_dst, c_src);
39236c6e
A		 1396 c_dst->c_offset = c_seg_dst->c_nextoffset;
1397
1398 if (c_seg_dst->c_firstemptyslot == c_seg_dst->c_nextslot)
1399 c_seg_dst->c_firstemptyslot++;
1400 c_seg_dst->c_nextslot++;
1401 c_seg_dst->c_bytes_used += c_rounded_size;
1402 c_seg_dst->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
1403
1404 PACK_C_SIZE(c_src, 0);
1405
1406 c_seg_src->c_bytes_used -= c_rounded_size;
1407 c_seg_src->c_bytes_unused += c_rounded_size;
1408 c_seg_src->c_firstemptyslot = 0;
1409
3e170ce0 1410 if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX_INDEX) {
39236c6e
A		 1411 /* dest segment is now full */
1412 keep_compacting = FALSE;
1413 break;
1414 }
1415 }
39037602
A		 1416#if DEVELOPMENT || DEBUG
1417 C_SEG_WRITE_PROTECT(c_seg_dst);
1418#endif
39236c6e
A		 1419 if (dst_slot < c_seg_dst->c_nextslot) {
1420
1421 PAGE_REPLACEMENT_ALLOWED(TRUE);
1422 /*
1423 * we've now locked out c_decompress from
1424 * converting the slot passed into it into
1425 * a c_segment_t which allows us to use
1426 * the backptr to change which c_segment and
1427 * index the slot points to
1428 */
1429 while (dst_slot < c_seg_dst->c_nextslot) {
1430
1431 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, dst_slot);
1432
1433 slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
1434 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
1435 slot_ptr->s_cseg = c_seg_dst->c_mysegno + 1;
1436 slot_ptr->s_cindx = dst_slot++;
1437 }
1438 PAGE_REPLACEMENT_ALLOWED(FALSE);
1439 }
1440 return (keep_compacting);
1441}
1442
1443
fe8ab488
A		 1444uint64_t
1445vm_compressor_compute_elapsed_msecs(clock_sec_t end_sec, clock_nsec_t end_nsec, clock_sec_t start_sec, clock_nsec_t start_nsec)
39236c6e
A		 1446{
1447 uint64_t end_msecs;
1448 uint64_t start_msecs;
1449
1450 end_msecs = (end_sec * 1000) + end_nsec / 1000000;
1451 start_msecs = (start_sec * 1000) + start_nsec / 1000000;
1452
1453 return (end_msecs - start_msecs);
1454}
1455
1456
1457
1458uint32_t compressor_eval_period_in_msecs = 250;
1459uint32_t compressor_sample_min_in_msecs = 500;
1460uint32_t compressor_sample_max_in_msecs = 10000;
1461uint32_t compressor_thrashing_threshold_per_10msecs = 50;
1462uint32_t compressor_thrashing_min_per_10msecs = 20;
1463
fe8ab488
A		 1464/* When true, reset sample data next chance we get. */
1465static boolean_t compressor_need_sample_reset = FALSE;
1466
39236c6e
A		 1467extern uint32_t vm_page_filecache_min;
1468
1469
1470void
1471compute_swapout_target_age(void)
1472{
1473 clock_sec_t cur_ts_sec;
1474 clock_nsec_t cur_ts_nsec;
1475 uint32_t min_operations_needed_in_this_sample;
1476 uint64_t elapsed_msecs_in_eval;
1477 uint64_t elapsed_msecs_in_sample;
39236c6e
A		 1478 boolean_t need_eval_reset = FALSE;
1479
1480 clock_get_system_nanotime(&cur_ts_sec, &cur_ts_nsec);
1481
fe8ab488 1482 elapsed_msecs_in_sample = vm_compressor_compute_elapsed_msecs(cur_ts_sec, cur_ts_nsec, start_of_sample_period_sec, start_of_sample_period_nsec);
39236c6e 1483
fe8ab488
A		 1484 if (compressor_need_sample_reset ||
1485 elapsed_msecs_in_sample >= compressor_sample_max_in_msecs) {
1486 compressor_need_sample_reset = TRUE;
39236c6e
A		 1487 need_eval_reset = TRUE;
1488 goto done;
1489 }
fe8ab488 1490 elapsed_msecs_in_eval = vm_compressor_compute_elapsed_msecs(cur_ts_sec, cur_ts_nsec, start_of_eval_period_sec, start_of_eval_period_nsec);
39236c6e
A		 1491
1492 if (elapsed_msecs_in_eval < compressor_eval_period_in_msecs)
1493 goto done;
1494 need_eval_reset = TRUE;
1495
1496 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_START, elapsed_msecs_in_eval, sample_period_compression_count, sample_period_decompression_count, 0, 0);
1497
1498 min_operations_needed_in_this_sample = (compressor_thrashing_min_per_10msecs * (uint32_t)elapsed_msecs_in_eval) / 10;
1499
1500 if ((sample_period_compression_count - last_eval_compression_count) < min_operations_needed_in_this_sample ||
1501 (sample_period_decompression_count - last_eval_decompression_count) < min_operations_needed_in_this_sample) {
1502
1503 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, sample_period_compression_count - last_eval_compression_count,
1504 sample_period_decompression_count - last_eval_decompression_count, 0, 1, 0);
1505
1506 swapout_target_age = 0;
1507
fe8ab488 1508 compressor_need_sample_reset = TRUE;
39236c6e
A		 1509 need_eval_reset = TRUE;
1510 goto done;
1511 }
1512 last_eval_compression_count = sample_period_compression_count;
1513 last_eval_decompression_count = sample_period_decompression_count;
1514
1515 if (elapsed_msecs_in_sample < compressor_sample_min_in_msecs) {
1516
1517 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, swapout_target_age, 0, 0, 5, 0);
1518 goto done;
1519 }
1520 if (sample_period_decompression_count > ((compressor_thrashing_threshold_per_10msecs * elapsed_msecs_in_sample) / 10)) {
1521
1522 uint64_t running_total;
1523 uint64_t working_target;
1524 uint64_t aging_target;
1525 uint32_t oldest_age_of_csegs_sampled = 0;
1526 uint64_t working_set_approximation = 0;
1527
1528 swapout_target_age = 0;
1529
1530 working_target = (sample_period_decompression_count / 100) * 95; /* 95 percent */
1531 aging_target = (sample_period_decompression_count / 100) * 1; /* 1 percent */
1532 running_total = 0;
1533
1534 for (oldest_age_of_csegs_sampled = 0; oldest_age_of_csegs_sampled < DECOMPRESSION_SAMPLE_MAX_AGE; oldest_age_of_csegs_sampled++) {
1535
1536 running_total += age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
1537
1538 working_set_approximation += oldest_age_of_csegs_sampled * age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
1539
1540 if (running_total >= working_target)
1541 break;
1542 }
1543 if (oldest_age_of_csegs_sampled < DECOMPRESSION_SAMPLE_MAX_AGE) {
1544
1545 working_set_approximation = (working_set_approximation * 1000) / elapsed_msecs_in_sample;
1546
1547 if (working_set_approximation < VM_PAGE_COMPRESSOR_COUNT) {
1548
1549 running_total = overage_decompressions_during_sample_period;
1550
1551 for (oldest_age_of_csegs_sampled = DECOMPRESSION_SAMPLE_MAX_AGE - 1; oldest_age_of_csegs_sampled; oldest_age_of_csegs_sampled--) {
1552 running_total += age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
1553
1554 if (running_total >= aging_target)
1555 break;
1556 }
1557 swapout_target_age = (uint32_t)cur_ts_sec - oldest_age_of_csegs_sampled;
1558
1559 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, swapout_target_age, working_set_approximation, VM_PAGE_COMPRESSOR_COUNT, 2, 0);
1560 } else {
1561 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, working_set_approximation, VM_PAGE_COMPRESSOR_COUNT, 0, 3, 0);
1562 }
1563 } else
1564 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, working_target, running_total, 0, 4, 0);
1565
fe8ab488 1566 compressor_need_sample_reset = TRUE;
39236c6e
A		 1567 need_eval_reset = TRUE;
1568 } else
1569 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, sample_period_decompression_count, (compressor_thrashing_threshold_per_10msecs * elapsed_msecs_in_sample) / 10, 0, 6, 0);
1570done:
fe8ab488 1571 if (compressor_need_sample_reset == TRUE) {
39236c6e
A		 1572 bzero(age_of_decompressions_during_sample_period, sizeof(age_of_decompressions_during_sample_period));
1573 overage_decompressions_during_sample_period = 0;
1574
1575 start_of_sample_period_sec = cur_ts_sec;
1576 start_of_sample_period_nsec = cur_ts_nsec;
1577 sample_period_decompression_count = 0;
1578 sample_period_compression_count = 0;
1579 last_eval_decompression_count = 0;
1580 last_eval_compression_count = 0;
fe8ab488 1581 compressor_need_sample_reset = FALSE;
39236c6e
A		 1582 }
1583 if (need_eval_reset == TRUE) {
1584 start_of_eval_period_sec = cur_ts_sec;
1585 start_of_eval_period_nsec = cur_ts_nsec;
1586 }
1587}
1588
1589
fe8ab488 1590int compaction_swapper_init_now = 0;
39236c6e 1591int compaction_swapper_running = 0;
39037602 1592int compaction_swapper_awakened = 0;
39236c6e
A		 1593int compaction_swapper_abort = 0;
1594
1595
1596#if CONFIG_JETSAM
1597boolean_t memorystatus_kill_on_VM_thrashing(boolean_t);
fe8ab488 1598boolean_t memorystatus_kill_on_FC_thrashing(boolean_t);
39236c6e 1599int compressor_thrashing_induced_jetsam = 0;
fe8ab488
A		 1600int filecache_thrashing_induced_jetsam = 0;
1601static boolean_t vm_compressor_thrashing_detected = FALSE;
39236c6e
A		 1602#endif /* CONFIG_JETSAM */
1603
1604static boolean_t
1605compressor_needs_to_swap(void)
1606{
1607 boolean_t should_swap = FALSE;
1608
3e170ce0
A		 1609 if (vm_swapout_ripe_segments == TRUE && c_overage_swapped_count < c_overage_swapped_limit) {
1610 c_segment_t c_seg;
1611 clock_sec_t now;
1612 clock_sec_t age;
1613 clock_nsec_t nsec;
1614
1615 clock_get_system_nanotime(&now, &nsec);
1616 age = 0;
1617
1618 lck_mtx_lock_spin_always(c_list_lock);
1619
1620 if ( !queue_empty(&c_age_list_head)) {
1621 c_seg = (c_segment_t) queue_first(&c_age_list_head);
1622
1623 age = now - c_seg->c_creation_ts;
1624 }
1625 lck_mtx_unlock_always(c_list_lock);
1626
1627 if (age >= vm_ripe_target_age)
1628 return (TRUE);
1629 }
39037602 1630 if (VM_CONFIG_SWAP_IS_ACTIVE) {
39236c6e
A		 1631 if (COMPRESSOR_NEEDS_TO_SWAP()) {
1632 return (TRUE);
1633 }
1634 if (VM_PAGE_Q_THROTTLED(&vm_pageout_queue_external) && vm_page_anonymous_count < (vm_page_inactive_count / 20)) {
1635 return (TRUE);
1636 }
3e170ce0 1637 if (vm_page_free_count < (vm_page_free_reserved - (COMPRESSOR_FREE_RESERVED_LIMIT * 2)))
39236c6e
A		 1638 return (TRUE);
1639 }
1640 compute_swapout_target_age();
1641
1642 if (swapout_target_age) {
1643 c_segment_t c_seg;
1644
1645 lck_mtx_lock_spin_always(c_list_lock);
1646
1647 if (!queue_empty(&c_age_list_head)) {
1648
1649 c_seg = (c_segment_t) queue_first(&c_age_list_head);
1650
fe8ab488 1651 if (c_seg->c_creation_ts > swapout_target_age)
39236c6e
A		 1652 swapout_target_age = 0;
1653 }
1654 lck_mtx_unlock_always(c_list_lock);
1655 }
fe8ab488
A		 1656#if CONFIG_PHANTOM_CACHE
1657 if (vm_phantom_cache_check_pressure())
1658 should_swap = TRUE;
1659#endif
1660 if (swapout_target_age)
1661 should_swap = TRUE;
39236c6e 1662
fe8ab488 1663#if CONFIG_JETSAM
3e170ce0
A		 1664 if (should_swap || c_segment_pages_compressed > c_segment_pages_compressed_nearing_limit) {
1665
1666 if (vm_compressor_thrashing_detected == FALSE) {
1667 vm_compressor_thrashing_detected = TRUE;
fe8ab488 1668
3e170ce0
A		 1669 if (swapout_target_age || c_segment_pages_compressed > c_segment_pages_compressed_nearing_limit) {
1670 memorystatus_kill_on_VM_thrashing(TRUE /* async */);
1671 compressor_thrashing_induced_jetsam++;
1672 } else {
1673 memorystatus_kill_on_FC_thrashing(TRUE /* async */);
1674 filecache_thrashing_induced_jetsam++;
39236c6e 1675 }
3e170ce0
A		 1676 }
1677 /*
1678 * let the jetsam take precedence over
1679 * any major compactions we might have
1680 * been able to do... otherwise we run
1681 * the risk of doing major compactions
1682 * on segments we're about to free up
1683 * due to the jetsam activity.
1684 */
1685 should_swap = FALSE;
39236c6e 1686 }
fe8ab488 1687
3e170ce0
A		 1688#endif /* CONFIG_JETSAM */
1689
1690 if (should_swap == FALSE) {
1691 /*
39037602 1692 * vm_compressor_needs_to_major_compact returns true only if we're
3e170ce0
A		 1693 * about to run out of available compressor segments... in this
1694 * case, we absolutely need to run a major compaction even if
1695 * we've just kicked off a jetsam or we don't otherwise need to
1696 * swap... terminating objects releases
1697 * pages back to the uncompressed cache, but does not guarantee
1698 * that we will free up even a single compression segment
1699 */
39037602 1700 should_swap = vm_compressor_needs_to_major_compact();
3e170ce0
A		 1701 }
1702
1703 /*
1704 * returning TRUE when swap_supported == FALSE
39236c6e
A		 1705 * will cause the major compaction engine to
1706 * run, but will not trigger any swapping...
1707 * segments that have been major compacted
3e170ce0 1708 * will be moved to the majorcompact queue
39236c6e
A		 1709 */
1710 return (should_swap);
1711}
1712
fe8ab488
A		 1713#if CONFIG_JETSAM
1714/*
1715 * This function is called from the jetsam thread after killing something to
1716 * mitigate thrashing.
1717 *
1718 * We need to restart our thrashing detection heuristics since memory pressure
1719 * has potentially changed significantly, and we don't want to detect on old
1720 * data from before the jetsam.
1721 */
1722void
1723vm_thrashing_jetsam_done(void)
39236c6e 1724{
fe8ab488 1725 vm_compressor_thrashing_detected = FALSE;
39236c6e 1726
fe8ab488
A		 1727 /* Were we compressor-thrashing or filecache-thrashing? */
1728 if (swapout_target_age) {
1729 swapout_target_age = 0;
1730 compressor_need_sample_reset = TRUE;
39236c6e 1731 }
fe8ab488
A		 1732#if CONFIG_PHANTOM_CACHE
1733 else {
1734 vm_phantom_cache_restart_sample();
1735 }
1736#endif
39236c6e 1737}
fe8ab488 1738#endif /* CONFIG_JETSAM */
39236c6e
A		 1739
1740uint32_t vm_wake_compactor_swapper_calls = 0;
39037602
A		 1741uint32_t vm_run_compactor_already_running = 0;
1742uint32_t vm_run_compactor_empty_minor_q = 0;
1743uint32_t vm_run_compactor_did_compact = 0;
1744uint32_t vm_run_compactor_waited = 0;
1745
1746void
1747vm_run_compactor(void)
1748{
1749 if (c_segment_count == 0)
1750 return;
1751
1752 lck_mtx_lock_spin_always(c_list_lock);
1753
1754 if (c_minor_count == 0) {
1755 vm_run_compactor_empty_minor_q++;
1756
1757 lck_mtx_unlock_always(c_list_lock);
1758 return;
1759 }
1760 if (compaction_swapper_running) {
1761
1762 if (vm_restricted_to_single_processor == FALSE) {
1763 vm_run_compactor_already_running++;
1764
1765 lck_mtx_unlock_always(c_list_lock);
1766 return;
1767 }
1768 vm_run_compactor_waited++;
1769
1770 assert_wait((event_t)&compaction_swapper_running, THREAD_UNINT);
1771
1772 lck_mtx_unlock_always(c_list_lock);
1773
1774 thread_block(THREAD_CONTINUE_NULL);
1775
1776 return;
1777 }
1778 vm_run_compactor_did_compact++;
1779
1780 fastwake_warmup = FALSE;
1781 compaction_swapper_running = 1;
1782
1783 vm_compressor_do_delayed_compactions(FALSE);
1784
1785 compaction_swapper_running = 0;
1786
1787 lck_mtx_unlock_always(c_list_lock);
1788
1789 thread_wakeup((event_t)&compaction_swapper_running);
1790}
1791
39236c6e
A		 1792
1793void
1794vm_wake_compactor_swapper(void)
1795{
39037602 1796 if (compaction_swapper_running || compaction_swapper_awakened || c_segment_count == 0)
39236c6e
A		 1797 return;
1798
39037602 1799 if (c_minor_count || vm_compressor_needs_to_major_compact()) {
3e170ce0
A		 1800
1801 lck_mtx_lock_spin_always(c_list_lock);
1802
1803 fastwake_warmup = FALSE;
1804
39037602 1805 if (compaction_swapper_running == 0 && compaction_swapper_awakened == 0) {
3e170ce0
A		 1806
1807 vm_wake_compactor_swapper_calls++;
1808
39037602 1809 compaction_swapper_awakened = 1;
3e170ce0 1810 thread_wakeup((event_t)&c_compressor_swap_trigger);
3e170ce0
A		 1811 }
1812 lck_mtx_unlock_always(c_list_lock);
1813 }
1814}
1815
1816
1817void
1818vm_consider_swapping()
1819{
1820 c_segment_t c_seg, c_seg_next;
1821 clock_sec_t now;
1822 clock_nsec_t nsec;
1823
39037602 1824 assert(VM_CONFIG_SWAP_IS_PRESENT);
39236c6e
A		 1825
1826 lck_mtx_lock_spin_always(c_list_lock);
1827
3e170ce0 1828 compaction_swapper_abort = 1;
39236c6e 1829
3e170ce0
A		 1830 while (compaction_swapper_running) {
1831 assert_wait((event_t)&compaction_swapper_running, THREAD_UNINT);
39236c6e 1832
3e170ce0 1833 lck_mtx_unlock_always(c_list_lock);
39236c6e 1834
3e170ce0
A		 1835 thread_block(THREAD_CONTINUE_NULL);
1836
1837 lck_mtx_lock_spin_always(c_list_lock);
1838 }
1839 compaction_swapper_abort = 0;
1840 compaction_swapper_running = 1;
1841
1842 vm_swapout_ripe_segments = TRUE;
1843
1844 if (!queue_empty(&c_major_list_head)) {
1845
1846 clock_get_system_nanotime(&now, &nsec);
1847
1848 c_seg = (c_segment_t)queue_first(&c_major_list_head);
1849
1850 while (!queue_end(&c_major_list_head, (queue_entry_t)c_seg)) {
1851
1852 if (c_overage_swapped_count >= c_overage_swapped_limit)
1853 break;
1854
1855 c_seg_next = (c_segment_t) queue_next(&c_seg->c_age_list);
1856
1857 if ((now - c_seg->c_creation_ts) >= vm_ripe_target_age) {
1858
1859 lck_mtx_lock_spin_always(&c_seg->c_lock);
1860
1861 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
1862
1863 lck_mtx_unlock_always(&c_seg->c_lock);
1864 }
1865 c_seg = c_seg_next;
1866 }
39236c6e 1867 }
3e170ce0
A		 1868 vm_compressor_compact_and_swap(FALSE);
1869
1870 compaction_swapper_running = 0;
1871
1872 vm_swapout_ripe_segments = FALSE;
1873
39236c6e 1874 lck_mtx_unlock_always(c_list_lock);
39037602
A		 1875
1876 thread_wakeup((event_t)&compaction_swapper_running);
39236c6e
A		 1877}
1878
fe8ab488 1879
39236c6e
A		 1880void
1881vm_consider_waking_compactor_swapper(void)
1882{
1883 boolean_t need_wakeup = FALSE;
1884
39037602 1885 if (c_segment_count == 0)
39236c6e 1886 return;
fe8ab488 1887
39037602 1888 if (compaction_swapper_running || compaction_swapper_awakened)
3e170ce0
A		 1889 return;
1890
fe8ab488
A		 1891 if (!compaction_swapper_inited && !compaction_swapper_init_now) {
1892 compaction_swapper_init_now = 1;
1893 need_wakeup = TRUE;
1894 }
39236c6e
A		 1895
1896 if (c_minor_count && (COMPRESSOR_NEEDS_TO_MINOR_COMPACT())) {
1897
1898 need_wakeup = TRUE;
1899
1900 } else if (compressor_needs_to_swap()) {
1901
1902 need_wakeup = TRUE;
1903
1904 } else if (c_minor_count) {
1905 uint64_t total_bytes;
1906
1907 total_bytes = compressor_object->resident_page_count * PAGE_SIZE_64;
1908
1909 if ((total_bytes - compressor_bytes_used) > total_bytes / 10)
1910 need_wakeup = TRUE;
1911 }
1912 if (need_wakeup == TRUE) {
1913
1914 lck_mtx_lock_spin_always(c_list_lock);
1915
1916 fastwake_warmup = FALSE;
1917
39037602 1918 if (compaction_swapper_running == 0 && compaction_swapper_awakened == 0) {
39236c6e
A		 1919 memoryshot(VM_WAKEUP_COMPACTOR_SWAPPER, DBG_FUNC_NONE);
1920
39037602 1921 compaction_swapper_awakened = 1;
39236c6e 1922 thread_wakeup((event_t)&c_compressor_swap_trigger);
39236c6e
A		 1923 }
1924 lck_mtx_unlock_always(c_list_lock);
1925 }
1926}
1927
1928
1929#define C_SWAPOUT_LIMIT 4
1930#define DELAYED_COMPACTIONS_PER_PASS 30
1931
1932void
1933vm_compressor_do_delayed_compactions(boolean_t flush_all)
1934{
1935 c_segment_t c_seg;
1936 int number_compacted = 0;
1937 boolean_t needs_to_swap = FALSE;
1938
1939
39037602 1940 LCK_MTX_ASSERT(c_list_lock, LCK_MTX_ASSERT_OWNED);
39236c6e
A		 1941
1942 while (!queue_empty(&c_minor_list_head) && needs_to_swap == FALSE) {
1943
1944 c_seg = (c_segment_t)queue_first(&c_minor_list_head);
1945
1946 lck_mtx_lock_spin_always(&c_seg->c_lock);
8a3053a0 1947
fe8ab488 1948 if (c_seg->c_busy) {
8a3053a0 1949
fe8ab488
A		 1950 lck_mtx_unlock_always(c_list_lock);
1951 c_seg_wait_on_busy(c_seg);
1952 lck_mtx_lock_spin_always(c_list_lock);
8a3053a0 1953
fe8ab488
A		 1954 continue;
1955 }
1956 C_SEG_BUSY(c_seg);
39236c6e
A		 1957
1958 c_seg_do_minor_compaction_and_unlock(c_seg, TRUE, FALSE, TRUE);
1959
39037602 1960 if (VM_CONFIG_SWAP_IS_ACTIVE && (number_compacted++ > DELAYED_COMPACTIONS_PER_PASS)) {
39236c6e
A		 1961
1962 if ((flush_all == TRUE || compressor_needs_to_swap() == TRUE) && c_swapout_count < C_SWAPOUT_LIMIT)
1963 needs_to_swap = TRUE;
1964
1965 number_compacted = 0;
1966 }
1967 lck_mtx_lock_spin_always(c_list_lock);
1968 }
1969}
1970
1971
1972#define C_SEGMENT_SWAPPEDIN_AGE_LIMIT 10
1973
1974static void
1975vm_compressor_age_swapped_in_segments(boolean_t flush_all)
1976{
1977 c_segment_t c_seg;
1978 clock_sec_t now;
1979 clock_nsec_t nsec;
1980
1981 clock_get_system_nanotime(&now, &nsec);
1982
1983 while (!queue_empty(&c_swappedin_list_head)) {
1984
1985 c_seg = (c_segment_t)queue_first(&c_swappedin_list_head);
1986
1987 if (flush_all == FALSE && (now - c_seg->c_swappedin_ts) < C_SEGMENT_SWAPPEDIN_AGE_LIMIT)
1988 break;
1989
1990 lck_mtx_lock_spin_always(&c_seg->c_lock);
1991
3e170ce0 1992 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
39236c6e
A		 1993
1994 lck_mtx_unlock_always(&c_seg->c_lock);
1995 }
1996}
1997
1998
39037602
A		 1999extern int vm_num_swap_files;
2000extern int vm_num_pinned_swap_files;
2001extern int vm_swappin_enabled;
2002
2003extern unsigned int vm_swapfile_total_segs_used;
2004extern unsigned int vm_swapfile_total_segs_alloced;
2005
2006
39236c6e
A		 2007void
2008vm_compressor_flush(void)
2009{
2010 uint64_t vm_swap_put_failures_at_start;
2011 wait_result_t wait_result = 0;
2012 AbsoluteTime startTime, endTime;
2013 clock_sec_t now_sec;
2014 clock_nsec_t now_nsec;
2015 uint64_t nsec;
2016
2017 HIBLOG("vm_compressor_flush - starting\n");
2018
2019 clock_get_uptime(&startTime);
2020
2021 lck_mtx_lock_spin_always(c_list_lock);
2022
2023 fastwake_warmup = FALSE;
2024 compaction_swapper_abort = 1;
2025
2026 while (compaction_swapper_running) {
2027 assert_wait((event_t)&compaction_swapper_running, THREAD_UNINT);
2028
2029 lck_mtx_unlock_always(c_list_lock);
2030
2031 thread_block(THREAD_CONTINUE_NULL);
2032
2033 lck_mtx_lock_spin_always(c_list_lock);
2034 }
2035 compaction_swapper_abort = 0;
2036 compaction_swapper_running = 1;
2037
2038 hibernate_flushing = TRUE;
2039 hibernate_no_swapspace = FALSE;
2040 c_generation_id_flush_barrier = c_generation_id + 1000;
2041
2042 clock_get_system_nanotime(&now_sec, &now_nsec);
2043 hibernate_flushing_deadline = now_sec + HIBERNATE_FLUSHING_SECS_TO_COMPLETE;
2044
2045 vm_swap_put_failures_at_start = vm_swap_put_failures;
2046
2047 vm_compressor_compact_and_swap(TRUE);
2048
2049 while (!queue_empty(&c_swapout_list_head)) {
2050
2051 assert_wait_timeout((event_t) &compaction_swapper_running, THREAD_INTERRUPTIBLE, 5000, 1000*NSEC_PER_USEC);
2052
2053 lck_mtx_unlock_always(c_list_lock);
2054
2055 wait_result = thread_block(THREAD_CONTINUE_NULL);
2056
2057 lck_mtx_lock_spin_always(c_list_lock);
2058
2059 if (wait_result == THREAD_TIMED_OUT)
2060 break;
2061 }
2062 hibernate_flushing = FALSE;
2063 compaction_swapper_running = 0;
2064
2065 if (vm_swap_put_failures > vm_swap_put_failures_at_start)
2066 HIBLOG("vm_compressor_flush failed to clean %llu segments - vm_page_compressor_count(%d)\n",
2067 vm_swap_put_failures - vm_swap_put_failures_at_start, VM_PAGE_COMPRESSOR_COUNT);
2068
2069 lck_mtx_unlock_always(c_list_lock);
2070
39037602
A		 2071 thread_wakeup((event_t)&compaction_swapper_running);
2072
39236c6e
A		 2073 clock_get_uptime(&endTime);
2074 SUB_ABSOLUTETIME(&endTime, &startTime);
2075 absolutetime_to_nanoseconds(endTime, &nsec);
2076
39037602
A		 2077 HIBLOG("vm_compressor_flush completed - took %qd msecs - vm_num_swap_files = %d, vm_num_pinned_swap_files = %d, vm_swappin_enabled = %d\n",
2078 nsec / 1000000ULL, vm_num_swap_files, vm_num_pinned_swap_files, vm_swappin_enabled);
39236c6e
A		 2079}
2080
2081
39236c6e
A		 2082int compaction_swap_trigger_thread_awakened = 0;
2083
2084static void
2085vm_compressor_swap_trigger_thread(void)
2086{
3e170ce0
A		 2087 current_thread()->options |= TH_OPT_VMPRIV;
2088
fe8ab488
A		 2089 /*
2090 * compaction_swapper_init_now is set when the first call to
2091 * vm_consider_waking_compactor_swapper is made from
2092 * vm_pageout_scan... since this function is called upon
2093 * thread creation, we want to make sure to delay adjusting
2094 * the tuneables until we are awakened via vm_pageout_scan
2095 * so that we are at a point where the vm_swapfile_open will
2096 * be operating on the correct directory (in case the default
2097 * of /var/vm/ is overridden by the dymanic_pager
2098 */
39037602
A		 2099 if (compaction_swapper_init_now) {
2100 vm_compaction_swapper_do_init();
39236c6e 2101
3e170ce0
A		 2102 if (vm_restricted_to_single_processor == TRUE)
2103 thread_vm_bind_group_add();
2104
39037602 2105 compaction_swapper_init_now = 0;
fe8ab488 2106 }
39236c6e
A		 2107 lck_mtx_lock_spin_always(c_list_lock);
2108
2109 compaction_swap_trigger_thread_awakened++;
39037602 2110 compaction_swapper_awakened = 0;
39236c6e 2111
39037602 2112 if (compaction_swapper_running == 0) {
39236c6e 2113
39037602
A		 2114 compaction_swapper_running = 1;
2115
2116 vm_compressor_compact_and_swap(FALSE);
2117
2118 compaction_swapper_running = 0;
2119 }
39236c6e
A		 2120 assert_wait((event_t)&c_compressor_swap_trigger, THREAD_UNINT);
2121
39037602
A		 2122 if (compaction_swapper_running == 0)
2123 thread_wakeup((event_t)&compaction_swapper_running);
39236c6e
A		 2124
2125 lck_mtx_unlock_always(c_list_lock);
2126
2127 thread_block((thread_continue_t)vm_compressor_swap_trigger_thread);
2128
2129 /* NOTREACHED */
2130}
2131
2132
2133void
2134vm_compressor_record_warmup_start(void)
2135{
2136 c_segment_t c_seg;
2137
2138 lck_mtx_lock_spin_always(c_list_lock);
2139
8a3053a0
A		 2140 if (first_c_segment_to_warm_generation_id == 0) {
2141 if (!queue_empty(&c_age_list_head)) {
39236c6e 2142
8a3053a0 2143 c_seg = (c_segment_t)queue_last(&c_age_list_head);
39236c6e 2144
8a3053a0
A		 2145 first_c_segment_to_warm_generation_id = c_seg->c_generation_id;
2146 } else
2147 first_c_segment_to_warm_generation_id = 0;
39236c6e 2148
8a3053a0
A		 2149 fastwake_recording_in_progress = TRUE;
2150 }
39236c6e
A		 2151 lck_mtx_unlock_always(c_list_lock);
2152}
2153
2154
2155void
2156vm_compressor_record_warmup_end(void)
2157{
2158 c_segment_t c_seg;
2159
2160 lck_mtx_lock_spin_always(c_list_lock);
2161
8a3053a0 2162 if (fastwake_recording_in_progress == TRUE) {
39236c6e 2163
8a3053a0 2164 if (!queue_empty(&c_age_list_head)) {
39236c6e 2165
8a3053a0
A		 2166 c_seg = (c_segment_t)queue_last(&c_age_list_head);
2167
2168 last_c_segment_to_warm_generation_id = c_seg->c_generation_id;
2169 } else
2170 last_c_segment_to_warm_generation_id = first_c_segment_to_warm_generation_id;
39236c6e 2171
8a3053a0 2172 fastwake_recording_in_progress = FALSE;
39236c6e 2173
8a3053a0
A		 2174 HIBLOG("vm_compressor_record_warmup (%qd - %qd)\n", first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id);
2175 }
39236c6e
A		 2176 lck_mtx_unlock_always(c_list_lock);
2177}
2178
2179
39037602 2180#define DELAY_TRIM_ON_WAKE_SECS 25
39236c6e
A		 2181
2182void
8a3053a0 2183vm_compressor_delay_trim(void)
39236c6e 2184{
8a3053a0 2185 clock_sec_t sec;
39236c6e
A		 2186 clock_nsec_t nsec;
2187
2188 clock_get_system_nanotime(&sec, &nsec);
2189 dont_trim_until_ts = sec + DELAY_TRIM_ON_WAKE_SECS;
8a3053a0 2190}
39236c6e 2191
39236c6e 2192
8a3053a0
A		 2193void
2194vm_compressor_do_warmup(void)
2195{
39236c6e
A		 2196 lck_mtx_lock_spin_always(c_list_lock);
2197
8a3053a0
A		 2198 if (first_c_segment_to_warm_generation_id == last_c_segment_to_warm_generation_id) {
2199 first_c_segment_to_warm_generation_id = last_c_segment_to_warm_generation_id = 0;
2200
2201 lck_mtx_unlock_always(c_list_lock);
2202 return;
2203 }
2204
39037602 2205 if (compaction_swapper_running == 0 && compaction_swapper_awakened == 0) {
39236c6e
A		 2206
2207 fastwake_warmup = TRUE;
39037602
A		 2208
2209 compaction_swapper_awakened = 1;
39236c6e
A		 2210 thread_wakeup((event_t)&c_compressor_swap_trigger);
2211 }
2212 lck_mtx_unlock_always(c_list_lock);
2213}
2214
d190cdc3
A		 2215void
2216do_fastwake_warmup_all(void)
2217{
2218
2219 lck_mtx_lock_spin_always(c_list_lock);
2220
2221 if (queue_empty(&c_swappedout_list_head) && queue_empty(&c_swappedout_sparse_list_head)) {
2222
2223 lck_mtx_unlock_always(c_list_lock);
2224 return;
2225 }
2226
2227 fastwake_warmup = TRUE;
2228
2229 do_fastwake_warmup(&c_swappedout_list_head, TRUE);
2230
2231 do_fastwake_warmup(&c_swappedout_sparse_list_head, TRUE);
2232
2233 fastwake_warmup = FALSE;
2234
2235 lck_mtx_unlock_always(c_list_lock);
2236
2237}
39236c6e
A		 2238
2239void
d190cdc3 2240do_fastwake_warmup(queue_head_t *c_queue, boolean_t consider_all_cseg)
39236c6e 2241{
39236c6e 2242 c_segment_t c_seg = NULL;
8a3053a0
A		 2243 AbsoluteTime startTime, endTime;
2244 uint64_t nsec;
2245
2246
2247 HIBLOG("vm_compressor_fastwake_warmup (%qd - %qd) - starting\n", first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id);
2248
2249 clock_get_uptime(&startTime);
39236c6e
A		 2250
2251 lck_mtx_unlock_always(c_list_lock);
2252
39037602
A		 2253 proc_set_thread_policy(current_thread(),
2254 TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER2);
39236c6e
A		 2255
2256 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2257
2258 lck_mtx_lock_spin_always(c_list_lock);
2259
d190cdc3 2260 while (!queue_empty(c_queue) && fastwake_warmup == TRUE) {
39236c6e 2261
d190cdc3 2262 c_seg = (c_segment_t) queue_first(c_queue);
39236c6e 2263
d190cdc3
A		 2264 if (consider_all_cseg == FALSE) {
2265 if (c_seg->c_generation_id < first_c_segment_to_warm_generation_id ||
2266 c_seg->c_generation_id > last_c_segment_to_warm_generation_id)
2267 break;
39236c6e 2268
d190cdc3
A		 2269 if (vm_page_free_count < (AVAILABLE_MEMORY / 4))
2270 break;
2271 }
4bd07ac2 2272
39236c6e
A		 2273 lck_mtx_lock_spin_always(&c_seg->c_lock);
2274 lck_mtx_unlock_always(c_list_lock);
2275
8a3053a0
A		 2276 if (c_seg->c_busy) {
2277 PAGE_REPLACEMENT_DISALLOWED(FALSE);
39236c6e 2278 c_seg_wait_on_busy(c_seg);
8a3053a0
A		 2279 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2280 } else {
39037602
A		 2281 if (c_seg_swapin(c_seg, TRUE, FALSE) == 0)
2282 lck_mtx_unlock_always(&c_seg->c_lock);
39236c6e 2283 c_segment_warmup_count++;
8a3053a0
A		 2284
2285 PAGE_REPLACEMENT_DISALLOWED(FALSE);
39236c6e 2286 vm_pageout_io_throttle();
8a3053a0 2287 PAGE_REPLACEMENT_DISALLOWED(TRUE);
39236c6e
A		 2288 }
2289 lck_mtx_lock_spin_always(c_list_lock);
2290 }
2291 lck_mtx_unlock_always(c_list_lock);
2292
2293 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2294
39037602
A		 2295 proc_set_thread_policy(current_thread(),
2296 TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER0);
39236c6e 2297
8a3053a0
A		 2298 clock_get_uptime(&endTime);
2299 SUB_ABSOLUTETIME(&endTime, &startTime);
2300 absolutetime_to_nanoseconds(endTime, &nsec);
2301
2302 HIBLOG("vm_compressor_fastwake_warmup completed - took %qd msecs\n", nsec / 1000000ULL);
2303
39236c6e 2304 lck_mtx_lock_spin_always(c_list_lock);
8a3053a0 2305
d190cdc3
A		 2306 if (consider_all_cseg == FALSE) {
2307 first_c_segment_to_warm_generation_id = last_c_segment_to_warm_generation_id = 0;
2308 }
39236c6e
A		 2309}
2310
2311
2312void
2313vm_compressor_compact_and_swap(boolean_t flush_all)
2314{
2315 c_segment_t c_seg, c_seg_next;
2316 boolean_t keep_compacting;
3e170ce0
A		 2317 clock_sec_t now;
2318 clock_nsec_t nsec;
39236c6e
A		 2319
2320
2321 if (fastwake_warmup == TRUE) {
2322 uint64_t starting_warmup_count;
2323
2324 starting_warmup_count = c_segment_warmup_count;
2325
2326 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 11) | DBG_FUNC_START, c_segment_warmup_count,
2327 first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id, 0, 0);
d190cdc3 2328 do_fastwake_warmup(&c_swappedout_list_head, FALSE);
39236c6e
A		 2329 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 11) | DBG_FUNC_END, c_segment_warmup_count, c_segment_warmup_count - starting_warmup_count, 0, 0, 0);
2330
2331 fastwake_warmup = FALSE;
2332 }
2333
8a3053a0
A		 2334 /*
2335 * it's possible for the c_age_list_head to be empty if we
2336 * hit our limits for growing the compressor pool and we subsequently
2337 * hibernated... on the next hibernation we could see the queue as
2338 * empty and not proceeed even though we have a bunch of segments on
2339 * the swapped in queue that need to be dealt with.
2340 */
2341 vm_compressor_do_delayed_compactions(flush_all);
2342
2343 vm_compressor_age_swapped_in_segments(flush_all);
2344
3e170ce0
A		 2345 /*
2346 * we only need to grab the timestamp once per
2347 * invocation of this function since the
2348 * timescale we're interested in is measured
2349 * in days
2350 */
2351 clock_get_system_nanotime(&now, &nsec);
8a3053a0 2352
39236c6e
A		 2353 while (!queue_empty(&c_age_list_head) && compaction_swapper_abort == 0) {
2354
2355 if (hibernate_flushing == TRUE) {
2356 clock_sec_t sec;
39236c6e
A		 2357
2358 if (hibernate_should_abort()) {
2359 HIBLOG("vm_compressor_flush - hibernate_should_abort returned TRUE\n");
2360 break;
2361 }
2362 if (hibernate_no_swapspace == TRUE) {
2363 HIBLOG("vm_compressor_flush - out of swap space\n");
2364 break;
2365 }
39037602
A		 2366 if (vm_swap_files_pinned() == FALSE) {
2367 HIBLOG("vm_compressor_flush - unpinned swap files\n");
2368 break;
2369 }
2370 if (hibernate_in_progress_with_pinned_swap == TRUE &&
2371 (vm_swapfile_total_segs_alloced == vm_swapfile_total_segs_used)) {
2372 HIBLOG("vm_compressor_flush - out of pinned swap space\n");
2373 break;
2374 }
39236c6e
A		 2375 clock_get_system_nanotime(&sec, &nsec);
2376
2377 if (sec > hibernate_flushing_deadline) {
2378 HIBLOG("vm_compressor_flush - failed to finish before deadline\n");
2379 break;
2380 }
2381 }
2382 if (c_swapout_count >= C_SWAPOUT_LIMIT) {
2383
2384 assert_wait_timeout((event_t) &compaction_swapper_running, THREAD_INTERRUPTIBLE, 100, 1000*NSEC_PER_USEC);
2385
2386 lck_mtx_unlock_always(c_list_lock);
2387
2388 thread_block(THREAD_CONTINUE_NULL);
2389
2390 lck_mtx_lock_spin_always(c_list_lock);
2391 }
2392 /*
2393 * Minor compactions
2394 */
2395 vm_compressor_do_delayed_compactions(flush_all);
2396
2397 vm_compressor_age_swapped_in_segments(flush_all);
2398
2399 if (c_swapout_count >= C_SWAPOUT_LIMIT) {
2400 /*
2401 * we timed out on the above thread_block
2402 * let's loop around and try again
2403 * the timeout allows us to continue
2404 * to do minor compactions to make
2405 * more memory available
2406 */
2407 continue;
2408 }
2409
2410 /*
2411 * Swap out segments?
2412 */
2413 if (flush_all == FALSE) {
2414 boolean_t needs_to_swap;
2415
2416 lck_mtx_unlock_always(c_list_lock);
2417
2418 needs_to_swap = compressor_needs_to_swap();
2419
3e170ce0
A		 2420 if (needs_to_swap == TRUE && vm_swap_low_on_space())
2421 vm_compressor_take_paging_space_action();
2422
39236c6e
A		 2423 lck_mtx_lock_spin_always(c_list_lock);
2424
2425 if (needs_to_swap == FALSE)
2426 break;
2427 }
2428 if (queue_empty(&c_age_list_head))
2429 break;
2430 c_seg = (c_segment_t) queue_first(&c_age_list_head);
2431
3e170ce0 2432 assert(c_seg->c_state == C_ON_AGE_Q);
39236c6e 2433
3e170ce0 2434 if (flush_all == TRUE && c_seg->c_generation_id > c_generation_id_flush_barrier)
39236c6e 2435 break;
3e170ce0 2436
39236c6e
A		 2437 lck_mtx_lock_spin_always(&c_seg->c_lock);
2438
2439 if (c_seg->c_busy) {
2440
2441 lck_mtx_unlock_always(c_list_lock);
2442 c_seg_wait_on_busy(c_seg);
2443 lck_mtx_lock_spin_always(c_list_lock);
2444
2445 continue;
2446 }
fe8ab488 2447 C_SEG_BUSY(c_seg);
39236c6e
A		 2448
2449 if (c_seg_do_minor_compaction_and_unlock(c_seg, FALSE, TRUE, TRUE)) {
2450 /*
2451 * found an empty c_segment and freed it
2452 * so go grab the next guy in the queue
2453 */
3e170ce0 2454 c_seg_major_compact_stats.count_of_freed_segs++;
39236c6e
A		 2455 continue;
2456 }
2457 /*
2458 * Major compaction
2459 */
2460 keep_compacting = TRUE;
2461
2462 while (keep_compacting == TRUE) {
2463
2464 assert(c_seg->c_busy);
2465
2466 /* look for another segment to consolidate */
2467
2468 c_seg_next = (c_segment_t) queue_next(&c_seg->c_age_list);
2469
2470 if (queue_end(&c_age_list_head, (queue_entry_t)c_seg_next))
2471 break;
2472
3e170ce0
A		 2473 assert(c_seg_next->c_state == C_ON_AGE_Q);
2474
39236c6e
A		 2475 if (c_seg_major_compact_ok(c_seg, c_seg_next) == FALSE)
2476 break;
2477
2478 lck_mtx_lock_spin_always(&c_seg_next->c_lock);
2479
2480 if (c_seg_next->c_busy) {
2481
2482 lck_mtx_unlock_always(c_list_lock);
2483 c_seg_wait_on_busy(c_seg_next);
2484 lck_mtx_lock_spin_always(c_list_lock);
2485
2486 continue;
2487 }
2488 /* grab that segment */
fe8ab488 2489 C_SEG_BUSY(c_seg_next);
39236c6e
A		 2490
2491 if (c_seg_do_minor_compaction_and_unlock(c_seg_next, FALSE, TRUE, TRUE)) {
2492 /*
2493 * found an empty c_segment and freed it
2494 * so we can't continue to use c_seg_next
2495 */
3e170ce0 2496 c_seg_major_compact_stats.count_of_freed_segs++;
39236c6e
A		 2497 continue;
2498 }
2499
2500 /* unlock the list ... */
2501 lck_mtx_unlock_always(c_list_lock);
2502
2503 /* do the major compaction */
2504
2505 keep_compacting = c_seg_major_compact(c_seg, c_seg_next);
2506
2507 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2508
2509 lck_mtx_lock_spin_always(&c_seg_next->c_lock);
2510 /*
2511 * run a minor compaction on the donor segment
2512 * since we pulled at least some of it's
2513 * data into our target... if we've emptied
2514 * it, now is a good time to free it which
2515 * c_seg_minor_compaction_and_unlock also takes care of
2516 *
2517 * by passing TRUE, we ask for c_busy to be cleared
2518 * and c_wanted to be taken care of
2519 */
3e170ce0
A		 2520 if (c_seg_minor_compaction_and_unlock(c_seg_next, TRUE))
2521 c_seg_major_compact_stats.count_of_freed_segs++;
39236c6e
A		 2522
2523 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2524
2525 /* relock the list */
2526 lck_mtx_lock_spin_always(c_list_lock);
2527
2528 } /* major compaction */
2529
39236c6e
A		 2530 lck_mtx_lock_spin_always(&c_seg->c_lock);
2531
2532 assert(c_seg->c_busy);
39236c6e
A		 2533 assert(!c_seg->c_on_minorcompact_q);
2534
39037602 2535 if (VM_CONFIG_SWAP_IS_ACTIVE) {
3e170ce0
A		 2536 /*
2537 * This mode of putting a generic c_seg on the swapout list is
39037602 2538 * only supported when we have general swapping enabled
3e170ce0 2539 */
39037602
A		 2540 c_seg_switch_state(c_seg, C_ON_SWAPOUT_Q, FALSE);
2541 } else {
2542 if ((vm_swapout_ripe_segments == TRUE && c_overage_swapped_count < c_overage_swapped_limit)) {
2543
2544 assert(VM_CONFIG_SWAP_IS_PRESENT);
2545 /*
2546 * we are running compressor sweeps with swap-behind
2547 * make sure the c_seg has aged enough before swapping it
2548 * out...
2549 */
2550 if ((now - c_seg->c_creation_ts) >= vm_ripe_target_age) {
2551 c_seg->c_overage_swap = TRUE;
2552 c_overage_swapped_count++;
2553 c_seg_switch_state(c_seg, C_ON_SWAPOUT_Q, FALSE);
3e170ce0
A		 2554 }
2555 }
2556 }
2557 if (c_seg->c_state == C_ON_AGE_Q) {
2558 /*
2559 * this c_seg didn't get moved to the swapout queue
2560 * so we need to move it out of the way...
2561 * we just did a major compaction on it so put it
2562 * on that queue
2563 */
2564 c_seg_switch_state(c_seg, C_ON_MAJORCOMPACT_Q, FALSE);
39236c6e 2565 } else {
3e170ce0
A		 2566 c_seg_major_compact_stats.wasted_space_in_swapouts += C_SEG_BUFSIZE - c_seg->c_bytes_used;
2567 c_seg_major_compact_stats.count_of_swapouts++;
39236c6e
A		 2568 }
2569 C_SEG_WAKEUP_DONE(c_seg);
2570
2571 lck_mtx_unlock_always(&c_seg->c_lock);
2572
2573 if (c_swapout_count) {
2574 lck_mtx_unlock_always(c_list_lock);
2575
2576 thread_wakeup((event_t)&c_swapout_list_head);
2577
2578 lck_mtx_lock_spin_always(c_list_lock);
2579 }
2580 }
2581}
2582
2583
2584static c_segment_t
2585c_seg_allocate(c_segment_t *current_chead)
2586{
39236c6e 2587 c_segment_t c_seg;
3e170ce0
A		 2588 int min_needed;
2589 int size_to_populate;
2590
2591 if (vm_compressor_low_on_space())
2592 vm_compressor_take_paging_space_action();
39236c6e
A		 2593
2594 if ( (c_seg = *current_chead) == NULL ) {
2595 uint32_t c_segno;
2596
39236c6e
A		 2597 lck_mtx_lock_spin_always(c_list_lock);
2598
2599 while (c_segments_busy == TRUE) {
2600 assert_wait((event_t) (&c_segments_busy), THREAD_UNINT);
2601
2602 lck_mtx_unlock_always(c_list_lock);
2603
2604 thread_block(THREAD_CONTINUE_NULL);
2605
2606 lck_mtx_lock_spin_always(c_list_lock);
2607 }
2608 if (c_free_segno_head == (uint32_t)-1) {
3e170ce0 2609 uint32_t c_segments_available_new;
39236c6e
A		 2610
2611 if (c_segments_available >= c_segments_limit || c_segment_pages_compressed >= c_segment_pages_compressed_limit) {
2612 lck_mtx_unlock_always(c_list_lock);
2613
39236c6e
A		 2614 return (NULL);
2615 }
2616 c_segments_busy = TRUE;
2617 lck_mtx_unlock_always(c_list_lock);
2618
39037602 2619 kernel_memory_populate(compressor_map, (vm_offset_t)c_segments_next_page,
3e170ce0 2620 PAGE_SIZE, KMA_KOBJECT, VM_KERN_MEMORY_COMPRESSOR);
39236c6e
A		 2621 c_segments_next_page += PAGE_SIZE;
2622
3e170ce0
A		 2623 c_segments_available_new = c_segments_available + C_SEGMENTS_PER_PAGE;
2624
2625 if (c_segments_available_new > c_segments_limit)
2626 c_segments_available_new = c_segments_limit;
2627
2628 for (c_segno = c_segments_available + 1; c_segno < c_segments_available_new; c_segno++)
39236c6e
A		 2629 c_segments[c_segno - 1].c_segno = c_segno;
2630
2631 lck_mtx_lock_spin_always(c_list_lock);
2632
2633 c_segments[c_segno - 1].c_segno = c_free_segno_head;
2634 c_free_segno_head = c_segments_available;
3e170ce0 2635 c_segments_available = c_segments_available_new;
39236c6e
A		 2636
2637 c_segments_busy = FALSE;
2638 thread_wakeup((event_t) (&c_segments_busy));
2639 }
2640 c_segno = c_free_segno_head;
3e170ce0
A		 2641 assert(c_segno >= 0 && c_segno < c_segments_limit);
2642
39037602 2643 c_free_segno_head = (uint32_t)c_segments[c_segno].c_segno;
39236c6e 2644
3e170ce0
A		 2645 /*
2646 * do the rest of the bookkeeping now while we're still behind
2647 * the list lock and grab our generation id now into a local
2648 * so that we can install it once we have the c_seg allocated
2649 */
2650 c_segment_count++;
2651 if (c_segment_count > c_segment_count_max)
2652 c_segment_count_max = c_segment_count;
2653
39236c6e
A		 2654 lck_mtx_unlock_always(c_list_lock);
2655
2656 c_seg = (c_segment_t)zalloc(compressor_segment_zone);
2657 bzero((char *)c_seg, sizeof(struct c_segment));
2658
3e170ce0 2659 c_seg->c_store.c_buffer = (int32_t *)C_SEG_BUFFER_ADDRESS(c_segno);
39236c6e 2660
39236c6e 2661 lck_mtx_init(&c_seg->c_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
39236c6e 2662
3e170ce0
A		 2663 c_seg->c_state = C_IS_EMPTY;
2664 c_seg->c_firstemptyslot = C_SLOT_MAX_INDEX;
39236c6e 2665 c_seg->c_mysegno = c_segno;
39236c6e
A		 2666
2667 lck_mtx_lock_spin_always(c_list_lock);
3e170ce0
A		 2668 c_empty_count++;
2669 c_seg_switch_state(c_seg, C_IS_FILLING, FALSE);
39236c6e 2670 c_segments[c_segno].c_seg = c_seg;
39037602 2671 assert(c_segments[c_segno].c_segno > c_segments_available);
39236c6e
A		 2672 lck_mtx_unlock_always(c_list_lock);
2673
39236c6e 2674 *current_chead = c_seg;
39037602
A		 2675
2676#if DEVELOPMENT || DEBUG
2677 C_SEG_MAKE_WRITEABLE(c_seg);
2678#endif
2679
39236c6e 2680 }
3e170ce0
A		 2681 c_seg_alloc_nextslot(c_seg);
2682
2683 size_to_populate = C_SEG_ALLOCSIZE - C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset);
2684
2685 if (size_to_populate) {
2686
2687 min_needed = PAGE_SIZE + (C_SEG_ALLOCSIZE - C_SEG_BUFSIZE);
39236c6e 2688
3e170ce0 2689 if (C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset - c_seg->c_nextoffset) < (unsigned) min_needed) {
39236c6e 2690
3e170ce0
A		 2691 if (size_to_populate > C_SEG_MAX_POPULATE_SIZE)
2692 size_to_populate = C_SEG_MAX_POPULATE_SIZE;
39236c6e 2693
39037602 2694 kernel_memory_populate(compressor_map,
3e170ce0
A		 2695 (vm_offset_t) &c_seg->c_store.c_buffer[c_seg->c_populated_offset],
2696 size_to_populate,
2697 KMA_COMPRESSOR,
2698 VM_KERN_MEMORY_COMPRESSOR);
2699 } else
2700 size_to_populate = 0;
39236c6e 2701 }
39236c6e
A		 2702 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2703
2704 lck_mtx_lock_spin_always(&c_seg->c_lock);
2705
3e170ce0
A		 2706 if (size_to_populate)
2707 c_seg->c_populated_offset += C_SEG_BYTES_TO_OFFSET(size_to_populate);
2708
39236c6e
A		 2709 return (c_seg);
2710}
2711
2712
39236c6e
A		 2713static void
2714c_current_seg_filled(c_segment_t c_seg, c_segment_t *current_chead)
2715{
2716 uint32_t unused_bytes;
2717 uint32_t offset_to_depopulate;
3e170ce0
A		 2718 int new_state = C_ON_AGE_Q;
2719 clock_sec_t sec;
2720 clock_nsec_t nsec;
39236c6e
A		 2721
2722 unused_bytes = trunc_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset - c_seg->c_nextoffset));
2723
2724 if (unused_bytes) {
2725
2726 offset_to_depopulate = C_SEG_BYTES_TO_OFFSET(round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_nextoffset)));
2727
2728 /*
2729 * release the extra physical page(s) at the end of the segment
2730 */
2731 lck_mtx_unlock_always(&c_seg->c_lock);
2732
2733 kernel_memory_depopulate(
39037602 2734 compressor_map,
39236c6e
A		 2735 (vm_offset_t) &c_seg->c_store.c_buffer[offset_to_depopulate],
2736 unused_bytes,
2737 KMA_COMPRESSOR);
2738
2739 lck_mtx_lock_spin_always(&c_seg->c_lock);
2740
2741 c_seg->c_populated_offset = offset_to_depopulate;
2742 }
3e170ce0 2743 assert(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset) <= C_SEG_BUFSIZE);
39236c6e 2744
39037602
A		 2745#if DEVELOPMENT || DEBUG
2746 {
2747 boolean_t c_seg_was_busy = FALSE;
2748
2749 if ( !c_seg->c_busy)
2750 C_SEG_BUSY(c_seg);
2751 else
2752 c_seg_was_busy = TRUE;
2753
2754 lck_mtx_unlock_always(&c_seg->c_lock);
2755
2756 C_SEG_WRITE_PROTECT(c_seg);
2757
2758 lck_mtx_lock_spin_always(&c_seg->c_lock);
2759
2760 if (c_seg_was_busy == FALSE)
2761 C_SEG_WAKEUP_DONE(c_seg);
2762 }
2763#endif
2764
3e170ce0 2765#if CONFIG_FREEZE
39037602
A		 2766 if (current_chead == (c_segment_t*)&freezer_chead &&
2767 VM_CONFIG_SWAP_IS_PRESENT &&
2768 VM_CONFIG_FREEZER_SWAP_IS_ACTIVE &&
3e170ce0
A		 2769 c_freezer_swapout_count < VM_MAX_FREEZER_CSEG_SWAP_COUNT) {
2770 new_state = C_ON_SWAPOUT_Q;
2771 }
2772#endif /* CONFIG_FREEZE */
39236c6e 2773
3e170ce0
A		 2774 clock_get_system_nanotime(&sec, &nsec);
2775 c_seg->c_creation_ts = (uint32_t)sec;
2776
2777 lck_mtx_lock_spin_always(c_list_lock);
2778
2779#if CONFIG_FREEZE
2780 if (c_seg->c_state == C_ON_SWAPOUT_Q)
2781 c_freezer_swapout_count++;
2782#endif /* CONFIG_FREEZE */
2783
2784 c_seg->c_generation_id = c_generation_id++;
2785 c_seg_switch_state(c_seg, new_state, FALSE);
2786
39037602
A		 2787 if (c_seg->c_state == C_ON_AGE_Q && C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE)
2788 c_seg_need_delayed_compaction(c_seg, TRUE);
2789
3e170ce0
A		 2790 lck_mtx_unlock_always(c_list_lock);
2791
2792#if CONFIG_FREEZE
2793 if (c_seg->c_state == C_ON_SWAPOUT_Q)
2794 thread_wakeup((event_t)&c_swapout_list_head);
2795#endif /* CONFIG_FREEZE */
2796
39236c6e
A		 2797 *current_chead = NULL;
2798}
2799
39037602 2800
39236c6e
A		 2801/*
2802 * returns with c_seg locked
2803 */
2804void
39037602 2805c_seg_swapin_requeue(c_segment_t c_seg, boolean_t has_data, boolean_t minor_compact_ok, boolean_t age_on_swapin_q)
39236c6e
A		 2806{
2807 clock_sec_t sec;
2808 clock_nsec_t nsec;
2809
2810 clock_get_system_nanotime(&sec, &nsec);
2811
2812 lck_mtx_lock_spin_always(c_list_lock);
2813 lck_mtx_lock_spin_always(&c_seg->c_lock);
2814
39037602
A		 2815 assert(c_seg->c_busy_swapping);
2816 assert(c_seg->c_busy);
2817
3e170ce0
A		 2818 c_seg->c_busy_swapping = 0;
2819
2820 if (c_seg->c_overage_swap == TRUE) {
2821 c_overage_swapped_count--;
2822 c_seg->c_overage_swap = FALSE;
2823 }
2824 if (has_data == TRUE) {
39037602
A		 2825 if (age_on_swapin_q == TRUE)
2826 c_seg_switch_state(c_seg, C_ON_SWAPPEDIN_Q, FALSE);
2827 else
2828 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
2829
2830 if (minor_compact_ok == TRUE && !c_seg->c_on_minorcompact_q && C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE)
2831 c_seg_need_delayed_compaction(c_seg, TRUE);
39236c6e 2832 } else {
3e170ce0
A		 2833 c_seg->c_store.c_buffer = (int32_t*) NULL;
2834 c_seg->c_populated_offset = C_SEG_BYTES_TO_OFFSET(0);
39236c6e 2835
3e170ce0 2836 c_seg_switch_state(c_seg, C_ON_BAD_Q, FALSE);
39236c6e 2837 }
39236c6e 2838 c_seg->c_swappedin_ts = (uint32_t)sec;
39236c6e
A		 2839
2840 lck_mtx_unlock_always(c_list_lock);
2841}
2842
2843
2844
2845/*
39037602 2846 * c_seg has to be locked and is returned locked if the c_seg isn't freed
39236c6e 2847 * PAGE_REPLACMENT_DISALLOWED has to be TRUE on entry and is returned TRUE
39037602 2848 * c_seg_swapin returns 1 if the c_seg was freed, 0 otherwise
39236c6e
A		 2849 */
2850
39037602
A		 2851int
2852c_seg_swapin(c_segment_t c_seg, boolean_t force_minor_compaction, boolean_t age_on_swapin_q)
39236c6e
A		 2853{
2854 vm_offset_t addr = 0;
2855 uint32_t io_size = 0;
2856 uint64_t f_offset;
2857
3e170ce0
A		 2858 assert(C_SEG_IS_ONDISK(c_seg));
2859
39236c6e 2860#if !CHECKSUM_THE_SWAP
3e170ce0 2861 c_seg_trim_tail(c_seg);
39236c6e
A		 2862#endif
2863 io_size = round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset));
2864 f_offset = c_seg->c_store.c_swap_handle;
fe8ab488
A		 2865
2866 C_SEG_BUSY(c_seg);
3e170ce0 2867 c_seg->c_busy_swapping = 1;
ecc0ceb4
A		 2868
2869 /*
2870 * This thread is likely going to block for I/O.
2871 * Make sure it is ready to run when the I/O completes because
2872 * it needs to clear the busy bit on the c_seg so that other
2873 * waiting threads can make progress too. To do that, boost
2874 * the rwlock_count so that the priority is boosted.
2875 */
2876 set_thread_rwlock_boost();
39236c6e 2877 lck_mtx_unlock_always(&c_seg->c_lock);
39236c6e 2878
3e170ce0 2879 PAGE_REPLACEMENT_DISALLOWED(FALSE);
39236c6e 2880
3e170ce0 2881 addr = (vm_offset_t)C_SEG_BUFFER_ADDRESS(c_seg->c_mysegno);
39037602 2882 c_seg->c_store.c_buffer = (int32_t*) addr;
39236c6e 2883
39037602 2884 kernel_memory_populate(compressor_map, addr, io_size, KMA_COMPRESSOR, VM_KERN_MEMORY_COMPRESSOR);
39236c6e 2885
39037602 2886 if (vm_swap_get(c_seg, f_offset, io_size) != KERN_SUCCESS) {
3e170ce0 2887 PAGE_REPLACEMENT_DISALLOWED(TRUE);
39236c6e 2888
39037602 2889 c_seg->c_store.c_swap_handle = f_offset;
39236c6e 2890
39037602
A		 2891 kernel_memory_depopulate(compressor_map, addr, io_size, KMA_COMPRESSOR);
2892
2893 c_seg_swapin_requeue(c_seg, FALSE, TRUE, age_on_swapin_q);
3e170ce0
A		 2894 } else {
2895 c_seg->c_store.c_buffer = (int32_t*) addr;
fe8ab488 2896#if ENCRYPTED_SWAP
3e170ce0 2897 vm_swap_decrypt(c_seg);
fe8ab488 2898#endif /* ENCRYPTED_SWAP */
39236c6e
A		 2899
2900#if CHECKSUM_THE_SWAP
3e170ce0
A		 2901 if (c_seg->cseg_swap_size != io_size)
2902 panic("swapin size doesn't match swapout size");
39236c6e 2903
3e170ce0
A		 2904 if (c_seg->cseg_hash != hash_string((char*) c_seg->c_store.c_buffer, (int)io_size)) {
2905 panic("c_seg_swapin - Swap hash mismatch\n");
2906 }
39236c6e
A		 2907#endif /* CHECKSUM_THE_SWAP */
2908
3e170ce0 2909 PAGE_REPLACEMENT_DISALLOWED(TRUE);
39236c6e 2910
39037602
A		 2911 c_seg_swapin_requeue(c_seg, TRUE, force_minor_compaction == TRUE ? FALSE : TRUE, age_on_swapin_q);
2912
2913 OSAddAtomic64(c_seg->c_bytes_used, &compressor_bytes_used);
2914
3e170ce0 2915 if (force_minor_compaction == TRUE) {
39037602
A		 2916 if (c_seg_minor_compaction_and_unlock(c_seg, FALSE)) {
2917 /*
2918 * Drop the rwlock_count so that the thread priority
2919 * is returned back to where it is supposed to be.
2920 */
2921 clear_thread_rwlock_boost();
2922 return (1);
2923 }
2924
3e170ce0 2925 lck_mtx_lock_spin_always(&c_seg->c_lock);
3e170ce0 2926 }
3e170ce0
A		 2927 }
2928 C_SEG_WAKEUP_DONE(c_seg);
ecc0ceb4
A		 2929
2930 /*
2931 * Drop the rwlock_count so that the thread priority
2932 * is returned back to where it is supposed to be.
2933 */
2934 clear_thread_rwlock_boost();
39037602
A		 2935
2936 return (0);
3e170ce0
A		 2937}
2938
2939
2940static void
2941c_segment_sv_hash_drop_ref(int hash_indx)
2942{
2943 struct c_sv_hash_entry o_sv_he, n_sv_he;
2944
2945 while (1) {
2946
2947 o_sv_he.he_record = c_segment_sv_hash_table[hash_indx].he_record;
2948
2949 n_sv_he.he_ref = o_sv_he.he_ref - 1;
2950 n_sv_he.he_data = o_sv_he.he_data;
2951
2952 if (OSCompareAndSwap64((UInt64)o_sv_he.he_record, (UInt64)n_sv_he.he_record, (UInt64 *) &c_segment_sv_hash_table[hash_indx].he_record) == TRUE) {
2953 if (n_sv_he.he_ref == 0)
2954 OSAddAtomic(-1, &c_segment_svp_in_hash);
2955 break;
2956 }
2957 }
2958}
2959
2960
2961static int
2962c_segment_sv_hash_insert(uint32_t data)
2963{
2964 int hash_sindx;
2965 int misses;
2966 struct c_sv_hash_entry o_sv_he, n_sv_he;
2967 boolean_t got_ref = FALSE;
2968
2969 if (data == 0)
2970 OSAddAtomic(1, &c_segment_svp_zero_compressions);
2971 else
2972 OSAddAtomic(1, &c_segment_svp_nonzero_compressions);
2973
2974 hash_sindx = data & C_SV_HASH_MASK;
2975
2976 for (misses = 0; misses < C_SV_HASH_MAX_MISS; misses++)
2977 {
2978 o_sv_he.he_record = c_segment_sv_hash_table[hash_sindx].he_record;
2979
2980 while (o_sv_he.he_data == data || o_sv_he.he_ref == 0) {
2981 n_sv_he.he_ref = o_sv_he.he_ref + 1;
2982 n_sv_he.he_data = data;
2983
2984 if (OSCompareAndSwap64((UInt64)o_sv_he.he_record, (UInt64)n_sv_he.he_record, (UInt64 *) &c_segment_sv_hash_table[hash_sindx].he_record) == TRUE) {
2985 if (n_sv_he.he_ref == 1)
2986 OSAddAtomic(1, &c_segment_svp_in_hash);
2987 got_ref = TRUE;
2988 break;
39236c6e 2989 }
3e170ce0 2990 o_sv_he.he_record = c_segment_sv_hash_table[hash_sindx].he_record;
39236c6e 2991 }
3e170ce0
A		 2992 if (got_ref == TRUE)
2993 break;
2994 hash_sindx++;
2995
2996 if (hash_sindx == C_SV_HASH_SIZE)
2997 hash_sindx = 0;
39236c6e 2998 }
3e170ce0
A		 2999 if (got_ref == FALSE)
3000 return(-1);
39236c6e 3001
3e170ce0
A		 3002 return (hash_sindx);
3003}
3004
3005
3006#if RECORD_THE_COMPRESSED_DATA
3007
3008static void
3009c_compressed_record_data(char *src, int c_size)
3010{
3011 if ((c_compressed_record_cptr + c_size + 4) >= c_compressed_record_ebuf)
3012 panic("c_compressed_record_cptr >= c_compressed_record_ebuf");
3013
3014 *(int *)((void *)c_compressed_record_cptr) = c_size;
3015
3016 c_compressed_record_cptr += 4;
3017
3018 memcpy(c_compressed_record_cptr, src, c_size);
3019 c_compressed_record_cptr += c_size;
39236c6e 3020}
3e170ce0 3021#endif
39236c6e
A		 3022
3023
3024static int
3025c_compress_page(char *src, c_slot_mapping_t slot_ptr, c_segment_t *current_chead, char *scratch_buf)
3026{
3027 int c_size;
04b8595b 3028 int c_rounded_size = 0;
39236c6e
A		 3029 int max_csize;
3030 c_slot_t cs;
3031 c_segment_t c_seg;
3032
3033 KERNEL_DEBUG(0xe0400000 | DBG_FUNC_START, *current_chead, 0, 0, 0, 0);
3034retry:
3035 if ((c_seg = c_seg_allocate(current_chead)) == NULL)
3036 return (1);
3037 /*
3038 * returns with c_seg lock held
3e170ce0
A		 3039 * and PAGE_REPLACEMENT_DISALLOWED(TRUE)...
3040 * c_nextslot has been allocated and
3041 * c_store.c_buffer populated
39236c6e 3042 */
3e170ce0
A		 3043 assert(c_seg->c_state == C_IS_FILLING);
3044
39236c6e
A		 3045 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_seg->c_nextslot);
3046
3047 cs->c_packed_ptr = C_SLOT_PACK_PTR(slot_ptr);
fe8ab488
A		 3048 assert(slot_ptr == (c_slot_mapping_t)C_SLOT_UNPACK_PTR(cs));
3049
39236c6e
A		 3050 cs->c_offset = c_seg->c_nextoffset;
3051
3052 max_csize = C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES((int32_t)cs->c_offset);
3053
3054 if (max_csize > PAGE_SIZE)
3055 max_csize = PAGE_SIZE;
3056
39236c6e
A		 3057#if CHECKSUM_THE_DATA
3058 cs->c_hash_data = hash_string(src, PAGE_SIZE);
3059#endif
39236c6e 3060
39037602
A		 3061 if (vm_compressor_algorithm() != VM_COMPRESSOR_DEFAULT_CODEC) {
3062 } else {
3e170ce0 3063 c_size = WKdm_compress_new((const WK_word *)(uintptr_t)src, (WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
fe8ab488 3064 (WK_word *)(uintptr_t)scratch_buf, max_csize - 4);
39037602 3065 }
39236c6e
A		 3066 assert(c_size <= (max_csize - 4) && c_size >= -1);
3067
3068 if (c_size == -1) {
3069
3070 if (max_csize < PAGE_SIZE) {
3071 c_current_seg_filled(c_seg, current_chead);
3e170ce0 3072 assert(*current_chead == NULL);
39236c6e 3073
3e170ce0 3074 lck_mtx_unlock_always(&c_seg->c_lock);
39236c6e 3075
3e170ce0 3076 PAGE_REPLACEMENT_DISALLOWED(FALSE);
39236c6e
A		 3077 goto retry;
3078 }
3079 c_size = PAGE_SIZE;
3080
3081 memcpy(&c_seg->c_store.c_buffer[cs->c_offset], src, c_size);
3e170ce0
A		 3082
3083 OSAddAtomic(1, &c_segment_noncompressible_pages);
3084
3085 } else if (c_size == 0) {
3086 int hash_index;
3087
3088 /*
3089 * special case - this is a page completely full of a single 32 bit value
3090 */
3091 hash_index = c_segment_sv_hash_insert(*(uint32_t *)(uintptr_t)src);
3092
3093 if (hash_index != -1) {
3094 slot_ptr->s_cindx = hash_index;
3095 slot_ptr->s_cseg = C_SV_CSEG_ID;
3096
3097 OSAddAtomic(1, &c_segment_svp_hash_succeeded);
3098#if RECORD_THE_COMPRESSED_DATA
3099 c_compressed_record_data(src, 4);
3100#endif
3101 goto sv_compression;
3102 }
3103 c_size = 4;
3104
3105 memcpy(&c_seg->c_store.c_buffer[cs->c_offset], src, c_size);
3106
3107 OSAddAtomic(1, &c_segment_svp_hash_failed);
39236c6e 3108 }
3e170ce0
A		 3109
3110#if RECORD_THE_COMPRESSED_DATA
3111 c_compressed_record_data((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size);
3112#endif
3113
39236c6e
A		 3114#if CHECKSUM_THE_COMPRESSED_DATA
3115 cs->c_hash_compressed_data = hash_string((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size);
3116#endif
3117 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
3118
3119 PACK_C_SIZE(cs, c_size);
3120 c_seg->c_bytes_used += c_rounded_size;
3121 c_seg->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
3122
3123 slot_ptr->s_cindx = c_seg->c_nextslot++;
3124 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
3125 slot_ptr->s_cseg = c_seg->c_mysegno + 1;
3126
3e170ce0
A		 3127sv_compression:
3128 if (c_seg->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg->c_nextslot >= C_SLOT_MAX_INDEX) {
39236c6e 3129 c_current_seg_filled(c_seg, current_chead);
3e170ce0
A		 3130 assert(*current_chead == NULL);
3131 }
3132 lck_mtx_unlock_always(&c_seg->c_lock);
39236c6e
A		 3133
3134 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3135
3e170ce0
A		 3136#if RECORD_THE_COMPRESSED_DATA
3137 if ((c_compressed_record_cptr - c_compressed_record_sbuf) >= C_SEG_ALLOCSIZE) {
3138 c_compressed_record_write(c_compressed_record_sbuf, (int)(c_compressed_record_cptr - c_compressed_record_sbuf));
3139 c_compressed_record_cptr = c_compressed_record_sbuf;
3140 }
3141#endif
3142 if (c_size) {
3143 OSAddAtomic64(c_size, &c_segment_compressed_bytes);
3144 OSAddAtomic64(c_rounded_size, &compressor_bytes_used);
3145 }
39236c6e 3146 OSAddAtomic64(PAGE_SIZE, &c_segment_input_bytes);
39236c6e
A		 3147
3148 OSAddAtomic(1, &c_segment_pages_compressed);
3149 OSAddAtomic(1, &sample_period_compression_count);
3150
3151 KERNEL_DEBUG(0xe0400000 | DBG_FUNC_END, *current_chead, c_size, c_segment_input_bytes, c_segment_compressed_bytes, 0);
3152
39236c6e
A		 3153 return (0);
3154}
3155
39037602
A		 3156static inline void sv_decompress(int32_t *ddst, int32_t pattern) {
3157#if __x86_64__
3158 memset_word(ddst, pattern, PAGE_SIZE / sizeof(int32_t));
3159#else
3160 size_t i;
3161
3162 /* Unroll the pattern fill loop 4x to encourage the
3163 * compiler to emit NEON stores, cf.
3164 * <rdar://problem/25839866> Loop autovectorization
3165 * anomalies.
3166 * We use separate loops for each PAGE_SIZE
3167 * to allow the autovectorizer to engage, as PAGE_SIZE
3168 * is currently not a constant.
3169 */
3170
3171 if (PAGE_SIZE == 4096) {
3172 for (i = 0; i < (4096U / sizeof(int32_t)); i += 4) {
3173 *ddst++ = pattern;
3174 *ddst++ = pattern;
3175 *ddst++ = pattern;
3176 *ddst++ = pattern;
3177 }
3178 } else {
3179 assert(PAGE_SIZE == 16384);
3180 for (i = 0; i < (int)(16384U / sizeof(int32_t)); i += 4) {
3181 *ddst++ = pattern;
3182 *ddst++ = pattern;
3183 *ddst++ = pattern;
3184 *ddst++ = pattern;
3185 }
3186 }
3187#endif
3188}
39236c6e
A		 3189
3190static int
3191c_decompress_page(char *dst, volatile c_slot_mapping_t slot_ptr, int flags, int *zeroslot)
3192{
3193 c_slot_t cs;
3194 c_segment_t c_seg;
39037602 3195 uint32_t c_segno;
39236c6e
A		 3196 int c_indx;
3197 int c_rounded_size;
3198 uint32_t c_size;
3199 int retval = 0;
39236c6e
A		 3200 boolean_t need_unlock = TRUE;
3201 boolean_t consider_defragmenting = FALSE;
3e170ce0
A		 3202 boolean_t kdp_mode = FALSE;
3203
39037602 3204 if (__improbable(flags & C_KDP)) {
3e170ce0
A		 3205 if (not_in_kdp) {
3206 panic("C_KDP passed to decompress page from outside of debugger context");
3207 }
3208
3209 assert((flags & C_KEEP) == C_KEEP);
3210 assert((flags & C_DONT_BLOCK) == C_DONT_BLOCK);
3211
3212 if ((flags & (C_DONT_BLOCK | C_KEEP)) != (C_DONT_BLOCK | C_KEEP)) {
3213 return (-2);
3214 }
3215
3216 kdp_mode = TRUE;
39037602 3217 *zeroslot = 0;
3e170ce0 3218 }
39236c6e
A		 3219
3220ReTry:
39037602 3221 if (__probable(!kdp_mode)) {
3e170ce0
A		 3222 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3223 } else {
3224 if (kdp_lck_rw_lock_is_acquired_exclusive(&c_master_lock)) {
3225 return (-2);
3226 }
3227 }
fe8ab488 3228
39236c6e 3229#if HIBERNATION
fe8ab488
A		 3230 /*
3231 * if hibernation is enabled, it indicates (via a call
3232 * to 'vm_decompressor_lock' that no further
3233 * decompressions are allowed once it reaches
3234 * the point of flushing all of the currently dirty
3235 * anonymous memory through the compressor and out
3236 * to disk... in this state we allow freeing of compressed
3237 * pages and must honor the C_DONT_BLOCK case
3238 */
3239 if (dst && decompressions_blocked == TRUE) {
3240 if (flags & C_DONT_BLOCK) {
3241
39037602 3242 if (__probable(!kdp_mode)) {
3e170ce0
A		 3243 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3244 }
fe8ab488
A		 3245
3246 *zeroslot = 0;
3247 return (-2);
39236c6e 3248 }
fe8ab488
A		 3249 /*
3250 * it's safe to atomically assert and block behind the
3251 * lock held in shared mode because "decompressions_blocked" is
3252 * only set and cleared and the thread_wakeup done when the lock
3253 * is held exclusively
3254 */
3255 assert_wait((event_t)&decompressions_blocked, THREAD_UNINT);
3256
3257 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3258
3259 thread_block(THREAD_CONTINUE_NULL);
3260
3261 goto ReTry;
39236c6e
A		 3262 }
3263#endif
39236c6e 3264 /* s_cseg is actually "segno+1" */
39037602
A		 3265 c_segno = slot_ptr->s_cseg - 1;
3266
3267 if (__improbable(c_segno >= c_segments_available))
3268 panic("c_decompress_page: c_segno %d >= c_segments_available %d, slot_ptr(%p), slot_data(%x)",
3269 c_segno, c_segments_available, slot_ptr, *(int *)((void *)slot_ptr));
3270
3271 if (__improbable(c_segments[c_segno].c_segno < c_segments_available))
3272 panic("c_decompress_page: c_segno %d is free, slot_ptr(%p), slot_data(%x)",
3273 c_segno, slot_ptr, *(int *)((void *)slot_ptr));
39236c6e 3274
39037602
A		 3275 c_seg = c_segments[c_segno].c_seg;
3276
3277 if (__probable(!kdp_mode)) {
3e170ce0
A		 3278 lck_mtx_lock_spin_always(&c_seg->c_lock);
3279 } else {
3280 if (kdp_lck_mtx_lock_spin_is_acquired(&c_seg->c_lock)) {
3281 return (-2);
3282 }
3283 }
39236c6e 3284
3e170ce0 3285 assert(c_seg->c_state != C_IS_EMPTY && c_seg->c_state != C_IS_FREE);
39236c6e 3286
39037602
A		 3287 if (dst == NULL && c_seg->c_busy_swapping) {
3288 assert(c_seg->c_busy);
3289
3290 goto bypass_busy_check;
3291 }
3e170ce0
A		 3292 if (flags & C_DONT_BLOCK) {
3293 if (c_seg->c_busy || (C_SEG_IS_ONDISK(c_seg) && dst)) {
39236c6e
A		 3294 *zeroslot = 0;
3295
3e170ce0 3296 retval = -2;
39236c6e
A		 3297 goto done;
3298 }
3299 }
3300 if (c_seg->c_busy) {
3301
3302 PAGE_REPLACEMENT_DISALLOWED(FALSE);
fe8ab488 3303
39236c6e
A		 3304 c_seg_wait_on_busy(c_seg);
3305
3306 goto ReTry;
3307 }
39037602
A		 3308bypass_busy_check:
3309
39236c6e
A		 3310 c_indx = slot_ptr->s_cindx;
3311
39037602
A		 3312 if (__improbable(c_indx >= c_seg->c_nextslot))
3313 panic("c_decompress_page: c_indx %d >= c_nextslot %d, c_seg(%p), slot_ptr(%p), slot_data(%x)",
3314 c_indx, c_seg->c_nextslot, c_seg, slot_ptr, *(int *)((void *)slot_ptr));
3315
39236c6e
A		 3316 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
3317
3318 c_size = UNPACK_C_SIZE(cs);
3319
39037602
A		 3320 if (__improbable(c_size == 0))
3321 panic("c_decompress_page: c_size == 0, c_seg(%p), slot_ptr(%p), slot_data(%x)",
3322 c_seg, slot_ptr, *(int *)((void *)slot_ptr));
3323
39236c6e
A		 3324 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
3325
3326 if (dst) {
3327 uint32_t age_of_cseg;
3328 clock_sec_t cur_ts_sec;
3329 clock_nsec_t cur_ts_nsec;
3330
3e170ce0
A		 3331 if (C_SEG_IS_ONDISK(c_seg)) {
3332 assert(kdp_mode == FALSE);
39037602
A		 3333 retval = c_seg_swapin(c_seg, FALSE, TRUE);
3334 assert(retval == 0);
3e170ce0
A		 3335
3336 retval = 1;
39236c6e 3337 }
3e170ce0
A		 3338 if (c_seg->c_state == C_ON_BAD_Q) {
3339 assert(c_seg->c_store.c_buffer == NULL);
3340
3341 retval = -1;
39236c6e
A		 3342 goto c_seg_invalid_data;
3343 }
3344#if CHECKSUM_THE_COMPRESSED_DATA
3345 if (cs->c_hash_compressed_data != hash_string((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size))
39037602 3346 panic("compressed data doesn't match original hash: 0x%x, seg: %p, offset: %d, c_size: %d", cs->c_hash_compressed_data, c_seg, cs->c_offset, c_size);
39236c6e
A		 3347#endif
3348 if (c_rounded_size == PAGE_SIZE) {
3349 /*
3350 * page wasn't compressible... just copy it out
3351 */
3352 memcpy(dst, &c_seg->c_store.c_buffer[cs->c_offset], PAGE_SIZE);
3e170ce0
A		 3353 } else if (c_size == 4) {
3354 int32_t data;
3355 int32_t *dptr;
3356
3357 /*
3358 * page was populated with a single value
3359 * that didn't fit into our fast hash
3360 * so we packed it in as a single non-compressed value
3361 * that we need to populate the page with
3362 */
3363 dptr = (int32_t *)(uintptr_t)dst;
3364 data = *(int32_t *)(&c_seg->c_store.c_buffer[cs->c_offset]);
39037602 3365 sv_decompress(dptr, data);
39236c6e
A		 3366 } else {
3367 uint32_t my_cpu_no;
3368 char *scratch_buf;
3369
39037602 3370 if (__probable(!kdp_mode)) {
3e170ce0
A		 3371 /*
3372 * we're behind the c_seg lock held in spin mode
3373 * which means pre-emption is disabled... therefore
3374 * the following sequence is atomic and safe
3375 */
3376 my_cpu_no = cpu_number();
39236c6e 3377
3e170ce0 3378 assert(my_cpu_no < compressor_cpus);
39236c6e 3379
39037602 3380 scratch_buf = &compressor_scratch_bufs[my_cpu_no * vm_compressor_get_decode_scratch_size()];
3e170ce0
A		 3381 } else {
3382 scratch_buf = kdp_compressor_scratch_buf;
3383 }
39037602
A		 3384
3385 if (vm_compressor_algorithm() != VM_COMPRESSOR_DEFAULT_CODEC) {
3386 } else {
39236c6e
A		 3387 WKdm_decompress_new((WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
3388 (WK_word *)(uintptr_t)dst, (WK_word *)(uintptr_t)scratch_buf, c_size);
39037602 3389 }
39236c6e
A		 3390 }
3391
3392#if CHECKSUM_THE_DATA
3393 if (cs->c_hash_data != hash_string(dst, PAGE_SIZE))
39037602
A		 3394 panic("decompressed data doesn't match original cs: %p, hash: %d, offset: %d, c_size: %d", cs, cs->c_hash_data, cs->c_offset, c_size);
3395
39236c6e 3396#endif
3e170ce0 3397 if (c_seg->c_swappedin_ts == 0 && !kdp_mode) {
39236c6e
A		 3398
3399 clock_get_system_nanotime(&cur_ts_sec, &cur_ts_nsec);
3400
3401 age_of_cseg = (uint32_t)cur_ts_sec - c_seg->c_creation_ts;
39236c6e
A		 3402 if (age_of_cseg < DECOMPRESSION_SAMPLE_MAX_AGE)
3403 OSAddAtomic(1, &age_of_decompressions_during_sample_period[age_of_cseg]);
3404 else
3405 OSAddAtomic(1, &overage_decompressions_during_sample_period);
3406
3407 OSAddAtomic(1, &sample_period_decompression_count);
3408 }
39236c6e
A		 3409 }
3410c_seg_invalid_data:
3411
39236c6e
A		 3412 if (flags & C_KEEP) {
3413 *zeroslot = 0;
3414 goto done;
3415 }
3e170ce0 3416 assert(kdp_mode == FALSE);
39037602 3417
39236c6e
A		 3418 c_seg->c_bytes_unused += c_rounded_size;
3419 c_seg->c_bytes_used -= c_rounded_size;
3420 PACK_C_SIZE(cs, 0);
3421
3422 if (c_indx < c_seg->c_firstemptyslot)
3423 c_seg->c_firstemptyslot = c_indx;
3424
3425 OSAddAtomic(-1, &c_segment_pages_compressed);
3426
3e170ce0 3427 if (c_seg->c_state != C_ON_BAD_Q && !(C_SEG_IS_ONDISK(c_seg))) {
39236c6e 3428 /*
3e170ce0 3429 * C_SEG_IS_ONDISK == TRUE can occur when we're doing a
39236c6e
A		 3430 * free of a compressed page (i.e. dst == NULL)
3431 */
3432 OSAddAtomic64(-c_rounded_size, &compressor_bytes_used);
3433 }
39037602
A		 3434 if (c_seg->c_busy_swapping) {
3435 /*
3436 * bypass case for c_busy_swapping...
3437 * let the swapin/swapout paths deal with putting
3438 * the c_seg on the minor compaction queue if needed
3439 */
3440 assert(c_seg->c_busy);
3441 goto done;
3442 }
3443 assert(!c_seg->c_busy);
3444
3e170ce0 3445 if (c_seg->c_state != C_IS_FILLING) {
39236c6e 3446 if (c_seg->c_bytes_used == 0) {
3e170ce0 3447 if ( !(C_SEG_IS_ONDISK(c_seg))) {
8a3053a0
A		 3448 int pages_populated;
3449
3450 pages_populated = (round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE;
3451 c_seg->c_populated_offset = C_SEG_BYTES_TO_OFFSET(0);
3452
3453 if (pages_populated) {
3e170ce0
A		 3454
3455 assert(c_seg->c_state != C_ON_BAD_Q);
8a3053a0
A		 3456 assert(c_seg->c_store.c_buffer != NULL);
3457
fe8ab488 3458 C_SEG_BUSY(c_seg);
8a3053a0
A		 3459 lck_mtx_unlock_always(&c_seg->c_lock);
3460
39037602 3461 kernel_memory_depopulate(compressor_map, (vm_offset_t) c_seg->c_store.c_buffer, pages_populated * PAGE_SIZE, KMA_COMPRESSOR);
8a3053a0
A		 3462
3463 lck_mtx_lock_spin_always(&c_seg->c_lock);
3464 C_SEG_WAKEUP_DONE(c_seg);
3465 }
39037602
A		 3466 if (!c_seg->c_on_minorcompact_q && c_seg->c_state != C_ON_SWAPOUT_Q)
3467 c_seg_need_delayed_compaction(c_seg, FALSE);
3468 } else {
3469 if (c_seg->c_state != C_ON_SWAPPEDOUTSPARSE_Q) {
8a3053a0 3470
39037602
A		 3471 c_seg_move_to_sparse_list(c_seg);
3472 consider_defragmenting = TRUE;
3473 }
3474 }
39236c6e
A		 3475 } else if (c_seg->c_on_minorcompact_q) {
3476
3e170ce0
A		 3477 assert(c_seg->c_state != C_ON_BAD_Q);
3478
3479 if (C_SEG_SHOULD_MINORCOMPACT(c_seg)) {
39236c6e
A		 3480 c_seg_try_minor_compaction_and_unlock(c_seg);
3481 need_unlock = FALSE;
3482 }
3e170ce0 3483 } else if ( !(C_SEG_IS_ONDISK(c_seg))) {
39236c6e 3484
3e170ce0 3485 if (c_seg->c_state != C_ON_BAD_Q && c_seg->c_state != C_ON_SWAPOUT_Q && C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE) {
39037602 3486 c_seg_need_delayed_compaction(c_seg, FALSE);
39236c6e 3487 }
3e170ce0 3488 } else if (c_seg->c_state != C_ON_SWAPPEDOUTSPARSE_Q && C_SEG_ONDISK_IS_SPARSE(c_seg)) {
39236c6e
A		 3489
3490 c_seg_move_to_sparse_list(c_seg);
3491 consider_defragmenting = TRUE;
3492 }
3493 }
3494done:
39037602 3495 if (__improbable(kdp_mode)) {
3e170ce0
A		 3496 return retval;
3497 }
3498
39236c6e
A		 3499 if (need_unlock == TRUE)
3500 lck_mtx_unlock_always(&c_seg->c_lock);
3501
3502 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3503
3504 if (consider_defragmenting == TRUE)
3505 vm_swap_consider_defragmenting();
fe8ab488
A		 3506
3507
39236c6e
A		 3508 return (retval);
3509}
3510
3511
3512int
3513vm_compressor_get(ppnum_t pn, int *slot, int flags)
3514{
3e170ce0 3515 c_slot_mapping_t slot_ptr;
39236c6e
A		 3516 char *dst;
3517 int zeroslot = 1;
3518 int retval;
3519
3520#if __x86_64__
3521 dst = PHYSMAP_PTOV((uint64_t)pn << (uint64_t)PAGE_SHIFT);
3522#else
3523#error "unsupported architecture"
3524#endif
3e170ce0
A		 3525 slot_ptr = (c_slot_mapping_t)slot;
3526
3527 if (slot_ptr->s_cseg == C_SV_CSEG_ID) {
3528 int32_t data;
3529 int32_t *dptr;
3530
3531 /*
3532 * page was populated with a single value
3533 * that found a home in our hash table
3534 * grab that value from the hash and populate the page
3535 * that we need to populate the page with
3536 */
3537 dptr = (int32_t *)(uintptr_t)dst;
3538 data = c_segment_sv_hash_table[slot_ptr->s_cindx].he_data;
3539#if __x86_64__
3540 memset_word(dptr, data, PAGE_SIZE / sizeof(int32_t));
3541#else
3542 {
3543 int i;
3544
3545 for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++)
3546 *dptr++ = data;
3547 }
3548#endif
3e170ce0 3549 if ( !(flags & C_KEEP)) {
743345f9
A		 3550 c_segment_sv_hash_drop_ref(slot_ptr->s_cindx);
3551
3e170ce0
A		 3552 OSAddAtomic(-1, &c_segment_pages_compressed);
3553 *slot = 0;
3554 }
3555 if (data)
3556 OSAddAtomic(1, &c_segment_svp_nonzero_decompressions);
3557 else
3558 OSAddAtomic(1, &c_segment_svp_zero_decompressions);
3559
3560 return (0);
3561 }
3562
3563 retval = c_decompress_page(dst, slot_ptr, flags, &zeroslot);
39236c6e
A		 3564
3565 /*
3566 * zeroslot will be set to 0 by c_decompress_page if (flags & C_KEEP)
3e170ce0 3567 * or (flags & C_DONT_BLOCK) and we found 'c_busy' or 'C_SEG_IS_ONDISK' to be TRUE
39236c6e
A		 3568 */
3569 if (zeroslot) {
39236c6e
A		 3570 *slot = 0;
3571 }
3572 /*
3573 * returns 0 if we successfully decompressed a page from a segment already in memory
3574 * returns 1 if we had to first swap in the segment, before successfully decompressing the page
3575 * returns -1 if we encountered an error swapping in the segment - decompression failed
3e170ce0 3576 * returns -2 if (flags & C_DONT_BLOCK) and we found 'c_busy' or 'C_SEG_IS_ONDISK' to be true
39236c6e
A		 3577 */
3578 return (retval);
3579}
3580
3581
fe8ab488
A		 3582int
3583vm_compressor_free(int *slot, int flags)
39236c6e 3584{
3e170ce0 3585 c_slot_mapping_t slot_ptr;
39236c6e 3586 int zeroslot = 1;
fe8ab488 3587 int retval;
39236c6e 3588
fe8ab488 3589 assert(flags == 0 || flags == C_DONT_BLOCK);
39236c6e 3590
3e170ce0
A		 3591 slot_ptr = (c_slot_mapping_t)slot;
3592
3593 if (slot_ptr->s_cseg == C_SV_CSEG_ID) {
3594
3595 c_segment_sv_hash_drop_ref(slot_ptr->s_cindx);
3596 OSAddAtomic(-1, &c_segment_pages_compressed);
3597
3598 *slot = 0;
3599 return (0);
3600 }
3601 retval = c_decompress_page(NULL, slot_ptr, flags, &zeroslot);
fe8ab488
A		 3602 /*
3603 * returns 0 if we successfully freed the specified compressed page
3604 * returns -2 if (flags & C_DONT_BLOCK) and we found 'c_busy' set
3605 */
3606
3607 if (retval == 0)
3608 *slot = 0;
3e170ce0
A		 3609 else
3610 assert(retval == -2);
fe8ab488
A		 3611
3612 return (retval);
39236c6e
A		 3613}
3614
3615
3616int
3617vm_compressor_put(ppnum_t pn, int *slot, void **current_chead, char *scratch_buf)
3618{
3619 char *src;
3620 int retval;
3621
39236c6e
A		 3622#if __x86_64__
3623 src = PHYSMAP_PTOV((uint64_t)pn << (uint64_t)PAGE_SHIFT);
3624#else
3625#error "unsupported architecture"
3626#endif
3627 retval = c_compress_page(src, (c_slot_mapping_t)slot, (c_segment_t *)current_chead, scratch_buf);
3628
3629 return (retval);
3630}
fe8ab488
A		 3631
3632void
3633vm_compressor_transfer(
3634 int *dst_slot_p,
3635 int *src_slot_p)
3636{
3637 c_slot_mapping_t dst_slot, src_slot;
3638 c_segment_t c_seg;
3639 int c_indx;
3640 c_slot_t cs;
3641
fe8ab488
A		 3642 src_slot = (c_slot_mapping_t) src_slot_p;
3643
3e170ce0
A		 3644 if (src_slot->s_cseg == C_SV_CSEG_ID) {
3645 *dst_slot_p = *src_slot_p;
3646 *src_slot_p = 0;
3647 return;
3648 }
3649 dst_slot = (c_slot_mapping_t) dst_slot_p;
fe8ab488
A		 3650Retry:
3651 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3652 /* get segment for src_slot */
3653 c_seg = c_segments[src_slot->s_cseg -1].c_seg;
3654 /* lock segment */
3655 lck_mtx_lock_spin_always(&c_seg->c_lock);
3656 /* wait if it's busy */
3e170ce0 3657 if (c_seg->c_busy && !c_seg->c_busy_swapping) {
fe8ab488
A		 3658 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3659 c_seg_wait_on_busy(c_seg);
3660 goto Retry;
3661 }
3662 /* find the c_slot */
3663 c_indx = src_slot->s_cindx;
3664 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
3665 /* point the c_slot back to dst_slot instead of src_slot */
3666 cs->c_packed_ptr = C_SLOT_PACK_PTR(dst_slot);
3667 /* transfer */
3668 *dst_slot_p = *src_slot_p;
3669 *src_slot_p = 0;
3670 lck_mtx_unlock_always(&c_seg->c_lock);
3671 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3672}
3e170ce0
A		 3673
3674#if CONFIG_FREEZE
3675
3676int freezer_finished_filling = 0;
3677
3678void
3679vm_compressor_finished_filling(
3680 void **current_chead)
3681{
3682 c_segment_t c_seg;
3683
3684 if ((c_seg = *(c_segment_t *)current_chead) == NULL)
3685 return;
3686
3687 assert(c_seg->c_state == C_IS_FILLING);
3688
3689 lck_mtx_lock_spin_always(&c_seg->c_lock);
3690
3691 c_current_seg_filled(c_seg, (c_segment_t *)current_chead);
3692
3693 lck_mtx_unlock_always(&c_seg->c_lock);
3694
3695 freezer_finished_filling++;
3696}
3697
3698
3699/*
3700 * This routine is used to transfer the compressed chunks from
3701 * the c_seg/cindx pointed to by slot_p into a new c_seg headed
3702 * by the current_chead and a new cindx within that c_seg.
3703 *
3704 * Currently, this routine is only used by the "freezer backed by
3705 * compressor with swap" mode to create a series of c_segs that
3706 * only contain compressed data belonging to one task. So, we
3707 * move a task's previously compressed data into a set of new
3708 * c_segs which will also hold the task's yet to be compressed data.
3709 */
3710
3711kern_return_t
3712vm_compressor_relocate(
3713 void **current_chead,
3714 int *slot_p)
3715{
3716 c_slot_mapping_t slot_ptr;
3717 c_slot_mapping_t src_slot;
3718 uint32_t c_rounded_size;
3719 uint32_t c_size;
3720 uint16_t dst_slot;
3721 c_slot_t c_dst;
3722 c_slot_t c_src;
3723 int c_indx;
3724 c_segment_t c_seg_dst = NULL;
3725 c_segment_t c_seg_src = NULL;
3726 kern_return_t kr = KERN_SUCCESS;
3727
3728
3729 src_slot = (c_slot_mapping_t) slot_p;
3730
3731 if (src_slot->s_cseg == C_SV_CSEG_ID) {
3732 /*
3733 * no need to relocate... this is a page full of a single
3734 * value which is hashed to a single entry not contained
3735 * in a c_segment_t
3736 */
3737 return (kr);
3738 }
3739
3740Relookup_dst:
3741 c_seg_dst = c_seg_allocate((c_segment_t *)current_chead);
3742 /*
3743 * returns with c_seg lock held
3744 * and PAGE_REPLACEMENT_DISALLOWED(TRUE)...
3745 * c_nextslot has been allocated and
3746 * c_store.c_buffer populated
3747 */
3748 if (c_seg_dst == NULL) {
3749 /*
3750 * Out of compression segments?
3751 */
3752 kr = KERN_RESOURCE_SHORTAGE;
3753 goto out;
3754 }
3755
3756 assert(c_seg_dst->c_busy == 0);
3757
3758 C_SEG_BUSY(c_seg_dst);
3759
3760 dst_slot = c_seg_dst->c_nextslot;
3761
3762 lck_mtx_unlock_always(&c_seg_dst->c_lock);
3763
3764Relookup_src:
3765 c_seg_src = c_segments[src_slot->s_cseg - 1].c_seg;
3766
3767 assert(c_seg_dst != c_seg_src);
3768
3769 lck_mtx_lock_spin_always(&c_seg_src->c_lock);
3770
3771 if (C_SEG_IS_ONDISK(c_seg_src)) {
3772
3773 /*
3774 * A "thaw" can mark a process as eligible for
3775 * another freeze cycle without bringing any of
3776 * its swapped out c_segs back from disk (because
3777 * that is done on-demand).
3778 *
3779 * If the src c_seg we find for our pre-compressed
3780 * data is already on-disk, then we are dealing
3781 * with an app's data that is already packed and
3782 * swapped out. Don't do anything.
3783 */
3784
3785 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3786
3787 lck_mtx_unlock_always(&c_seg_src->c_lock);
3788
3789 c_seg_src = NULL;
3790
3791 goto out;
3792 }
3793
3794 if (c_seg_src->c_busy) {
3795
3796 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3797 c_seg_wait_on_busy(c_seg_src);
3798
3799 c_seg_src = NULL;
3800
3801 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3802
3803 goto Relookup_src;
3804 }
3805
3806 C_SEG_BUSY(c_seg_src);
3807
3808 lck_mtx_unlock_always(&c_seg_src->c_lock);
3809
3810 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3811
3812 /* find the c_slot */
3813 c_indx = src_slot->s_cindx;
3814
3815 c_src = C_SEG_SLOT_FROM_INDEX(c_seg_src, c_indx);
3816
3817 c_size = UNPACK_C_SIZE(c_src);
3818
3819 assert(c_size);
3820
3821 if (c_size > (uint32_t)(C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES((int32_t)c_seg_dst->c_nextoffset))) {
3822 /*
3823 * This segment is full. We need a new one.
3824 */
3825
3826 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3827
3828 lck_mtx_lock_spin_always(&c_seg_src->c_lock);
3829 C_SEG_WAKEUP_DONE(c_seg_src);
3830 lck_mtx_unlock_always(&c_seg_src->c_lock);
3831
3832 c_seg_src = NULL;
3833
3834 lck_mtx_lock_spin_always(&c_seg_dst->c_lock);
3835
3836 assert(c_seg_dst->c_busy);
3837 assert(c_seg_dst->c_state == C_IS_FILLING);
3838 assert(!c_seg_dst->c_on_minorcompact_q);
3839
3840 c_current_seg_filled(c_seg_dst, (c_segment_t *)current_chead);
3841 assert(*current_chead == NULL);
3842
3843 C_SEG_WAKEUP_DONE(c_seg_dst);
3844
3845 lck_mtx_unlock_always(&c_seg_dst->c_lock);
3846
3847 c_seg_dst = NULL;
3848
3849 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3850
3851 goto Relookup_dst;
3852 }
3853
3854 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, c_seg_dst->c_nextslot);
3855
3856 memcpy(&c_seg_dst->c_store.c_buffer[c_seg_dst->c_nextoffset], &c_seg_src->c_store.c_buffer[c_src->c_offset], c_size);
3857
3858 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
3859
39037602 3860 cslot_copy(c_dst, c_src);
3e170ce0
A		 3861 c_dst->c_offset = c_seg_dst->c_nextoffset;
3862
3863 if (c_seg_dst->c_firstemptyslot == c_seg_dst->c_nextslot)
3864 c_seg_dst->c_firstemptyslot++;
3865
3866 c_seg_dst->c_nextslot++;
3867 c_seg_dst->c_bytes_used += c_rounded_size;
3868 c_seg_dst->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
3869
3870
3871 PACK_C_SIZE(c_src, 0);
3872
3873 c_seg_src->c_bytes_used -= c_rounded_size;
3874 c_seg_src->c_bytes_unused += c_rounded_size;
3875
3876 if (c_indx < c_seg_src->c_firstemptyslot) {
3877 c_seg_src->c_firstemptyslot = c_indx;
3878 }
3879
3880 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, dst_slot);
3881
3882 PAGE_REPLACEMENT_ALLOWED(TRUE);
3883 slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
3884 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
3885 slot_ptr->s_cseg = c_seg_dst->c_mysegno + 1;
3886 slot_ptr->s_cindx = dst_slot;
3887
3888 PAGE_REPLACEMENT_ALLOWED(FALSE);
3889
3890out:
3891 if (c_seg_src) {
3892
3893 lck_mtx_lock_spin_always(&c_seg_src->c_lock);
3894
3895 C_SEG_WAKEUP_DONE(c_seg_src);
3896
3897 if (c_seg_src->c_bytes_used == 0 && c_seg_src->c_state != C_IS_FILLING) {
3898 if (!c_seg_src->c_on_minorcompact_q)
39037602 3899 c_seg_need_delayed_compaction(c_seg_src, FALSE);
3e170ce0
A		 3900 }
3901
3902 lck_mtx_unlock_always(&c_seg_src->c_lock);
3903 }
3904
3905 if (c_seg_dst) {
3906
3907 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3908
3909 lck_mtx_lock_spin_always(&c_seg_dst->c_lock);
3910
3911 if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX_INDEX) {
3912 /*
3913 * Nearing or exceeded maximum slot and offset capacity.
3914 */
3915 assert(c_seg_dst->c_busy);
3916 assert(c_seg_dst->c_state == C_IS_FILLING);
3917 assert(!c_seg_dst->c_on_minorcompact_q);
3918
3919 c_current_seg_filled(c_seg_dst, (c_segment_t *)current_chead);
3920 assert(*current_chead == NULL);
3921 }
3922
3923 C_SEG_WAKEUP_DONE(c_seg_dst);
3924
3925 lck_mtx_unlock_always(&c_seg_dst->c_lock);
3926
3927 c_seg_dst = NULL;
3928
3929 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3930 }
3931
3932 return kr;
3933}
3934#endif /* CONFIG_FREEZE */
mirror of XNU