]> git.saurik.com Git - apple/xnu.git/blame - osfmk/vm/vm_compressor.c
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-3248.60.10.tar.gz
[apple/xnu.git] / osfmk / vm / vm_compressor.c
CommitLineData
39236c6e
A		 1/*
2 * Copyright (c) 2000-2013 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <vm/vm_compressor.h>
fe8ab488
A		 30
31#if CONFIG_PHANTOM_CACHE
32#include <vm/vm_phantom_cache.h>
33#endif
34
39236c6e
A		 35#include <vm/vm_map.h>
36#include <vm/vm_pageout.h>
37#include <vm/memory_object.h>
38#include <mach/mach_host.h> /* for host_info() */
39#include <kern/ledger.h>
40
3e170ce0
A		 41#include <i386/misc_protos.h>
42
39236c6e
A		 43#include <default_pager/default_pager_alerts.h>
44#include <default_pager/default_pager_object_server.h>
45
46#include <IOKit/IOHibernatePrivate.h>
47
48/*
49 * vm_compressor_mode has a heirarchy of control to set its value.
50 * boot-args are checked first, then device-tree, and finally
51 * the default value that is defined below. See vm_fault_init() for
52 * the boot-arg & device-tree code.
53 */
54
39236c6e
A		 55
56int vm_compressor_mode = VM_PAGER_COMPRESSOR_WITH_SWAP;
57int vm_scale = 16;
58
59
04b8595b 60int vm_compressor_is_active = 0;
39236c6e 61int vm_compression_limit = 0;
3e170ce0 62int vm_compressor_available = 0;
39236c6e
A		 63
64extern boolean_t vm_swap_up;
65extern void vm_pageout_io_throttle(void);
3e170ce0 66extern int not_in_kdp;
39236c6e
A		 67
68#if CHECKSUM_THE_DATA || CHECKSUM_THE_SWAP || CHECKSUM_THE_COMPRESSED_DATA
69extern unsigned int hash_string(char *cp, int len);
70#endif
71
3e170ce0
A		 72#define UNPACK_C_SIZE(cs) ((cs->c_size == (PAGE_SIZE-1)) ? PAGE_SIZE : cs->c_size)
73#define PACK_C_SIZE(cs, size) (cs->c_size = ((size == PAGE_SIZE) ? PAGE_SIZE - 1 : size))
fe8ab488 74
39236c6e 75
3e170ce0
A		 76struct c_sv_hash_entry {
77 union {
78 struct {
79 uint32_t c_sv_he_ref;
80 uint32_t c_sv_he_data;
81 } c_sv_he;
82 uint64_t c_sv_he_record;
83
84 } c_sv_he_un;
39236c6e
A		 85};
86
3e170ce0
A		 87#define he_ref c_sv_he_un.c_sv_he.c_sv_he_ref
88#define he_data c_sv_he_un.c_sv_he.c_sv_he_data
89#define he_record c_sv_he_un.c_sv_he_record
90
91#define C_SV_HASH_MAX_MISS 32
92#define C_SV_HASH_SIZE ((1 << 10))
93#define C_SV_HASH_MASK ((1 << 10) - 1)
94#define C_SV_CSEG_ID ((1 << 22) - 1)
39236c6e
A		 95
96
97struct c_slot_mapping {
98 uint32_t s_cseg:22, /* segment number + 1 */
99 s_cindx:10; /* index in the segment */
100};
3e170ce0 101#define C_SLOT_MAX_INDEX (1 << 10)
39236c6e
A		 102
103typedef struct c_slot_mapping *c_slot_mapping_t;
104
105
106union c_segu {
107 c_segment_t c_seg;
108 uint32_t c_segno;
109};
110
111
112
113#define C_SLOT_PACK_PTR(ptr) (((uintptr_t)ptr - (uintptr_t) VM_MIN_KERNEL_AND_KEXT_ADDRESS) >> 2)
114#define C_SLOT_UNPACK_PTR(cslot) ((uintptr_t)(cslot->c_packed_ptr << 2) + (uintptr_t) VM_MIN_KERNEL_AND_KEXT_ADDRESS)
115
116
117uint32_t c_segment_count = 0;
3e170ce0 118uint32_t c_segment_count_max = 0;
39236c6e
A		 119
120uint64_t c_generation_id = 0;
121uint64_t c_generation_id_flush_barrier;
122
123
124#define HIBERNATE_FLUSHING_SECS_TO_COMPLETE 120
125
126boolean_t hibernate_no_swapspace = FALSE;
127clock_sec_t hibernate_flushing_deadline = 0;
128
129
3e170ce0
A		 130#if RECORD_THE_COMPRESSED_DATA
131char *c_compressed_record_sbuf;
132char *c_compressed_record_ebuf;
133char *c_compressed_record_cptr;
39236c6e
A		 134#endif
135
3e170ce0 136
39236c6e
A		 137queue_head_t c_age_list_head;
138queue_head_t c_swapout_list_head;
139queue_head_t c_swappedin_list_head;
140queue_head_t c_swappedout_list_head;
141queue_head_t c_swappedout_sparse_list_head;
3e170ce0
A		 142queue_head_t c_major_list_head;
143queue_head_t c_filling_list_head;
144queue_head_t c_bad_list_head;
39236c6e
A		 145
146uint32_t c_age_count = 0;
147uint32_t c_swapout_count = 0;
148uint32_t c_swappedin_count = 0;
149uint32_t c_swappedout_count = 0;
150uint32_t c_swappedout_sparse_count = 0;
3e170ce0
A		 151uint32_t c_major_count = 0;
152uint32_t c_filling_count = 0;
153uint32_t c_empty_count = 0;
154uint32_t c_bad_count = 0;
155
39236c6e
A		 156
157queue_head_t c_minor_list_head;
158uint32_t c_minor_count = 0;
159
3e170ce0
A		 160int c_overage_swapped_count = 0;
161int c_overage_swapped_limit = 0;
162
163int c_seg_fixed_array_len;
39236c6e 164union c_segu *c_segments;
3e170ce0
A		 165vm_offset_t c_buffers;
166vm_size_t c_buffers_size;
39236c6e
A		 167caddr_t c_segments_next_page;
168boolean_t c_segments_busy;
169uint32_t c_segments_available;
170uint32_t c_segments_limit;
fe8ab488 171uint32_t c_segments_nearing_limit;
3e170ce0
A		 172
173uint32_t c_segment_svp_in_hash;
174uint32_t c_segment_svp_hash_succeeded;
175uint32_t c_segment_svp_hash_failed;
176uint32_t c_segment_svp_zero_compressions;
177uint32_t c_segment_svp_nonzero_compressions;
178uint32_t c_segment_svp_zero_decompressions;
179uint32_t c_segment_svp_nonzero_decompressions;
180
181uint32_t c_segment_noncompressible_pages;
182
39236c6e
A		 183uint32_t c_segment_pages_compressed;
184uint32_t c_segment_pages_compressed_limit;
fe8ab488 185uint32_t c_segment_pages_compressed_nearing_limit;
39236c6e
A		 186uint32_t c_free_segno_head = (uint32_t)-1;
187
188uint32_t vm_compressor_minorcompact_threshold_divisor = 10;
189uint32_t vm_compressor_majorcompact_threshold_divisor = 10;
190uint32_t vm_compressor_unthrottle_threshold_divisor = 10;
191uint32_t vm_compressor_catchup_threshold_divisor = 10;
192
193#define C_SEGMENTS_PER_PAGE (PAGE_SIZE / sizeof(union c_segu))
194
195
196lck_grp_attr_t vm_compressor_lck_grp_attr;
197lck_attr_t vm_compressor_lck_attr;
198lck_grp_t vm_compressor_lck_grp;
199
39236c6e
A		 200#if __i386__ || __x86_64__
201lck_mtx_t *c_list_lock;
202#else /* __i386__ || __x86_64__ */
203lck_spin_t *c_list_lock;
204#endif /* __i386__ || __x86_64__ */
205
206lck_rw_t c_master_lock;
fe8ab488 207boolean_t decompressions_blocked = FALSE;
39236c6e
A		 208
209zone_t compressor_segment_zone;
210int c_compressor_swap_trigger = 0;
211
212uint32_t compressor_cpus;
213char *compressor_scratch_bufs;
3e170ce0
A		 214char *kdp_compressor_scratch_buf;
215char *kdp_compressor_decompressed_page;
216addr64_t kdp_compressor_decompressed_page_paddr;
217ppnum_t kdp_compressor_decompressed_page_ppnum;
39236c6e
A		 218
219clock_sec_t start_of_sample_period_sec = 0;
220clock_nsec_t start_of_sample_period_nsec = 0;
221clock_sec_t start_of_eval_period_sec = 0;
222clock_nsec_t start_of_eval_period_nsec = 0;
223uint32_t sample_period_decompression_count = 0;
224uint32_t sample_period_compression_count = 0;
225uint32_t last_eval_decompression_count = 0;
226uint32_t last_eval_compression_count = 0;
227
228#define DECOMPRESSION_SAMPLE_MAX_AGE (60 * 30)
229
3e170ce0
A		 230boolean_t vm_swapout_ripe_segments = FALSE;
231uint32_t vm_ripe_target_age = (60 * 60 * 48);
232
39236c6e
A		 233uint32_t swapout_target_age = 0;
234uint32_t age_of_decompressions_during_sample_period[DECOMPRESSION_SAMPLE_MAX_AGE];
235uint32_t overage_decompressions_during_sample_period = 0;
236
237void do_fastwake_warmup(void);
238boolean_t fastwake_warmup = FALSE;
239boolean_t fastwake_recording_in_progress = FALSE;
240clock_sec_t dont_trim_until_ts = 0;
241
242uint64_t c_segment_warmup_count;
243uint64_t first_c_segment_to_warm_generation_id = 0;
244uint64_t last_c_segment_to_warm_generation_id = 0;
245boolean_t hibernate_flushing = FALSE;
246
fe8ab488
A		 247int64_t c_segment_input_bytes __attribute__((aligned(8))) = 0;
248int64_t c_segment_compressed_bytes __attribute__((aligned(8))) = 0;
249int64_t compressor_bytes_used __attribute__((aligned(8))) = 0;
3e170ce0
A		 250
251
252struct c_sv_hash_entry c_segment_sv_hash_table[C_SV_HASH_SIZE] __attribute__ ((aligned (8)));
253
39236c6e
A		 254
255static boolean_t compressor_needs_to_swap(void);
256static void vm_compressor_swap_trigger_thread(void);
257static void vm_compressor_do_delayed_compactions(boolean_t);
258static void vm_compressor_compact_and_swap(boolean_t);
259static void vm_compressor_age_swapped_in_segments(boolean_t);
39236c6e 260
3e170ce0
A		 261static void vm_compressor_take_paging_space_action(void);
262
39236c6e
A		 263boolean_t vm_compressor_low_on_space(void);
264
265void compute_swapout_target_age(void);
266
267boolean_t c_seg_major_compact(c_segment_t, c_segment_t);
268boolean_t c_seg_major_compact_ok(c_segment_t, c_segment_t);
269
270int c_seg_minor_compaction_and_unlock(c_segment_t, boolean_t);
271int c_seg_do_minor_compaction_and_unlock(c_segment_t, boolean_t, boolean_t, boolean_t);
272void c_seg_try_minor_compaction_and_unlock(c_segment_t c_seg);
273void c_seg_need_delayed_compaction(c_segment_t);
274
275void c_seg_move_to_sparse_list(c_segment_t);
276void c_seg_insert_into_q(queue_head_t *, c_segment_t);
277
39236c6e 278uint64_t vm_available_memory(void);
fe8ab488 279uint64_t vm_compressor_pages_compressed(void);
39236c6e
A		 280
281extern unsigned int dp_pages_free, dp_pages_reserve;
282
283uint64_t
284vm_available_memory(void)
285{
286 return (((uint64_t)AVAILABLE_NON_COMPRESSED_MEMORY) * PAGE_SIZE_64);
287}
288
289
fe8ab488
A		 290uint64_t
291vm_compressor_pages_compressed(void)
292{
293 return (c_segment_pages_compressed * PAGE_SIZE_64);
294}
295
296
39236c6e
A		 297boolean_t
298vm_compression_available(void)
299{
300 if ( !(COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE))
301 return (FALSE);
302
303 if (c_segments_available >= c_segments_limit || c_segment_pages_compressed >= c_segment_pages_compressed_limit)
304 return (FALSE);
305
306 return (TRUE);
307}
308
309
310boolean_t
311vm_compressor_low_on_space(void)
312{
fe8ab488
A		 313 if ((c_segment_pages_compressed > c_segment_pages_compressed_nearing_limit) ||
314 (c_segment_count > c_segments_nearing_limit))
39236c6e
A		 315 return (TRUE);
316
317 return (FALSE);
318}
319
320
321int
fe8ab488 322vm_wants_task_throttled(task_t task)
39236c6e 323{
fe8ab488
A		 324 if (task == kernel_task)
325 return (0);
326
4bd07ac2
A		 327 if (COMPRESSED_PAGER_IS_SWAPLESS || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_SWAPLESS)
328 return (0);
329
330 if (COMPRESSED_PAGER_IS_SWAPBACKED || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_SWAPBACKED) {
fe8ab488
A		 331 if ((vm_compressor_low_on_space() || HARD_THROTTLE_LIMIT_REACHED()) &&
332 (unsigned int)pmap_compressed(task->map->pmap) > (c_segment_pages_compressed / 4))
39236c6e
A		 333 return (1);
334 } else {
fe8ab488
A		 335 if (((dp_pages_free + dp_pages_reserve < 2000) && VM_DYNAMIC_PAGING_ENABLED(memory_manager_default)) &&
336 get_task_resident_size(task) > (((AVAILABLE_NON_COMPRESSED_MEMORY) * PAGE_SIZE) / 5))
39236c6e
A		 337 return (1);
338 }
339 return (0);
340}
341
342
3e170ce0
A		 343
344static uint32_t no_paging_space_action_in_progress = 0;
345extern void memorystatus_send_low_swap_note(void);
346
347static void
348vm_compressor_take_paging_space_action(void)
349{
350 if (no_paging_space_action_in_progress == 0) {
351
352 if (OSCompareAndSwap(0, 1, (UInt32 *)&no_paging_space_action_in_progress)) {
353
354 if (no_paging_space_action()) {
355 memorystatus_send_low_swap_note();
356 }
357
358 no_paging_space_action_in_progress = 0;
359 }
360 }
361}
362
363
364
39236c6e
A		 365void
366vm_compressor_init_locks(void)
367{
368 lck_grp_attr_setdefault(&vm_compressor_lck_grp_attr);
369 lck_grp_init(&vm_compressor_lck_grp, "vm_compressor", &vm_compressor_lck_grp_attr);
370 lck_attr_setdefault(&vm_compressor_lck_attr);
371
372 lck_rw_init(&c_master_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
39236c6e
A		 373}
374
375
376void
377vm_decompressor_lock(void)
378{
fe8ab488
A		 379 PAGE_REPLACEMENT_ALLOWED(TRUE);
380
381 decompressions_blocked = TRUE;
382
383 PAGE_REPLACEMENT_ALLOWED(FALSE);
39236c6e
A		 384}
385
386void
387vm_decompressor_unlock(void)
388{
fe8ab488
A		 389 PAGE_REPLACEMENT_ALLOWED(TRUE);
390
391 decompressions_blocked = FALSE;
392
393 PAGE_REPLACEMENT_ALLOWED(FALSE);
39236c6e 394
fe8ab488 395 thread_wakeup((event_t)&decompressions_blocked);
39236c6e
A		 396}
397
398
399
400void
401vm_compressor_init(void)
402{
403 thread_t thread;
fe8ab488
A		 404 struct c_slot cs_dummy;
405 c_slot_t cs = &cs_dummy;
3e170ce0
A		 406 int c_segment_min_size;
407 int c_segment_padded_size;
fe8ab488
A		 408
409 /*
410 * ensure that any pointer that gets created from
411 * the vm_page zone can be packed properly
412 */
413 cs->c_packed_ptr = C_SLOT_PACK_PTR(zone_map_min_address);
414
415 if (C_SLOT_UNPACK_PTR(cs) != (uintptr_t)zone_map_min_address)
416 panic("C_SLOT_UNPACK_PTR failed on zone_map_min_address - %p", (void *)zone_map_min_address);
417
418 cs->c_packed_ptr = C_SLOT_PACK_PTR(zone_map_max_address);
419
420 if (C_SLOT_UNPACK_PTR(cs) != (uintptr_t)zone_map_max_address)
421 panic("C_SLOT_UNPACK_PTR failed on zone_map_max_address - %p", (void *)zone_map_max_address);
422
39236c6e
A		 423
424 assert((C_SEGMENTS_PER_PAGE * sizeof(union c_segu)) == PAGE_SIZE);
425
426 PE_parse_boot_argn("vm_compression_limit", &vm_compression_limit, sizeof (vm_compression_limit));
427
428 if (max_mem <= (3ULL * 1024ULL * 1024ULL * 1024ULL)) {
429 vm_compressor_minorcompact_threshold_divisor = 11;
430 vm_compressor_majorcompact_threshold_divisor = 13;
431 vm_compressor_unthrottle_threshold_divisor = 20;
432 vm_compressor_catchup_threshold_divisor = 35;
433 } else {
434 vm_compressor_minorcompact_threshold_divisor = 20;
435 vm_compressor_majorcompact_threshold_divisor = 25;
436 vm_compressor_unthrottle_threshold_divisor = 35;
437 vm_compressor_catchup_threshold_divisor = 50;
438 }
439 /*
440 * vm_page_init_lck_grp is now responsible for calling vm_compressor_init_locks
441 * c_master_lock needs to be available early so that "vm_page_find_contiguous" can
442 * use PAGE_REPLACEMENT_ALLOWED to coordinate with the compressor.
443 */
444
445#if __i386__ || __x86_64__
446 c_list_lock = lck_mtx_alloc_init(&vm_compressor_lck_grp, &vm_compressor_lck_attr);
447#else /* __i386__ || __x86_64__ */
448 c_list_lock = lck_spin_alloc_init(&vm_compressor_lck_grp, &vm_compressor_lck_attr);
449#endif /* __i386__ || __x86_64__ */
450
3e170ce0 451
39236c6e 452 queue_init(&c_bad_list_head);
39236c6e
A		 453 queue_init(&c_age_list_head);
454 queue_init(&c_minor_list_head);
3e170ce0
A		 455 queue_init(&c_major_list_head);
456 queue_init(&c_filling_list_head);
39236c6e
A		 457 queue_init(&c_swapout_list_head);
458 queue_init(&c_swappedin_list_head);
459 queue_init(&c_swappedout_list_head);
460 queue_init(&c_swappedout_sparse_list_head);
461
3e170ce0
A		 462 c_segment_min_size = sizeof(struct c_segment) + (C_SEG_SLOT_VAR_ARRAY_MIN_LEN * sizeof(struct c_slot));
463
464 for (c_segment_padded_size = 128; c_segment_padded_size < c_segment_min_size; c_segment_padded_size = c_segment_padded_size << 1);
465
466 compressor_segment_zone = zinit(c_segment_padded_size, 128000 * c_segment_padded_size, PAGE_SIZE, "compressor_segment");
39236c6e
A		 467 zone_change(compressor_segment_zone, Z_CALLERACCT, FALSE);
468 zone_change(compressor_segment_zone, Z_NOENCRYPT, TRUE);
469
3e170ce0 470 c_seg_fixed_array_len = (c_segment_padded_size - sizeof(struct c_segment)) / sizeof(struct c_slot);
39236c6e
A		 471
472 c_free_segno_head = -1;
473 c_segments_available = 0;
474
475 if (vm_compression_limit == 0) {
476 c_segment_pages_compressed_limit = (uint32_t)((max_mem / PAGE_SIZE)) * vm_scale;
477
478#define OLD_SWAP_LIMIT (1024 * 1024 * 16)
479#define MAX_SWAP_LIMIT (1024 * 1024 * 128)
480
481 if (c_segment_pages_compressed_limit > (OLD_SWAP_LIMIT))
482 c_segment_pages_compressed_limit = OLD_SWAP_LIMIT;
483
484 if (c_segment_pages_compressed_limit < (uint32_t)(max_mem / PAGE_SIZE_64))
485 c_segment_pages_compressed_limit = (uint32_t)(max_mem / PAGE_SIZE_64);
486 } else {
487 if (vm_compression_limit < MAX_SWAP_LIMIT)
488 c_segment_pages_compressed_limit = vm_compression_limit;
489 else
490 c_segment_pages_compressed_limit = MAX_SWAP_LIMIT;
491 }
492 if ((c_segments_limit = c_segment_pages_compressed_limit / (C_SEG_BUFSIZE / PAGE_SIZE)) > C_SEG_MAX_LIMIT)
493 c_segments_limit = C_SEG_MAX_LIMIT;
494
fe8ab488
A		 495 c_segment_pages_compressed_nearing_limit = (c_segment_pages_compressed_limit * 98) / 100;
496 c_segments_nearing_limit = (c_segments_limit * 98) / 100;
497
39236c6e
A		 498 c_segments_busy = FALSE;
499
3e170ce0
A		 500 if (kernel_memory_allocate(kernel_map, (vm_offset_t *)(&c_segments), (sizeof(union c_segu) * c_segments_limit), 0, KMA_KOBJECT | KMA_VAONLY | KMA_PERMANENT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS)
501 panic("vm_compressor_init: kernel_memory_allocate failed - c_segments\n");
502 c_buffers_size = (vm_size_t)C_SEG_ALLOCSIZE * (vm_size_t)c_segments_limit;
503 if (kernel_memory_allocate(kernel_map, &c_buffers, c_buffers_size, 0, KMA_COMPRESSOR | KMA_VAONLY | KMA_PERMANENT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS)
504 panic("vm_compressor_init: kernel_memory_allocate failed - c_buffers\n");
39236c6e
A		 505
506 c_segments_next_page = (caddr_t)c_segments;
507
508 {
509 host_basic_info_data_t hinfo;
510 mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
511
512#define BSD_HOST 1
513 host_info((host_t)BSD_HOST, HOST_BASIC_INFO, (host_info_t)&hinfo, &count);
514
515 compressor_cpus = hinfo.max_cpus;
516
3e170ce0
A		 517 compressor_scratch_bufs = kalloc_tag(compressor_cpus * WKdm_SCRATCH_BUF_SIZE, VM_KERN_MEMORY_COMPRESSOR);
518
519 kdp_compressor_scratch_buf = kalloc_tag(WKdm_SCRATCH_BUF_SIZE, VM_KERN_MEMORY_COMPRESSOR);
520 kdp_compressor_decompressed_page = kalloc_tag(PAGE_SIZE, VM_KERN_MEMORY_COMPRESSOR);
521 kdp_compressor_decompressed_page_paddr = kvtophys((vm_offset_t)kdp_compressor_decompressed_page);
522 kdp_compressor_decompressed_page_ppnum = (ppnum_t) atop(kdp_compressor_decompressed_page_paddr);
39236c6e 523 }
3e170ce0
A		 524#if CONFIG_FREEZE
525 freezer_compressor_scratch_buf = kalloc_tag(WKdm_SCRATCH_BUF_SIZE, VM_KERN_MEMORY_COMPRESSOR);
526#endif
527
528#if RECORD_THE_COMPRESSED_DATA
529 if (kernel_memory_allocate(kernel_map, (vm_offset_t *)&c_compressed_record_sbuf, (vm_size_t)C_SEG_ALLOCSIZE + (PAGE_SIZE * 2), 0, KMA_KOBJECT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS)
530 panic("vm_compressor_init: kernel_memory_allocate failed - c_compressed_record_sbuf\n");
531
532 c_compressed_record_cptr = c_compressed_record_sbuf;
533 c_compressed_record_ebuf = c_compressed_record_sbuf + C_SEG_ALLOCSIZE + (PAGE_SIZE * 2);
534#endif
39236c6e
A		 535
536 if (kernel_thread_start_priority((thread_continue_t)vm_compressor_swap_trigger_thread, NULL,
537 BASEPRI_PREEMPT - 1, &thread) != KERN_SUCCESS) {
538 panic("vm_compressor_swap_trigger_thread: create failed");
539 }
39236c6e
A		 540 thread_deallocate(thread);
541
542 assert(default_pager_init_flag == 0);
543
544 if (vm_pageout_internal_start() != KERN_SUCCESS) {
545 panic("vm_compressor_init: Failed to start the internal pageout thread.\n");
546 }
3e170ce0 547 if (COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE)
fe8ab488 548 vm_compressor_swap_init();
fe8ab488 549
04b8595b
A		 550 if (COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_SWAPBACKED)
551 vm_compressor_is_active = 1;
552
39236c6e
A		 553#if CONFIG_FREEZE
554 memorystatus_freeze_enabled = TRUE;
555#endif /* CONFIG_FREEZE */
556
557 default_pager_init_flag = 1;
3e170ce0 558 vm_compressor_available = 1;
39236c6e
A		 559
560 vm_page_reactivate_all_throttled();
561}
562
563
564#if VALIDATE_C_SEGMENTS
565
566static void
567c_seg_validate(c_segment_t c_seg, boolean_t must_be_compact)
568{
569 int c_indx;
570 int32_t bytes_used;
571 int32_t bytes_unused;
572 uint32_t c_rounded_size;
573 uint32_t c_size;
574 c_slot_t cs;
575
576 if (c_seg->c_firstemptyslot < c_seg->c_nextslot) {
577 c_indx = c_seg->c_firstemptyslot;
578 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
579
580 if (cs == NULL)
581 panic("c_seg_validate: no slot backing c_firstemptyslot");
582
583 if (cs->c_size)
584 panic("c_seg_validate: c_firstemptyslot has non-zero size (%d)\n", cs->c_size);
585 }
586 bytes_used = 0;
587 bytes_unused = 0;
588
589 for (c_indx = 0; c_indx < c_seg->c_nextslot; c_indx++) {
590
591 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
592
593 c_size = UNPACK_C_SIZE(cs);
594
595 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
596
597 bytes_used += c_rounded_size;
598
599#if CHECKSUM_THE_COMPRESSED_DATA
600 if (c_size && cs->c_hash_compressed_data != hash_string((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size))
601 panic("compressed data doesn't match original");
602#endif
603 }
604
605 if (bytes_used != c_seg->c_bytes_used)
606 panic("c_seg_validate: bytes_used mismatch - found %d, segment has %d\n", bytes_used, c_seg->c_bytes_used);
607
608 if (c_seg->c_bytes_used > C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset))
609 panic("c_seg_validate: c_bytes_used > c_nextoffset - c_nextoffset = %d, c_bytes_used = %d\n",
610 (int32_t)C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset), c_seg->c_bytes_used);
611
612 if (must_be_compact) {
613 if (c_seg->c_bytes_used != C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset))
614 panic("c_seg_validate: c_bytes_used doesn't match c_nextoffset - c_nextoffset = %d, c_bytes_used = %d\n",
615 (int32_t)C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset), c_seg->c_bytes_used);
616 }
617}
618
619#endif
620
621
622void
623c_seg_need_delayed_compaction(c_segment_t c_seg)
624{
625 boolean_t clear_busy = FALSE;
626
627 if ( !lck_mtx_try_lock_spin_always(c_list_lock)) {
fe8ab488 628 C_SEG_BUSY(c_seg);
39236c6e
A		 629
630 lck_mtx_unlock_always(&c_seg->c_lock);
631 lck_mtx_lock_spin_always(c_list_lock);
632 lck_mtx_lock_spin_always(&c_seg->c_lock);
633
634 clear_busy = TRUE;
635 }
3e170ce0
A		 636 assert(c_seg->c_state != C_IS_FILLING);
637
638 if (!c_seg->c_on_minorcompact_q && !(C_SEG_IS_ONDISK(c_seg))) {
39236c6e
A		 639 queue_enter(&c_minor_list_head, c_seg, c_segment_t, c_list);
640 c_seg->c_on_minorcompact_q = 1;
641 c_minor_count++;
642 }
643 lck_mtx_unlock_always(c_list_lock);
644
645 if (clear_busy == TRUE)
646 C_SEG_WAKEUP_DONE(c_seg);
647}
648
649
650unsigned int c_seg_moved_to_sparse_list = 0;
651
652void
653c_seg_move_to_sparse_list(c_segment_t c_seg)
654{
655 boolean_t clear_busy = FALSE;
656
657 if ( !lck_mtx_try_lock_spin_always(c_list_lock)) {
fe8ab488 658 C_SEG_BUSY(c_seg);
39236c6e
A		 659
660 lck_mtx_unlock_always(&c_seg->c_lock);
661 lck_mtx_lock_spin_always(c_list_lock);
662 lck_mtx_lock_spin_always(&c_seg->c_lock);
663
664 clear_busy = TRUE;
665 }
3e170ce0 666 c_seg_switch_state(c_seg, C_ON_SWAPPEDOUTSPARSE_Q, FALSE);
39236c6e
A		 667
668 c_seg_moved_to_sparse_list++;
669
670 lck_mtx_unlock_always(c_list_lock);
671
672 if (clear_busy == TRUE)
673 C_SEG_WAKEUP_DONE(c_seg);
674}
675
676
677void
678c_seg_insert_into_q(queue_head_t *qhead, c_segment_t c_seg)
679{
680 c_segment_t c_seg_next;
681
682 if (queue_empty(qhead)) {
683 queue_enter(qhead, c_seg, c_segment_t, c_age_list);
684 } else {
685 c_seg_next = (c_segment_t)queue_first(qhead);
686
687 while (TRUE) {
688
689 if (c_seg->c_generation_id < c_seg_next->c_generation_id) {
690 queue_insert_before(qhead, c_seg, c_seg_next, c_segment_t, c_age_list);
691 break;
692 }
693 c_seg_next = (c_segment_t) queue_next(&c_seg_next->c_age_list);
694
695 if (queue_end(qhead, (queue_entry_t) c_seg_next)) {
696 queue_enter(qhead, c_seg, c_segment_t, c_age_list);
697 break;
698 }
699 }
700 }
701}
702
703
704int try_minor_compaction_failed = 0;
705int try_minor_compaction_succeeded = 0;
706
707void
708c_seg_try_minor_compaction_and_unlock(c_segment_t c_seg)
709{
710
711 assert(c_seg->c_on_minorcompact_q);
712 /*
713 * c_seg is currently on the delayed minor compaction
714 * queue and we have c_seg locked... if we can get the
715 * c_list_lock w/o blocking (if we blocked we could deadlock
716 * because the lock order is c_list_lock then c_seg's lock)
717 * we'll pull it from the delayed list and free it directly
718 */
719 if ( !lck_mtx_try_lock_spin_always(c_list_lock)) {
720 /*
721 * c_list_lock is held, we need to bail
722 */
723 try_minor_compaction_failed++;
724
725 lck_mtx_unlock_always(&c_seg->c_lock);
726 } else {
727 try_minor_compaction_succeeded++;
728
fe8ab488 729 C_SEG_BUSY(c_seg);
39236c6e
A		 730 c_seg_do_minor_compaction_and_unlock(c_seg, TRUE, FALSE, FALSE);
731 }
732}
733
734
735int
736c_seg_do_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy, boolean_t need_list_lock, boolean_t disallow_page_replacement)
737{
738 int c_seg_freed;
739
740 assert(c_seg->c_busy);
741
3e170ce0
A		 742 /*
743 * check for the case that can occur when we are not swapping
744 * and this segment has been major compacted in the past
745 * and moved to the majorcompact q to remove it from further
746 * consideration... if the occupancy falls too low we need
747 * to put it back on the age_q so that it will be considered
748 * in the next major compaction sweep... if we don't do this
749 * we will eventually run into the c_segments_limit
750 */
751 if (c_seg->c_state == C_ON_MAJORCOMPACT_Q && C_SEG_SHOULD_MAJORCOMPACT(c_seg)) {
752
753 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
754 }
39236c6e
A		 755 if (!c_seg->c_on_minorcompact_q) {
756 if (clear_busy == TRUE)
757 C_SEG_WAKEUP_DONE(c_seg);
758
759 lck_mtx_unlock_always(&c_seg->c_lock);
760
761 return (0);
762 }
763 queue_remove(&c_minor_list_head, c_seg, c_segment_t, c_list);
764 c_seg->c_on_minorcompact_q = 0;
765 c_minor_count--;
766
767 lck_mtx_unlock_always(c_list_lock);
768
769 if (disallow_page_replacement == TRUE) {
770 lck_mtx_unlock_always(&c_seg->c_lock);
771
772 PAGE_REPLACEMENT_DISALLOWED(TRUE);
773
774 lck_mtx_lock_spin_always(&c_seg->c_lock);
775 }
776 c_seg_freed = c_seg_minor_compaction_and_unlock(c_seg, clear_busy);
777
778 if (disallow_page_replacement == TRUE)
779 PAGE_REPLACEMENT_DISALLOWED(FALSE);
780
781 if (need_list_lock == TRUE)
782 lck_mtx_lock_spin_always(c_list_lock);
783
784 return (c_seg_freed);
785}
786
787
788void
789c_seg_wait_on_busy(c_segment_t c_seg)
790{
791 c_seg->c_wanted = 1;
792 assert_wait((event_t) (c_seg), THREAD_UNINT);
793
794 lck_mtx_unlock_always(&c_seg->c_lock);
795 thread_block(THREAD_CONTINUE_NULL);
796}
797
798
3e170ce0
A		 799void
800c_seg_switch_state(c_segment_t c_seg, int new_state, boolean_t insert_head)
801{
802 int old_state = c_seg->c_state;
39236c6e 803
3e170ce0
A		 804#if DEVELOPMENT || DEBUG
805#if __i386__ || __x86_64__
806 if (new_state != C_IS_FILLING)
807 lck_mtx_assert(&c_seg->c_lock, LCK_MTX_ASSERT_OWNED);
808 lck_mtx_assert(c_list_lock, LCK_MTX_ASSERT_OWNED);
809#endif
810#endif
811 switch (old_state) {
39236c6e 812
3e170ce0
A		 813 case C_IS_EMPTY:
814 assert(new_state == C_IS_FILLING || new_state == C_IS_FREE);
39236c6e 815
3e170ce0
A		 816 c_empty_count--;
817 break;
818
819 case C_IS_FILLING:
820 assert(new_state == C_ON_AGE_Q || new_state == C_ON_SWAPOUT_Q);
821
822 queue_remove(&c_filling_list_head, c_seg, c_segment_t, c_age_list);
823 c_filling_count--;
824 break;
825
826 case C_ON_AGE_Q:
827 assert(new_state == C_ON_SWAPOUT_Q || new_state == C_ON_MAJORCOMPACT_Q ||
828 new_state == C_IS_FREE);
829
830 queue_remove(&c_age_list_head, c_seg, c_segment_t, c_age_list);
831 c_age_count--;
832 break;
833
834 case C_ON_SWAPPEDIN_Q:
835 assert(new_state == C_ON_AGE_Q || new_state == C_IS_FREE);
836
837 queue_remove(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
838 c_swappedin_count--;
839 break;
840
841 case C_ON_SWAPOUT_Q:
842 assert(new_state == C_ON_SWAPPEDOUT_Q || new_state == C_ON_SWAPPEDOUTSPARSE_Q ||
843 new_state == C_ON_AGE_Q || new_state == C_IS_FREE || new_state == C_IS_EMPTY);
844
845 queue_remove(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
846 thread_wakeup((event_t)&compaction_swapper_running);
847 c_swapout_count--;
848 break;
849
850 case C_ON_SWAPPEDOUT_Q:
851 assert(new_state == C_ON_SWAPPEDIN_Q || new_state == C_ON_SWAPPEDOUTSPARSE_Q ||
852 new_state == C_ON_BAD_Q || new_state == C_IS_EMPTY || new_state == C_IS_FREE);
853
854 queue_remove(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
855 c_swappedout_count--;
856 break;
857
858 case C_ON_SWAPPEDOUTSPARSE_Q:
859 assert(new_state == C_ON_SWAPPEDIN_Q ||
860 new_state == C_ON_BAD_Q || new_state == C_IS_EMPTY || new_state == C_IS_FREE);
861
862 queue_remove(&c_swappedout_sparse_list_head, c_seg, c_segment_t, c_age_list);
863 c_swappedout_sparse_count--;
864 break;
865
866 case C_ON_MAJORCOMPACT_Q:
867 assert(new_state == C_ON_AGE_Q || new_state == C_IS_FREE);
868
869 queue_remove(&c_major_list_head, c_seg, c_segment_t, c_age_list);
870 c_major_count--;
871 break;
872
873 case C_ON_BAD_Q:
874 assert(new_state == C_IS_FREE);
875
876 queue_remove(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
877 c_bad_count--;
878 break;
879
880 default:
881 panic("c_seg %p has bad c_state = %d\n", c_seg, old_state);
39236c6e 882 }
39236c6e 883
3e170ce0
A		 884 switch(new_state) {
885 case C_IS_FREE:
886 assert(old_state != C_IS_FILLING);
887
888 break;
39236c6e 889
3e170ce0
A		 890 case C_IS_EMPTY:
891 assert(old_state == C_ON_SWAPOUT_Q || old_state == C_ON_SWAPPEDOUT_Q || old_state == C_ON_SWAPPEDOUTSPARSE_Q);
39236c6e 892
3e170ce0
A		 893 c_empty_count++;
894 break;
895
896 case C_IS_FILLING:
897 assert(old_state == C_IS_EMPTY);
898
899 queue_enter(&c_filling_list_head, c_seg, c_segment_t, c_age_list);
900 c_filling_count++;
901 break;
902
903 case C_ON_AGE_Q:
904 assert(old_state == C_IS_FILLING || old_state == C_ON_SWAPPEDIN_Q ||
905 old_state == C_ON_MAJORCOMPACT_Q || old_state == C_ON_SWAPOUT_Q);
906
907 if (old_state == C_IS_FILLING)
908 queue_enter(&c_age_list_head, c_seg, c_segment_t, c_age_list);
909 else
910 c_seg_insert_into_q(&c_age_list_head, c_seg);
911 c_age_count++;
912 break;
913
914 case C_ON_SWAPPEDIN_Q:
915 assert(c_seg->c_state == C_ON_SWAPPEDOUT_Q || c_seg->c_state == C_ON_SWAPPEDOUTSPARSE_Q);
916
917 if (insert_head == TRUE)
918 queue_enter_first(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
919 else
920 queue_enter(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
921 c_swappedin_count++;
922 break;
923
924 case C_ON_SWAPOUT_Q:
925 assert(old_state == C_ON_AGE_Q || old_state == C_IS_FILLING);
926
927 if (insert_head == TRUE)
928 queue_enter_first(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
929 else
930 queue_enter(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
931 c_swapout_count++;
932 break;
933
934 case C_ON_SWAPPEDOUT_Q:
935 assert(c_seg->c_state == C_ON_SWAPOUT_Q);
936
937 if (insert_head == TRUE)
938 queue_enter_first(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
939 else
940 queue_enter(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
941 c_swappedout_count++;
942 break;
943
944 case C_ON_SWAPPEDOUTSPARSE_Q:
945 assert(c_seg->c_state == C_ON_SWAPOUT_Q || c_seg->c_state == C_ON_SWAPPEDOUT_Q);
946
947 c_seg_insert_into_q(&c_swappedout_sparse_list_head, c_seg);
948 c_swappedout_sparse_count++;
949 break;
950
951 case C_ON_MAJORCOMPACT_Q:
952 assert(c_seg->c_state == C_ON_AGE_Q);
953
954 if (insert_head == TRUE)
955 queue_enter_first(&c_major_list_head, c_seg, c_segment_t, c_age_list);
956 else
957 queue_enter(&c_major_list_head, c_seg, c_segment_t, c_age_list);
958 c_major_count++;
959 break;
960
961 case C_ON_BAD_Q:
962 assert(c_seg->c_state == C_ON_SWAPPEDOUT_Q || c_seg->c_state == C_ON_SWAPPEDOUTSPARSE_Q);
963
964 if (insert_head == TRUE)
965 queue_enter_first(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
966 else
967 queue_enter(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
968 c_bad_count++;
969 break;
970
971 default:
972 panic("c_seg %p requesting bad c_state = %d\n", c_seg, new_state);
973 }
974 c_seg->c_state = new_state;
39236c6e
A		 975}
976
977
3e170ce0 978
39236c6e
A		 979void
980c_seg_free(c_segment_t c_seg)
981{
fe8ab488 982 assert(c_seg->c_busy);
39236c6e
A		 983
984 lck_mtx_unlock_always(&c_seg->c_lock);
985 lck_mtx_lock_spin_always(c_list_lock);
986 lck_mtx_lock_spin_always(&c_seg->c_lock);
987
988 c_seg_free_locked(c_seg);
989}
990
991
992void
993c_seg_free_locked(c_segment_t c_seg)
994{
3e170ce0 995 int segno;
04b8595b 996 int pages_populated = 0;
39236c6e 997 int32_t *c_buffer = NULL;
04b8595b 998 uint64_t c_swap_handle = 0;
39236c6e 999
3e170ce0 1000 assert(c_seg->c_busy);
39236c6e 1001 assert(!c_seg->c_on_minorcompact_q);
3e170ce0 1002 assert(!c_seg->c_busy_swapping);
39236c6e 1003
3e170ce0
A		 1004 if (c_seg->c_overage_swap == TRUE) {
1005 c_overage_swapped_count--;
1006 c_seg->c_overage_swap = FALSE;
1007 }
1008 if ( !(C_SEG_IS_ONDISK(c_seg)))
1009 c_buffer = c_seg->c_store.c_buffer;
1010 else
1011 c_swap_handle = c_seg->c_store.c_swap_handle;
39236c6e 1012
3e170ce0 1013 c_seg_switch_state(c_seg, C_IS_FREE, FALSE);
39236c6e 1014
3e170ce0 1015 lck_mtx_unlock_always(c_list_lock);
39236c6e 1016
3e170ce0 1017 if (c_buffer) {
39236c6e 1018 pages_populated = (round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE;
39236c6e 1019 c_seg->c_store.c_buffer = NULL;
3e170ce0 1020 } else
39236c6e 1021 c_seg->c_store.c_swap_handle = (uint64_t)-1;
3e170ce0 1022
39236c6e
A		 1023 lck_mtx_unlock_always(&c_seg->c_lock);
1024
1025 if (c_buffer) {
8a3053a0
A		 1026 if (pages_populated)
1027 kernel_memory_depopulate(kernel_map, (vm_offset_t) c_buffer, pages_populated * PAGE_SIZE, KMA_COMPRESSOR);
39236c6e 1028
3e170ce0
A		 1029 } else if (c_swap_handle) {
1030 /*
1031 * Free swap space on disk.
1032 */
39236c6e 1033 vm_swap_free(c_swap_handle);
3e170ce0
A		 1034 }
1035 lck_mtx_lock_spin_always(&c_seg->c_lock);
1036
1037 C_SEG_WAKEUP_DONE(c_seg);
1038 lck_mtx_unlock_always(&c_seg->c_lock);
1039
1040 segno = c_seg->c_mysegno;
39236c6e 1041
3e170ce0
A		 1042 lck_mtx_lock_spin_always(c_list_lock);
1043 /*
1044 * because the c_buffer is now associated with the segno,
1045 * we can't put the segno back on the free list until
1046 * after we have depopulated the c_buffer range, or
1047 * we run the risk of depopulating a range that is
1048 * now being used in one of the compressor heads
1049 */
1050 c_segments[segno].c_segno = c_free_segno_head;
1051 c_free_segno_head = segno;
1052 c_segment_count--;
1053
1054 lck_mtx_unlock_always(c_list_lock);
39236c6e
A		 1055
1056#if __i386__ || __x86_64__
1057 lck_mtx_destroy(&c_seg->c_lock, &vm_compressor_lck_grp);
1058#else /* __i386__ || __x86_64__ */
1059 lck_spin_destroy(&c_seg->c_lock, &vm_compressor_lck_grp);
1060#endif /* __i386__ || __x86_64__ */
1061
3e170ce0
A		 1062 if (c_seg->c_slot_var_array_len)
1063 kfree(c_seg->c_slot_var_array, sizeof(struct c_slot) * c_seg->c_slot_var_array_len);
39236c6e 1064
39236c6e
A		 1065 zfree(compressor_segment_zone, c_seg);
1066}
1067
1068
1069int c_seg_trim_page_count = 0;
1070
1071void
1072c_seg_trim_tail(c_segment_t c_seg)
1073{
1074 c_slot_t cs;
1075 uint32_t c_size;
1076 uint32_t c_offset;
1077 uint32_t c_rounded_size;
1078 uint16_t current_nextslot;
1079 uint32_t current_populated_offset;
1080
1081 if (c_seg->c_bytes_used == 0)
1082 return;
1083 current_nextslot = c_seg->c_nextslot;
1084 current_populated_offset = c_seg->c_populated_offset;
1085
1086 while (c_seg->c_nextslot) {
1087
1088 cs = C_SEG_SLOT_FROM_INDEX(c_seg, (c_seg->c_nextslot - 1));
1089
1090 c_size = UNPACK_C_SIZE(cs);
1091
1092 if (c_size) {
1093 if (current_nextslot != c_seg->c_nextslot) {
1094 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
1095 c_offset = cs->c_offset + C_SEG_BYTES_TO_OFFSET(c_rounded_size);
1096
1097 c_seg->c_nextoffset = c_offset;
1098 c_seg->c_populated_offset = (c_offset + (C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1)) & ~(C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1);
1099
1100 if (c_seg->c_firstemptyslot > c_seg->c_nextslot)
1101 c_seg->c_firstemptyslot = c_seg->c_nextslot;
1102
1103 c_seg_trim_page_count += ((round_page_32(C_SEG_OFFSET_TO_BYTES(current_populated_offset)) -
1104 round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE);
1105 }
1106 break;
1107 }
1108 c_seg->c_nextslot--;
1109 }
1110 assert(c_seg->c_nextslot);
1111}
1112
1113
1114int
1115c_seg_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy)
1116{
1117 c_slot_mapping_t slot_ptr;
1118 uint32_t c_offset = 0;
1119 uint32_t old_populated_offset;
1120 uint32_t c_rounded_size;
1121 uint32_t c_size;
1122 int c_indx = 0;
1123 int i;
1124 c_slot_t c_dst;
1125 c_slot_t c_src;
1126 boolean_t need_unlock = TRUE;
1127
1128 assert(c_seg->c_busy);
1129
1130#if VALIDATE_C_SEGMENTS
1131 c_seg_validate(c_seg, FALSE);
1132#endif
1133 if (c_seg->c_bytes_used == 0) {
1134 c_seg_free(c_seg);
1135 return (1);
1136 }
1137 if (c_seg->c_firstemptyslot >= c_seg->c_nextslot || C_SEG_UNUSED_BYTES(c_seg) < PAGE_SIZE)
1138 goto done;
1139
1140#if VALIDATE_C_SEGMENTS
1141 c_seg->c_was_minor_compacted++;
1142#endif
1143 c_indx = c_seg->c_firstemptyslot;
1144 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
1145
1146 old_populated_offset = c_seg->c_populated_offset;
1147 c_offset = c_dst->c_offset;
1148
1149 for (i = c_indx + 1; i < c_seg->c_nextslot && c_offset < c_seg->c_nextoffset; i++) {
1150
1151 c_src = C_SEG_SLOT_FROM_INDEX(c_seg, i);
1152
1153 c_size = UNPACK_C_SIZE(c_src);
1154
1155 if (c_size == 0)
1156 continue;
1157
1158 memcpy(&c_seg->c_store.c_buffer[c_offset], &c_seg->c_store.c_buffer[c_src->c_offset], c_size);
1159
1160#if CHECKSUM_THE_DATA
1161 c_dst->c_hash_data = c_src->c_hash_data;
1162#endif
1163#if CHECKSUM_THE_COMPRESSED_DATA
1164 c_dst->c_hash_compressed_data = c_src->c_hash_compressed_data;
1165#endif
1166 c_dst->c_size = c_src->c_size;
1167 c_dst->c_packed_ptr = c_src->c_packed_ptr;
1168 c_dst->c_offset = c_offset;
1169
1170 slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
1171 slot_ptr->s_cindx = c_indx;
1172
1173 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
1174
1175 c_offset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
1176 PACK_C_SIZE(c_src, 0);
1177 c_indx++;
1178
1179 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
1180 }
1181 c_seg->c_firstemptyslot = c_indx;
1182 c_seg->c_nextslot = c_indx;
1183 c_seg->c_nextoffset = c_offset;
1184 c_seg->c_populated_offset = (c_offset + (C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1)) & ~(C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1);
1185 c_seg->c_bytes_unused = 0;
1186
1187#if VALIDATE_C_SEGMENTS
1188 c_seg_validate(c_seg, TRUE);
1189#endif
1190
1191 if (old_populated_offset > c_seg->c_populated_offset) {
1192 uint32_t gc_size;
1193 int32_t *gc_ptr;
1194
1195 gc_size = C_SEG_OFFSET_TO_BYTES(old_populated_offset - c_seg->c_populated_offset);
1196 gc_ptr = &c_seg->c_store.c_buffer[c_seg->c_populated_offset];
1197
1198 lck_mtx_unlock_always(&c_seg->c_lock);
1199
1200 kernel_memory_depopulate(kernel_map, (vm_offset_t)gc_ptr, gc_size, KMA_COMPRESSOR);
1201
1202 if (clear_busy == TRUE)
1203 lck_mtx_lock_spin_always(&c_seg->c_lock);
1204 else
1205 need_unlock = FALSE;
1206 }
1207done:
1208 if (need_unlock == TRUE) {
1209 if (clear_busy == TRUE)
1210 C_SEG_WAKEUP_DONE(c_seg);
1211
1212 lck_mtx_unlock_always(&c_seg->c_lock);
1213 }
1214 return (0);
1215}
1216
1217
3e170ce0
A		 1218static void
1219c_seg_alloc_nextslot(c_segment_t c_seg)
1220{
1221 struct c_slot *old_slot_array = NULL;
1222 struct c_slot *new_slot_array = NULL;
1223 int newlen;
1224 int oldlen;
1225
1226 if (c_seg->c_nextslot < c_seg_fixed_array_len)
1227 return;
1228
1229 if ((c_seg->c_nextslot - c_seg_fixed_array_len) >= c_seg->c_slot_var_array_len) {
1230
1231 oldlen = c_seg->c_slot_var_array_len;
1232 old_slot_array = c_seg->c_slot_var_array;
1233
1234 if (oldlen == 0)
1235 newlen = C_SEG_SLOT_VAR_ARRAY_MIN_LEN;
1236 else
1237 newlen = oldlen * 2;
1238
1239 new_slot_array = (struct c_slot *)kalloc(sizeof(struct c_slot) * newlen);
1240
1241 lck_mtx_lock_spin_always(&c_seg->c_lock);
1242
1243 if (old_slot_array)
1244 memcpy((char *)new_slot_array, (char *)old_slot_array, sizeof(struct c_slot) * oldlen);
1245
1246 c_seg->c_slot_var_array_len = newlen;
1247 c_seg->c_slot_var_array = new_slot_array;
1248
1249 lck_mtx_unlock_always(&c_seg->c_lock);
1250
1251 if (old_slot_array)
1252 kfree(old_slot_array, sizeof(struct c_slot) * oldlen);
1253 }
1254}
1255
1256
39236c6e
A		 1257
1258struct {
1259 uint64_t asked_permission;
1260 uint64_t compactions;
1261 uint64_t moved_slots;
1262 uint64_t moved_bytes;
1263 uint64_t wasted_space_in_swapouts;
1264 uint64_t count_of_swapouts;
3e170ce0 1265 uint64_t count_of_freed_segs;
39236c6e
A		 1266} c_seg_major_compact_stats;
1267
1268
04b8595b 1269#define C_MAJOR_COMPACTION_SIZE_APPROPRIATE ((C_SEG_BUFSIZE * 90) / 100)
39236c6e
A		 1270
1271
1272boolean_t
1273c_seg_major_compact_ok(
1274 c_segment_t c_seg_dst,
1275 c_segment_t c_seg_src)
1276{
1277
1278 c_seg_major_compact_stats.asked_permission++;
1279
39236c6e
A		 1280 if (c_seg_src->c_bytes_used >= C_MAJOR_COMPACTION_SIZE_APPROPRIATE &&
1281 c_seg_dst->c_bytes_used >= C_MAJOR_COMPACTION_SIZE_APPROPRIATE)
1282 return (FALSE);
1283
3e170ce0 1284 if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX_INDEX) {
39236c6e
A		 1285 /*
1286 * destination segment is full... can't compact
1287 */
1288 return (FALSE);
1289 }
1290
1291 return (TRUE);
1292}
1293
1294
1295boolean_t
1296c_seg_major_compact(
1297 c_segment_t c_seg_dst,
1298 c_segment_t c_seg_src)
1299{
1300 c_slot_mapping_t slot_ptr;
1301 uint32_t c_rounded_size;
1302 uint32_t c_size;
1303 uint16_t dst_slot;
1304 int i;
1305 c_slot_t c_dst;
1306 c_slot_t c_src;
39236c6e
A		 1307 boolean_t keep_compacting = TRUE;
1308
1309 /*
1310 * segments are not locked but they are both marked c_busy
1311 * which keeps c_decompress from working on them...
1312 * we can safely allocate new pages, move compressed data
1313 * from c_seg_src to c_seg_dst and update both c_segment's
1314 * state w/o holding the master lock
1315 */
1316
1317#if VALIDATE_C_SEGMENTS
1318 c_seg_dst->c_was_major_compacted++;
1319 c_seg_src->c_was_major_donor++;
1320#endif
1321 c_seg_major_compact_stats.compactions++;
1322
1323 dst_slot = c_seg_dst->c_nextslot;
1324
1325 for (i = 0; i < c_seg_src->c_nextslot; i++) {
1326
1327 c_src = C_SEG_SLOT_FROM_INDEX(c_seg_src, i);
1328
1329 c_size = UNPACK_C_SIZE(c_src);
1330
1331 if (c_size == 0) {
1332 /* BATCH: move what we have so far; */
1333 continue;
1334 }
1335
1336 if (C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset - c_seg_dst->c_nextoffset) < (unsigned) c_size) {
3e170ce0
A		 1337 int size_to_populate;
1338
39236c6e 1339 /* doesn't fit */
3e170ce0
A		 1340 size_to_populate = C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset);
1341
1342 if (size_to_populate == 0) {
39236c6e
A		 1343 /* can't fit */
1344 keep_compacting = FALSE;
1345 break;
1346 }
3e170ce0
A		 1347 if (size_to_populate > C_SEG_MAX_POPULATE_SIZE)
1348 size_to_populate = C_SEG_MAX_POPULATE_SIZE;
1349
39236c6e
A		 1350 kernel_memory_populate(kernel_map,
1351 (vm_offset_t) &c_seg_dst->c_store.c_buffer[c_seg_dst->c_populated_offset],
3e170ce0
A		 1352 size_to_populate,
1353 KMA_COMPRESSOR,
1354 VM_KERN_MEMORY_COMPRESSOR);
39236c6e 1355
3e170ce0 1356 c_seg_dst->c_populated_offset += C_SEG_BYTES_TO_OFFSET(size_to_populate);
39236c6e
A		 1357 assert(C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset) <= C_SEG_BUFSIZE);
1358 }
3e170ce0 1359 c_seg_alloc_nextslot(c_seg_dst);
39236c6e 1360
39236c6e
A		 1361 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, c_seg_dst->c_nextslot);
1362
1363 memcpy(&c_seg_dst->c_store.c_buffer[c_seg_dst->c_nextoffset], &c_seg_src->c_store.c_buffer[c_src->c_offset], c_size);
1364
1365 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
1366
1367 c_seg_major_compact_stats.moved_slots++;
1368 c_seg_major_compact_stats.moved_bytes += c_size;
1369
1370#if CHECKSUM_THE_DATA
1371 c_dst->c_hash_data = c_src->c_hash_data;
1372#endif
1373#if CHECKSUM_THE_COMPRESSED_DATA
1374 c_dst->c_hash_compressed_data = c_src->c_hash_compressed_data;
1375#endif
1376 c_dst->c_size = c_src->c_size;
1377 c_dst->c_packed_ptr = c_src->c_packed_ptr;
1378 c_dst->c_offset = c_seg_dst->c_nextoffset;
1379
1380 if (c_seg_dst->c_firstemptyslot == c_seg_dst->c_nextslot)
1381 c_seg_dst->c_firstemptyslot++;
1382 c_seg_dst->c_nextslot++;
1383 c_seg_dst->c_bytes_used += c_rounded_size;
1384 c_seg_dst->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
1385
1386 PACK_C_SIZE(c_src, 0);
1387
1388 c_seg_src->c_bytes_used -= c_rounded_size;
1389 c_seg_src->c_bytes_unused += c_rounded_size;
1390 c_seg_src->c_firstemptyslot = 0;
1391
3e170ce0 1392 if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX_INDEX) {
39236c6e
A		 1393 /* dest segment is now full */
1394 keep_compacting = FALSE;
1395 break;
1396 }
1397 }
1398 if (dst_slot < c_seg_dst->c_nextslot) {
1399
1400 PAGE_REPLACEMENT_ALLOWED(TRUE);
1401 /*
1402 * we've now locked out c_decompress from
1403 * converting the slot passed into it into
1404 * a c_segment_t which allows us to use
1405 * the backptr to change which c_segment and
1406 * index the slot points to
1407 */
1408 while (dst_slot < c_seg_dst->c_nextslot) {
1409
1410 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, dst_slot);
1411
1412 slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
1413 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
1414 slot_ptr->s_cseg = c_seg_dst->c_mysegno + 1;
1415 slot_ptr->s_cindx = dst_slot++;
1416 }
1417 PAGE_REPLACEMENT_ALLOWED(FALSE);
1418 }
1419 return (keep_compacting);
1420}
1421
1422
fe8ab488
A		 1423uint64_t
1424vm_compressor_compute_elapsed_msecs(clock_sec_t end_sec, clock_nsec_t end_nsec, clock_sec_t start_sec, clock_nsec_t start_nsec)
39236c6e
A		 1425{
1426 uint64_t end_msecs;
1427 uint64_t start_msecs;
1428
1429 end_msecs = (end_sec * 1000) + end_nsec / 1000000;
1430 start_msecs = (start_sec * 1000) + start_nsec / 1000000;
1431
1432 return (end_msecs - start_msecs);
1433}
1434
1435
1436
1437uint32_t compressor_eval_period_in_msecs = 250;
1438uint32_t compressor_sample_min_in_msecs = 500;
1439uint32_t compressor_sample_max_in_msecs = 10000;
1440uint32_t compressor_thrashing_threshold_per_10msecs = 50;
1441uint32_t compressor_thrashing_min_per_10msecs = 20;
1442
fe8ab488
A		 1443/* When true, reset sample data next chance we get. */
1444static boolean_t compressor_need_sample_reset = FALSE;
1445
39236c6e
A		 1446extern uint32_t vm_page_filecache_min;
1447
1448
1449void
1450compute_swapout_target_age(void)
1451{
1452 clock_sec_t cur_ts_sec;
1453 clock_nsec_t cur_ts_nsec;
1454 uint32_t min_operations_needed_in_this_sample;
1455 uint64_t elapsed_msecs_in_eval;
1456 uint64_t elapsed_msecs_in_sample;
39236c6e
A		 1457 boolean_t need_eval_reset = FALSE;
1458
1459 clock_get_system_nanotime(&cur_ts_sec, &cur_ts_nsec);
1460
fe8ab488 1461 elapsed_msecs_in_sample = vm_compressor_compute_elapsed_msecs(cur_ts_sec, cur_ts_nsec, start_of_sample_period_sec, start_of_sample_period_nsec);
39236c6e 1462
fe8ab488
A		 1463 if (compressor_need_sample_reset ||
1464 elapsed_msecs_in_sample >= compressor_sample_max_in_msecs) {
1465 compressor_need_sample_reset = TRUE;
39236c6e
A		 1466 need_eval_reset = TRUE;
1467 goto done;
1468 }
fe8ab488 1469 elapsed_msecs_in_eval = vm_compressor_compute_elapsed_msecs(cur_ts_sec, cur_ts_nsec, start_of_eval_period_sec, start_of_eval_period_nsec);
39236c6e
A		 1470
1471 if (elapsed_msecs_in_eval < compressor_eval_period_in_msecs)
1472 goto done;
1473 need_eval_reset = TRUE;
1474
1475 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_START, elapsed_msecs_in_eval, sample_period_compression_count, sample_period_decompression_count, 0, 0);
1476
1477 min_operations_needed_in_this_sample = (compressor_thrashing_min_per_10msecs * (uint32_t)elapsed_msecs_in_eval) / 10;
1478
1479 if ((sample_period_compression_count - last_eval_compression_count) < min_operations_needed_in_this_sample ||
1480 (sample_period_decompression_count - last_eval_decompression_count) < min_operations_needed_in_this_sample) {
1481
1482 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, sample_period_compression_count - last_eval_compression_count,
1483 sample_period_decompression_count - last_eval_decompression_count, 0, 1, 0);
1484
1485 swapout_target_age = 0;
1486
fe8ab488 1487 compressor_need_sample_reset = TRUE;
39236c6e
A		 1488 need_eval_reset = TRUE;
1489 goto done;
1490 }
1491 last_eval_compression_count = sample_period_compression_count;
1492 last_eval_decompression_count = sample_period_decompression_count;
1493
1494 if (elapsed_msecs_in_sample < compressor_sample_min_in_msecs) {
1495
1496 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, swapout_target_age, 0, 0, 5, 0);
1497 goto done;
1498 }
1499 if (sample_period_decompression_count > ((compressor_thrashing_threshold_per_10msecs * elapsed_msecs_in_sample) / 10)) {
1500
1501 uint64_t running_total;
1502 uint64_t working_target;
1503 uint64_t aging_target;
1504 uint32_t oldest_age_of_csegs_sampled = 0;
1505 uint64_t working_set_approximation = 0;
1506
1507 swapout_target_age = 0;
1508
1509 working_target = (sample_period_decompression_count / 100) * 95; /* 95 percent */
1510 aging_target = (sample_period_decompression_count / 100) * 1; /* 1 percent */
1511 running_total = 0;
1512
1513 for (oldest_age_of_csegs_sampled = 0; oldest_age_of_csegs_sampled < DECOMPRESSION_SAMPLE_MAX_AGE; oldest_age_of_csegs_sampled++) {
1514
1515 running_total += age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
1516
1517 working_set_approximation += oldest_age_of_csegs_sampled * age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
1518
1519 if (running_total >= working_target)
1520 break;
1521 }
1522 if (oldest_age_of_csegs_sampled < DECOMPRESSION_SAMPLE_MAX_AGE) {
1523
1524 working_set_approximation = (working_set_approximation * 1000) / elapsed_msecs_in_sample;
1525
1526 if (working_set_approximation < VM_PAGE_COMPRESSOR_COUNT) {
1527
1528 running_total = overage_decompressions_during_sample_period;
1529
1530 for (oldest_age_of_csegs_sampled = DECOMPRESSION_SAMPLE_MAX_AGE - 1; oldest_age_of_csegs_sampled; oldest_age_of_csegs_sampled--) {
1531 running_total += age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
1532
1533 if (running_total >= aging_target)
1534 break;
1535 }
1536 swapout_target_age = (uint32_t)cur_ts_sec - oldest_age_of_csegs_sampled;
1537
1538 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, swapout_target_age, working_set_approximation, VM_PAGE_COMPRESSOR_COUNT, 2, 0);
1539 } else {
1540 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, working_set_approximation, VM_PAGE_COMPRESSOR_COUNT, 0, 3, 0);
1541 }
1542 } else
1543 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, working_target, running_total, 0, 4, 0);
1544
fe8ab488 1545 compressor_need_sample_reset = TRUE;
39236c6e
A		 1546 need_eval_reset = TRUE;
1547 } else
1548 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, sample_period_decompression_count, (compressor_thrashing_threshold_per_10msecs * elapsed_msecs_in_sample) / 10, 0, 6, 0);
1549done:
fe8ab488 1550 if (compressor_need_sample_reset == TRUE) {
39236c6e
A		 1551 bzero(age_of_decompressions_during_sample_period, sizeof(age_of_decompressions_during_sample_period));
1552 overage_decompressions_during_sample_period = 0;
1553
1554 start_of_sample_period_sec = cur_ts_sec;
1555 start_of_sample_period_nsec = cur_ts_nsec;
1556 sample_period_decompression_count = 0;
1557 sample_period_compression_count = 0;
1558 last_eval_decompression_count = 0;
1559 last_eval_compression_count = 0;
fe8ab488 1560 compressor_need_sample_reset = FALSE;
39236c6e
A		 1561 }
1562 if (need_eval_reset == TRUE) {
1563 start_of_eval_period_sec = cur_ts_sec;
1564 start_of_eval_period_nsec = cur_ts_nsec;
1565 }
1566}
1567
1568
fe8ab488
A		 1569int compaction_swapper_inited = 0;
1570int compaction_swapper_init_now = 0;
39236c6e
A		 1571int compaction_swapper_running = 0;
1572int compaction_swapper_abort = 0;
1573
1574
1575#if CONFIG_JETSAM
1576boolean_t memorystatus_kill_on_VM_thrashing(boolean_t);
fe8ab488 1577boolean_t memorystatus_kill_on_FC_thrashing(boolean_t);
39236c6e 1578int compressor_thrashing_induced_jetsam = 0;
fe8ab488
A		 1579int filecache_thrashing_induced_jetsam = 0;
1580static boolean_t vm_compressor_thrashing_detected = FALSE;
39236c6e
A		 1581#endif /* CONFIG_JETSAM */
1582
1583static boolean_t
1584compressor_needs_to_swap(void)
1585{
1586 boolean_t should_swap = FALSE;
1587
3e170ce0
A		 1588 if (vm_swapout_ripe_segments == TRUE && c_overage_swapped_count < c_overage_swapped_limit) {
1589 c_segment_t c_seg;
1590 clock_sec_t now;
1591 clock_sec_t age;
1592 clock_nsec_t nsec;
1593
1594 clock_get_system_nanotime(&now, &nsec);
1595 age = 0;
1596
1597 lck_mtx_lock_spin_always(c_list_lock);
1598
1599 if ( !queue_empty(&c_age_list_head)) {
1600 c_seg = (c_segment_t) queue_first(&c_age_list_head);
1601
1602 age = now - c_seg->c_creation_ts;
1603 }
1604 lck_mtx_unlock_always(c_list_lock);
1605
1606 if (age >= vm_ripe_target_age)
1607 return (TRUE);
1608 }
1609 if ((vm_compressor_mode == VM_PAGER_COMPRESSOR_WITH_SWAP) && vm_swap_up == TRUE) {
39236c6e
A		 1610 if (COMPRESSOR_NEEDS_TO_SWAP()) {
1611 return (TRUE);
1612 }
1613 if (VM_PAGE_Q_THROTTLED(&vm_pageout_queue_external) && vm_page_anonymous_count < (vm_page_inactive_count / 20)) {
1614 return (TRUE);
1615 }
3e170ce0 1616 if (vm_page_free_count < (vm_page_free_reserved - (COMPRESSOR_FREE_RESERVED_LIMIT * 2)))
39236c6e
A		 1617 return (TRUE);
1618 }
1619 compute_swapout_target_age();
1620
1621 if (swapout_target_age) {
1622 c_segment_t c_seg;
1623
1624 lck_mtx_lock_spin_always(c_list_lock);
1625
1626 if (!queue_empty(&c_age_list_head)) {
1627
1628 c_seg = (c_segment_t) queue_first(&c_age_list_head);
1629
fe8ab488 1630 if (c_seg->c_creation_ts > swapout_target_age)
39236c6e
A		 1631 swapout_target_age = 0;
1632 }
1633 lck_mtx_unlock_always(c_list_lock);
1634 }
fe8ab488
A		 1635#if CONFIG_PHANTOM_CACHE
1636 if (vm_phantom_cache_check_pressure())
1637 should_swap = TRUE;
1638#endif
1639 if (swapout_target_age)
1640 should_swap = TRUE;
39236c6e 1641
fe8ab488 1642#if CONFIG_JETSAM
3e170ce0
A		 1643 if (should_swap || c_segment_pages_compressed > c_segment_pages_compressed_nearing_limit) {
1644
1645 if (vm_compressor_thrashing_detected == FALSE) {
1646 vm_compressor_thrashing_detected = TRUE;
fe8ab488 1647
3e170ce0
A		 1648 if (swapout_target_age || c_segment_pages_compressed > c_segment_pages_compressed_nearing_limit) {
1649 memorystatus_kill_on_VM_thrashing(TRUE /* async */);
1650 compressor_thrashing_induced_jetsam++;
1651 } else {
1652 memorystatus_kill_on_FC_thrashing(TRUE /* async */);
1653 filecache_thrashing_induced_jetsam++;
39236c6e 1654 }
3e170ce0
A		 1655 }
1656 /*
1657 * let the jetsam take precedence over
1658 * any major compactions we might have
1659 * been able to do... otherwise we run
1660 * the risk of doing major compactions
1661 * on segments we're about to free up
1662 * due to the jetsam activity.
1663 */
1664 should_swap = FALSE;
39236c6e 1665 }
fe8ab488 1666
3e170ce0
A		 1667#endif /* CONFIG_JETSAM */
1668
1669 if (should_swap == FALSE) {
1670 /*
1671 * COMPRESSOR_NEEDS_TO_MAJOR_COMPACT returns true only if we're
1672 * about to run out of available compressor segments... in this
1673 * case, we absolutely need to run a major compaction even if
1674 * we've just kicked off a jetsam or we don't otherwise need to
1675 * swap... terminating objects releases
1676 * pages back to the uncompressed cache, but does not guarantee
1677 * that we will free up even a single compression segment
1678 */
1679 should_swap = COMPRESSOR_NEEDS_TO_MAJOR_COMPACT();
1680 }
1681
1682 /*
1683 * returning TRUE when swap_supported == FALSE
39236c6e
A		 1684 * will cause the major compaction engine to
1685 * run, but will not trigger any swapping...
1686 * segments that have been major compacted
3e170ce0 1687 * will be moved to the majorcompact queue
39236c6e
A		 1688 */
1689 return (should_swap);
1690}
1691
fe8ab488
A		 1692#if CONFIG_JETSAM
1693/*
1694 * This function is called from the jetsam thread after killing something to
1695 * mitigate thrashing.
1696 *
1697 * We need to restart our thrashing detection heuristics since memory pressure
1698 * has potentially changed significantly, and we don't want to detect on old
1699 * data from before the jetsam.
1700 */
1701void
1702vm_thrashing_jetsam_done(void)
39236c6e 1703{
fe8ab488 1704 vm_compressor_thrashing_detected = FALSE;
39236c6e 1705
fe8ab488
A		 1706 /* Were we compressor-thrashing or filecache-thrashing? */
1707 if (swapout_target_age) {
1708 swapout_target_age = 0;
1709 compressor_need_sample_reset = TRUE;
39236c6e 1710 }
fe8ab488
A		 1711#if CONFIG_PHANTOM_CACHE
1712 else {
1713 vm_phantom_cache_restart_sample();
1714 }
1715#endif
39236c6e 1716}
fe8ab488 1717#endif /* CONFIG_JETSAM */
39236c6e
A		 1718
1719uint32_t vm_wake_compactor_swapper_calls = 0;
1720
1721void
1722vm_wake_compactor_swapper(void)
1723{
3e170ce0 1724 if (compaction_swapper_running || c_segment_count == 0)
39236c6e
A		 1725 return;
1726
3e170ce0
A		 1727 if (c_minor_count || COMPRESSOR_NEEDS_TO_MAJOR_COMPACT()) {
1728
1729 lck_mtx_lock_spin_always(c_list_lock);
1730
1731 fastwake_warmup = FALSE;
1732
1733 if (compaction_swapper_running == 0) {
1734
1735 vm_wake_compactor_swapper_calls++;
1736
1737 thread_wakeup((event_t)&c_compressor_swap_trigger);
1738
1739 compaction_swapper_running = 1;
1740 }
1741 lck_mtx_unlock_always(c_list_lock);
1742 }
1743}
1744
1745
1746void
1747vm_consider_swapping()
1748{
1749 c_segment_t c_seg, c_seg_next;
1750 clock_sec_t now;
1751 clock_nsec_t nsec;
1752
39236c6e
A		 1753
1754 lck_mtx_lock_spin_always(c_list_lock);
1755
3e170ce0 1756 compaction_swapper_abort = 1;
39236c6e 1757
3e170ce0
A		 1758 while (compaction_swapper_running) {
1759 assert_wait((event_t)&compaction_swapper_running, THREAD_UNINT);
39236c6e 1760
3e170ce0 1761 lck_mtx_unlock_always(c_list_lock);
39236c6e 1762
3e170ce0
A		 1763 thread_block(THREAD_CONTINUE_NULL);
1764
1765 lck_mtx_lock_spin_always(c_list_lock);
1766 }
1767 compaction_swapper_abort = 0;
1768 compaction_swapper_running = 1;
1769
1770 vm_swapout_ripe_segments = TRUE;
1771
1772 if (!queue_empty(&c_major_list_head)) {
1773
1774 clock_get_system_nanotime(&now, &nsec);
1775
1776 c_seg = (c_segment_t)queue_first(&c_major_list_head);
1777
1778 while (!queue_end(&c_major_list_head, (queue_entry_t)c_seg)) {
1779
1780 if (c_overage_swapped_count >= c_overage_swapped_limit)
1781 break;
1782
1783 c_seg_next = (c_segment_t) queue_next(&c_seg->c_age_list);
1784
1785 if ((now - c_seg->c_creation_ts) >= vm_ripe_target_age) {
1786
1787 lck_mtx_lock_spin_always(&c_seg->c_lock);
1788
1789 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
1790
1791 lck_mtx_unlock_always(&c_seg->c_lock);
1792 }
1793 c_seg = c_seg_next;
1794 }
39236c6e 1795 }
3e170ce0
A		 1796 vm_compressor_compact_and_swap(FALSE);
1797
1798 compaction_swapper_running = 0;
1799
1800 vm_swapout_ripe_segments = FALSE;
1801
39236c6e
A		 1802 lck_mtx_unlock_always(c_list_lock);
1803}
1804
fe8ab488 1805
39236c6e
A		 1806void
1807vm_consider_waking_compactor_swapper(void)
1808{
1809 boolean_t need_wakeup = FALSE;
1810
fe8ab488 1811 if (compaction_swapper_running)
39236c6e 1812 return;
fe8ab488 1813
3e170ce0
A		 1814 if (c_segment_count == 0)
1815 return;
1816
fe8ab488
A		 1817 if (!compaction_swapper_inited && !compaction_swapper_init_now) {
1818 compaction_swapper_init_now = 1;
1819 need_wakeup = TRUE;
1820 }
39236c6e
A		 1821
1822 if (c_minor_count && (COMPRESSOR_NEEDS_TO_MINOR_COMPACT())) {
1823
1824 need_wakeup = TRUE;
1825
1826 } else if (compressor_needs_to_swap()) {
1827
1828 need_wakeup = TRUE;
1829
1830 } else if (c_minor_count) {
1831 uint64_t total_bytes;
1832
1833 total_bytes = compressor_object->resident_page_count * PAGE_SIZE_64;
1834
1835 if ((total_bytes - compressor_bytes_used) > total_bytes / 10)
1836 need_wakeup = TRUE;
1837 }
1838 if (need_wakeup == TRUE) {
1839
1840 lck_mtx_lock_spin_always(c_list_lock);
1841
1842 fastwake_warmup = FALSE;
1843
1844 if (compaction_swapper_running == 0) {
1845 memoryshot(VM_WAKEUP_COMPACTOR_SWAPPER, DBG_FUNC_NONE);
1846
1847 thread_wakeup((event_t)&c_compressor_swap_trigger);
1848
1849 compaction_swapper_running = 1;
1850 }
1851 lck_mtx_unlock_always(c_list_lock);
1852 }
1853}
1854
1855
1856#define C_SWAPOUT_LIMIT 4
1857#define DELAYED_COMPACTIONS_PER_PASS 30
1858
1859void
1860vm_compressor_do_delayed_compactions(boolean_t flush_all)
1861{
1862 c_segment_t c_seg;
1863 int number_compacted = 0;
1864 boolean_t needs_to_swap = FALSE;
1865
1866
1867 lck_mtx_assert(c_list_lock, LCK_MTX_ASSERT_OWNED);
1868
1869 while (!queue_empty(&c_minor_list_head) && needs_to_swap == FALSE) {
1870
1871 c_seg = (c_segment_t)queue_first(&c_minor_list_head);
1872
1873 lck_mtx_lock_spin_always(&c_seg->c_lock);
8a3053a0 1874
fe8ab488 1875 if (c_seg->c_busy) {
8a3053a0 1876
fe8ab488
A		 1877 lck_mtx_unlock_always(c_list_lock);
1878 c_seg_wait_on_busy(c_seg);
1879 lck_mtx_lock_spin_always(c_list_lock);
8a3053a0 1880
fe8ab488
A		 1881 continue;
1882 }
1883 C_SEG_BUSY(c_seg);
39236c6e
A		 1884
1885 c_seg_do_minor_compaction_and_unlock(c_seg, TRUE, FALSE, TRUE);
1886
1887 if (vm_swap_up == TRUE && (number_compacted++ > DELAYED_COMPACTIONS_PER_PASS)) {
1888
1889 if ((flush_all == TRUE || compressor_needs_to_swap() == TRUE) && c_swapout_count < C_SWAPOUT_LIMIT)
1890 needs_to_swap = TRUE;
1891
1892 number_compacted = 0;
1893 }
1894 lck_mtx_lock_spin_always(c_list_lock);
1895 }
1896}
1897
1898
1899#define C_SEGMENT_SWAPPEDIN_AGE_LIMIT 10
1900
1901static void
1902vm_compressor_age_swapped_in_segments(boolean_t flush_all)
1903{
1904 c_segment_t c_seg;
1905 clock_sec_t now;
1906 clock_nsec_t nsec;
1907
1908 clock_get_system_nanotime(&now, &nsec);
1909
1910 while (!queue_empty(&c_swappedin_list_head)) {
1911
1912 c_seg = (c_segment_t)queue_first(&c_swappedin_list_head);
1913
1914 if (flush_all == FALSE && (now - c_seg->c_swappedin_ts) < C_SEGMENT_SWAPPEDIN_AGE_LIMIT)
1915 break;
1916
1917 lck_mtx_lock_spin_always(&c_seg->c_lock);
1918
3e170ce0 1919 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
39236c6e
A		 1920
1921 lck_mtx_unlock_always(&c_seg->c_lock);
1922 }
1923}
1924
1925
1926void
1927vm_compressor_flush(void)
1928{
1929 uint64_t vm_swap_put_failures_at_start;
1930 wait_result_t wait_result = 0;
1931 AbsoluteTime startTime, endTime;
1932 clock_sec_t now_sec;
1933 clock_nsec_t now_nsec;
1934 uint64_t nsec;
1935
1936 HIBLOG("vm_compressor_flush - starting\n");
1937
1938 clock_get_uptime(&startTime);
1939
1940 lck_mtx_lock_spin_always(c_list_lock);
1941
1942 fastwake_warmup = FALSE;
1943 compaction_swapper_abort = 1;
1944
1945 while (compaction_swapper_running) {
1946 assert_wait((event_t)&compaction_swapper_running, THREAD_UNINT);
1947
1948 lck_mtx_unlock_always(c_list_lock);
1949
1950 thread_block(THREAD_CONTINUE_NULL);
1951
1952 lck_mtx_lock_spin_always(c_list_lock);
1953 }
1954 compaction_swapper_abort = 0;
1955 compaction_swapper_running = 1;
1956
1957 hibernate_flushing = TRUE;
1958 hibernate_no_swapspace = FALSE;
1959 c_generation_id_flush_barrier = c_generation_id + 1000;
1960
1961 clock_get_system_nanotime(&now_sec, &now_nsec);
1962 hibernate_flushing_deadline = now_sec + HIBERNATE_FLUSHING_SECS_TO_COMPLETE;
1963
1964 vm_swap_put_failures_at_start = vm_swap_put_failures;
1965
1966 vm_compressor_compact_and_swap(TRUE);
1967
1968 while (!queue_empty(&c_swapout_list_head)) {
1969
1970 assert_wait_timeout((event_t) &compaction_swapper_running, THREAD_INTERRUPTIBLE, 5000, 1000*NSEC_PER_USEC);
1971
1972 lck_mtx_unlock_always(c_list_lock);
1973
1974 wait_result = thread_block(THREAD_CONTINUE_NULL);
1975
1976 lck_mtx_lock_spin_always(c_list_lock);
1977
1978 if (wait_result == THREAD_TIMED_OUT)
1979 break;
1980 }
1981 hibernate_flushing = FALSE;
1982 compaction_swapper_running = 0;
1983
1984 if (vm_swap_put_failures > vm_swap_put_failures_at_start)
1985 HIBLOG("vm_compressor_flush failed to clean %llu segments - vm_page_compressor_count(%d)\n",
1986 vm_swap_put_failures - vm_swap_put_failures_at_start, VM_PAGE_COMPRESSOR_COUNT);
1987
1988 lck_mtx_unlock_always(c_list_lock);
1989
1990 clock_get_uptime(&endTime);
1991 SUB_ABSOLUTETIME(&endTime, &startTime);
1992 absolutetime_to_nanoseconds(endTime, &nsec);
1993
1994 HIBLOG("vm_compressor_flush completed - took %qd msecs\n", nsec / 1000000ULL);
1995}
1996
1997
fe8ab488 1998extern void vm_swap_file_set_tuneables(void);
39236c6e
A		 1999int compaction_swap_trigger_thread_awakened = 0;
2000
fe8ab488 2001
39236c6e
A		 2002static void
2003vm_compressor_swap_trigger_thread(void)
2004{
3e170ce0
A		 2005 current_thread()->options |= TH_OPT_VMPRIV;
2006
fe8ab488
A		 2007 /*
2008 * compaction_swapper_init_now is set when the first call to
2009 * vm_consider_waking_compactor_swapper is made from
2010 * vm_pageout_scan... since this function is called upon
2011 * thread creation, we want to make sure to delay adjusting
2012 * the tuneables until we are awakened via vm_pageout_scan
2013 * so that we are at a point where the vm_swapfile_open will
2014 * be operating on the correct directory (in case the default
2015 * of /var/vm/ is overridden by the dymanic_pager
2016 */
2017 if (compaction_swapper_init_now && !compaction_swapper_inited) {
2018 if (vm_compressor_mode == VM_PAGER_COMPRESSOR_WITH_SWAP)
2019 vm_swap_file_set_tuneables();
39236c6e 2020
3e170ce0
A		 2021 if (vm_restricted_to_single_processor == TRUE)
2022 thread_vm_bind_group_add();
2023
fe8ab488
A		 2024 compaction_swapper_inited = 1;
2025 }
39236c6e
A		 2026 lck_mtx_lock_spin_always(c_list_lock);
2027
2028 compaction_swap_trigger_thread_awakened++;
2029
2030 vm_compressor_compact_and_swap(FALSE);
2031
2032 assert_wait((event_t)&c_compressor_swap_trigger, THREAD_UNINT);
2033
2034 compaction_swapper_running = 0;
2035 thread_wakeup((event_t)&compaction_swapper_running);
2036
2037 lck_mtx_unlock_always(c_list_lock);
2038
2039 thread_block((thread_continue_t)vm_compressor_swap_trigger_thread);
2040
2041 /* NOTREACHED */
2042}
2043
2044
2045void
2046vm_compressor_record_warmup_start(void)
2047{
2048 c_segment_t c_seg;
2049
2050 lck_mtx_lock_spin_always(c_list_lock);
2051
8a3053a0
A		 2052 if (first_c_segment_to_warm_generation_id == 0) {
2053 if (!queue_empty(&c_age_list_head)) {
39236c6e 2054
8a3053a0 2055 c_seg = (c_segment_t)queue_last(&c_age_list_head);
39236c6e 2056
8a3053a0
A		 2057 first_c_segment_to_warm_generation_id = c_seg->c_generation_id;
2058 } else
2059 first_c_segment_to_warm_generation_id = 0;
39236c6e 2060
8a3053a0
A		 2061 fastwake_recording_in_progress = TRUE;
2062 }
39236c6e
A		 2063 lck_mtx_unlock_always(c_list_lock);
2064}
2065
2066
2067void
2068vm_compressor_record_warmup_end(void)
2069{
2070 c_segment_t c_seg;
2071
2072 lck_mtx_lock_spin_always(c_list_lock);
2073
8a3053a0 2074 if (fastwake_recording_in_progress == TRUE) {
39236c6e 2075
8a3053a0 2076 if (!queue_empty(&c_age_list_head)) {
39236c6e 2077
8a3053a0
A		 2078 c_seg = (c_segment_t)queue_last(&c_age_list_head);
2079
2080 last_c_segment_to_warm_generation_id = c_seg->c_generation_id;
2081 } else
2082 last_c_segment_to_warm_generation_id = first_c_segment_to_warm_generation_id;
39236c6e 2083
8a3053a0 2084 fastwake_recording_in_progress = FALSE;
39236c6e 2085
8a3053a0
A		 2086 HIBLOG("vm_compressor_record_warmup (%qd - %qd)\n", first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id);
2087 }
39236c6e
A		 2088 lck_mtx_unlock_always(c_list_lock);
2089}
2090
2091
2092#define DELAY_TRIM_ON_WAKE_SECS 4
2093
2094void
8a3053a0 2095vm_compressor_delay_trim(void)
39236c6e 2096{
8a3053a0 2097 clock_sec_t sec;
39236c6e
A		 2098 clock_nsec_t nsec;
2099
2100 clock_get_system_nanotime(&sec, &nsec);
2101 dont_trim_until_ts = sec + DELAY_TRIM_ON_WAKE_SECS;
8a3053a0 2102}
39236c6e 2103
39236c6e 2104
8a3053a0
A		 2105void
2106vm_compressor_do_warmup(void)
2107{
39236c6e
A		 2108 lck_mtx_lock_spin_always(c_list_lock);
2109
8a3053a0
A		 2110 if (first_c_segment_to_warm_generation_id == last_c_segment_to_warm_generation_id) {
2111 first_c_segment_to_warm_generation_id = last_c_segment_to_warm_generation_id = 0;
2112
2113 lck_mtx_unlock_always(c_list_lock);
2114 return;
2115 }
2116
39236c6e
A		 2117 if (compaction_swapper_running == 0) {
2118
2119 fastwake_warmup = TRUE;
2120 compaction_swapper_running = 1;
2121 thread_wakeup((event_t)&c_compressor_swap_trigger);
2122 }
2123 lck_mtx_unlock_always(c_list_lock);
2124}
2125
2126
2127void
2128do_fastwake_warmup(void)
2129{
2130 uint64_t my_thread_id;
2131 c_segment_t c_seg = NULL;
8a3053a0
A		 2132 AbsoluteTime startTime, endTime;
2133 uint64_t nsec;
2134
2135
2136 HIBLOG("vm_compressor_fastwake_warmup (%qd - %qd) - starting\n", first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id);
2137
2138 clock_get_uptime(&startTime);
39236c6e
A		 2139
2140 lck_mtx_unlock_always(c_list_lock);
2141
2142 my_thread_id = current_thread()->thread_id;
2143 proc_set_task_policy_thread(kernel_task, my_thread_id,
2144 TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER2);
2145
2146 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2147
2148 lck_mtx_lock_spin_always(c_list_lock);
2149
2150 while (!queue_empty(&c_swappedout_list_head) && fastwake_warmup == TRUE) {
2151
2152 c_seg = (c_segment_t) queue_first(&c_swappedout_list_head);
2153
2154 if (c_seg->c_generation_id < first_c_segment_to_warm_generation_id ||
2155 c_seg->c_generation_id > last_c_segment_to_warm_generation_id)
2156 break;
2157
4bd07ac2
A		 2158 if (vm_page_free_count < (AVAILABLE_MEMORY / 4))
2159 break;
2160
39236c6e
A		 2161 lck_mtx_lock_spin_always(&c_seg->c_lock);
2162 lck_mtx_unlock_always(c_list_lock);
2163
8a3053a0
A		 2164 if (c_seg->c_busy) {
2165 PAGE_REPLACEMENT_DISALLOWED(FALSE);
39236c6e 2166 c_seg_wait_on_busy(c_seg);
8a3053a0
A		 2167 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2168 } else {
39236c6e
A		 2169 c_seg_swapin(c_seg, TRUE);
2170
2171 lck_mtx_unlock_always(&c_seg->c_lock);
39236c6e 2172 c_segment_warmup_count++;
8a3053a0
A		 2173
2174 PAGE_REPLACEMENT_DISALLOWED(FALSE);
39236c6e 2175 vm_pageout_io_throttle();
8a3053a0 2176 PAGE_REPLACEMENT_DISALLOWED(TRUE);
39236c6e
A		 2177 }
2178 lck_mtx_lock_spin_always(c_list_lock);
2179 }
2180 lck_mtx_unlock_always(c_list_lock);
2181
2182 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2183
2184 proc_set_task_policy_thread(kernel_task, my_thread_id,
2185 TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER0);
2186
8a3053a0
A		 2187 clock_get_uptime(&endTime);
2188 SUB_ABSOLUTETIME(&endTime, &startTime);
2189 absolutetime_to_nanoseconds(endTime, &nsec);
2190
2191 HIBLOG("vm_compressor_fastwake_warmup completed - took %qd msecs\n", nsec / 1000000ULL);
2192
39236c6e 2193 lck_mtx_lock_spin_always(c_list_lock);
8a3053a0
A		 2194
2195 first_c_segment_to_warm_generation_id = last_c_segment_to_warm_generation_id = 0;
39236c6e
A		 2196}
2197
2198
2199void
2200vm_compressor_compact_and_swap(boolean_t flush_all)
2201{
2202 c_segment_t c_seg, c_seg_next;
2203 boolean_t keep_compacting;
3e170ce0
A		 2204 clock_sec_t now;
2205 clock_nsec_t nsec;
39236c6e
A		 2206
2207
2208 if (fastwake_warmup == TRUE) {
2209 uint64_t starting_warmup_count;
2210
2211 starting_warmup_count = c_segment_warmup_count;
2212
2213 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 11) | DBG_FUNC_START, c_segment_warmup_count,
2214 first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id, 0, 0);
2215 do_fastwake_warmup();
2216 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 11) | DBG_FUNC_END, c_segment_warmup_count, c_segment_warmup_count - starting_warmup_count, 0, 0, 0);
2217
2218 fastwake_warmup = FALSE;
2219 }
2220
8a3053a0
A		 2221 /*
2222 * it's possible for the c_age_list_head to be empty if we
2223 * hit our limits for growing the compressor pool and we subsequently
2224 * hibernated... on the next hibernation we could see the queue as
2225 * empty and not proceeed even though we have a bunch of segments on
2226 * the swapped in queue that need to be dealt with.
2227 */
2228 vm_compressor_do_delayed_compactions(flush_all);
2229
2230 vm_compressor_age_swapped_in_segments(flush_all);
2231
3e170ce0
A		 2232 /*
2233 * we only need to grab the timestamp once per
2234 * invocation of this function since the
2235 * timescale we're interested in is measured
2236 * in days
2237 */
2238 clock_get_system_nanotime(&now, &nsec);
8a3053a0 2239
39236c6e
A		 2240 while (!queue_empty(&c_age_list_head) && compaction_swapper_abort == 0) {
2241
2242 if (hibernate_flushing == TRUE) {
2243 clock_sec_t sec;
39236c6e
A		 2244
2245 if (hibernate_should_abort()) {
2246 HIBLOG("vm_compressor_flush - hibernate_should_abort returned TRUE\n");
2247 break;
2248 }
2249 if (hibernate_no_swapspace == TRUE) {
2250 HIBLOG("vm_compressor_flush - out of swap space\n");
2251 break;
2252 }
2253 clock_get_system_nanotime(&sec, &nsec);
2254
2255 if (sec > hibernate_flushing_deadline) {
2256 HIBLOG("vm_compressor_flush - failed to finish before deadline\n");
2257 break;
2258 }
2259 }
2260 if (c_swapout_count >= C_SWAPOUT_LIMIT) {
2261
2262 assert_wait_timeout((event_t) &compaction_swapper_running, THREAD_INTERRUPTIBLE, 100, 1000*NSEC_PER_USEC);
2263
2264 lck_mtx_unlock_always(c_list_lock);
2265
2266 thread_block(THREAD_CONTINUE_NULL);
2267
2268 lck_mtx_lock_spin_always(c_list_lock);
2269 }
2270 /*
2271 * Minor compactions
2272 */
2273 vm_compressor_do_delayed_compactions(flush_all);
2274
2275 vm_compressor_age_swapped_in_segments(flush_all);
2276
2277 if (c_swapout_count >= C_SWAPOUT_LIMIT) {
2278 /*
2279 * we timed out on the above thread_block
2280 * let's loop around and try again
2281 * the timeout allows us to continue
2282 * to do minor compactions to make
2283 * more memory available
2284 */
2285 continue;
2286 }
2287
2288 /*
2289 * Swap out segments?
2290 */
2291 if (flush_all == FALSE) {
2292 boolean_t needs_to_swap;
2293
2294 lck_mtx_unlock_always(c_list_lock);
2295
2296 needs_to_swap = compressor_needs_to_swap();
2297
3e170ce0
A		 2298 if (needs_to_swap == TRUE && vm_swap_low_on_space())
2299 vm_compressor_take_paging_space_action();
2300
39236c6e
A		 2301 lck_mtx_lock_spin_always(c_list_lock);
2302
2303 if (needs_to_swap == FALSE)
2304 break;
2305 }
2306 if (queue_empty(&c_age_list_head))
2307 break;
2308 c_seg = (c_segment_t) queue_first(&c_age_list_head);
2309
3e170ce0 2310 assert(c_seg->c_state == C_ON_AGE_Q);
39236c6e 2311
3e170ce0 2312 if (flush_all == TRUE && c_seg->c_generation_id > c_generation_id_flush_barrier)
39236c6e 2313 break;
3e170ce0 2314
39236c6e
A		 2315 lck_mtx_lock_spin_always(&c_seg->c_lock);
2316
2317 if (c_seg->c_busy) {
2318
2319 lck_mtx_unlock_always(c_list_lock);
2320 c_seg_wait_on_busy(c_seg);
2321 lck_mtx_lock_spin_always(c_list_lock);
2322
2323 continue;
2324 }
fe8ab488 2325 C_SEG_BUSY(c_seg);
39236c6e
A		 2326
2327 if (c_seg_do_minor_compaction_and_unlock(c_seg, FALSE, TRUE, TRUE)) {
2328 /*
2329 * found an empty c_segment and freed it
2330 * so go grab the next guy in the queue
2331 */
3e170ce0 2332 c_seg_major_compact_stats.count_of_freed_segs++;
39236c6e
A		 2333 continue;
2334 }
2335 /*
2336 * Major compaction
2337 */
2338 keep_compacting = TRUE;
2339
2340 while (keep_compacting == TRUE) {
2341
2342 assert(c_seg->c_busy);
2343
2344 /* look for another segment to consolidate */
2345
2346 c_seg_next = (c_segment_t) queue_next(&c_seg->c_age_list);
2347
2348 if (queue_end(&c_age_list_head, (queue_entry_t)c_seg_next))
2349 break;
2350
3e170ce0
A		 2351 assert(c_seg_next->c_state == C_ON_AGE_Q);
2352
39236c6e
A		 2353 if (c_seg_major_compact_ok(c_seg, c_seg_next) == FALSE)
2354 break;
2355
2356 lck_mtx_lock_spin_always(&c_seg_next->c_lock);
2357
2358 if (c_seg_next->c_busy) {
2359
2360 lck_mtx_unlock_always(c_list_lock);
2361 c_seg_wait_on_busy(c_seg_next);
2362 lck_mtx_lock_spin_always(c_list_lock);
2363
2364 continue;
2365 }
2366 /* grab that segment */
fe8ab488 2367 C_SEG_BUSY(c_seg_next);
39236c6e
A		 2368
2369 if (c_seg_do_minor_compaction_and_unlock(c_seg_next, FALSE, TRUE, TRUE)) {
2370 /*
2371 * found an empty c_segment and freed it
2372 * so we can't continue to use c_seg_next
2373 */
3e170ce0 2374 c_seg_major_compact_stats.count_of_freed_segs++;
39236c6e
A		 2375 continue;
2376 }
2377
2378 /* unlock the list ... */
2379 lck_mtx_unlock_always(c_list_lock);
2380
2381 /* do the major compaction */
2382
2383 keep_compacting = c_seg_major_compact(c_seg, c_seg_next);
2384
2385 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2386
2387 lck_mtx_lock_spin_always(&c_seg_next->c_lock);
2388 /*
2389 * run a minor compaction on the donor segment
2390 * since we pulled at least some of it's
2391 * data into our target... if we've emptied
2392 * it, now is a good time to free it which
2393 * c_seg_minor_compaction_and_unlock also takes care of
2394 *
2395 * by passing TRUE, we ask for c_busy to be cleared
2396 * and c_wanted to be taken care of
2397 */
3e170ce0
A		 2398 if (c_seg_minor_compaction_and_unlock(c_seg_next, TRUE))
2399 c_seg_major_compact_stats.count_of_freed_segs++;
39236c6e
A		 2400
2401 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2402
2403 /* relock the list */
2404 lck_mtx_lock_spin_always(c_list_lock);
2405
2406 } /* major compaction */
2407
39236c6e
A		 2408 lck_mtx_lock_spin_always(&c_seg->c_lock);
2409
2410 assert(c_seg->c_busy);
39236c6e
A		 2411 assert(!c_seg->c_on_minorcompact_q);
2412
39236c6e 2413 if (vm_swap_up == TRUE) {
3e170ce0
A		 2414 /*
2415 * This mode of putting a generic c_seg on the swapout list is
2416 * only supported when we have general swap ON i.e.
2417 * we compress pages into c_segs as we process them off
2418 * the paging queues in vm_pageout_scan().
2419 */
2420 if (COMPRESSED_PAGER_IS_SWAPBACKED)
2421 c_seg_switch_state(c_seg, C_ON_SWAPOUT_Q, FALSE);
2422 else {
2423 if ((vm_swapout_ripe_segments == TRUE && c_overage_swapped_count < c_overage_swapped_limit)) {
2424 /*
2425 * we are running compressor sweeps with swap-behind
2426 * make sure the c_seg has aged enough before swapping it
2427 * out...
2428 */
2429 if ((now - c_seg->c_creation_ts) >= vm_ripe_target_age) {
2430 c_seg->c_overage_swap = TRUE;
2431 c_overage_swapped_count++;
2432 c_seg_switch_state(c_seg, C_ON_SWAPOUT_Q, FALSE);
2433 }
2434 }
2435 }
2436 }
2437 if (c_seg->c_state == C_ON_AGE_Q) {
2438 /*
2439 * this c_seg didn't get moved to the swapout queue
2440 * so we need to move it out of the way...
2441 * we just did a major compaction on it so put it
2442 * on that queue
2443 */
2444 c_seg_switch_state(c_seg, C_ON_MAJORCOMPACT_Q, FALSE);
39236c6e 2445 } else {
3e170ce0
A		 2446 c_seg_major_compact_stats.wasted_space_in_swapouts += C_SEG_BUFSIZE - c_seg->c_bytes_used;
2447 c_seg_major_compact_stats.count_of_swapouts++;
39236c6e
A		 2448 }
2449 C_SEG_WAKEUP_DONE(c_seg);
2450
2451 lck_mtx_unlock_always(&c_seg->c_lock);
2452
2453 if (c_swapout_count) {
2454 lck_mtx_unlock_always(c_list_lock);
2455
2456 thread_wakeup((event_t)&c_swapout_list_head);
2457
2458 lck_mtx_lock_spin_always(c_list_lock);
2459 }
2460 }
2461}
2462
2463
2464static c_segment_t
2465c_seg_allocate(c_segment_t *current_chead)
2466{
39236c6e 2467 c_segment_t c_seg;
3e170ce0
A		 2468 int min_needed;
2469 int size_to_populate;
2470
2471 if (vm_compressor_low_on_space())
2472 vm_compressor_take_paging_space_action();
39236c6e
A		 2473
2474 if ( (c_seg = *current_chead) == NULL ) {
2475 uint32_t c_segno;
2476
39236c6e
A		 2477 lck_mtx_lock_spin_always(c_list_lock);
2478
2479 while (c_segments_busy == TRUE) {
2480 assert_wait((event_t) (&c_segments_busy), THREAD_UNINT);
2481
2482 lck_mtx_unlock_always(c_list_lock);
2483
2484 thread_block(THREAD_CONTINUE_NULL);
2485
2486 lck_mtx_lock_spin_always(c_list_lock);
2487 }
2488 if (c_free_segno_head == (uint32_t)-1) {
3e170ce0 2489 uint32_t c_segments_available_new;
39236c6e
A		 2490
2491 if (c_segments_available >= c_segments_limit || c_segment_pages_compressed >= c_segment_pages_compressed_limit) {
2492 lck_mtx_unlock_always(c_list_lock);
2493
39236c6e
A		 2494 return (NULL);
2495 }
2496 c_segments_busy = TRUE;
2497 lck_mtx_unlock_always(c_list_lock);
2498
3e170ce0
A		 2499 kernel_memory_populate(kernel_map, (vm_offset_t)c_segments_next_page,
2500 PAGE_SIZE, KMA_KOBJECT, VM_KERN_MEMORY_COMPRESSOR);
39236c6e
A		 2501 c_segments_next_page += PAGE_SIZE;
2502
3e170ce0
A		 2503 c_segments_available_new = c_segments_available + C_SEGMENTS_PER_PAGE;
2504
2505 if (c_segments_available_new > c_segments_limit)
2506 c_segments_available_new = c_segments_limit;
2507
2508 for (c_segno = c_segments_available + 1; c_segno < c_segments_available_new; c_segno++)
39236c6e
A		 2509 c_segments[c_segno - 1].c_segno = c_segno;
2510
2511 lck_mtx_lock_spin_always(c_list_lock);
2512
2513 c_segments[c_segno - 1].c_segno = c_free_segno_head;
2514 c_free_segno_head = c_segments_available;
3e170ce0 2515 c_segments_available = c_segments_available_new;
39236c6e
A		 2516
2517 c_segments_busy = FALSE;
2518 thread_wakeup((event_t) (&c_segments_busy));
2519 }
2520 c_segno = c_free_segno_head;
3e170ce0
A		 2521 assert(c_segno >= 0 && c_segno < c_segments_limit);
2522
39236c6e
A		 2523 c_free_segno_head = c_segments[c_segno].c_segno;
2524
3e170ce0
A		 2525 /*
2526 * do the rest of the bookkeeping now while we're still behind
2527 * the list lock and grab our generation id now into a local
2528 * so that we can install it once we have the c_seg allocated
2529 */
2530 c_segment_count++;
2531 if (c_segment_count > c_segment_count_max)
2532 c_segment_count_max = c_segment_count;
2533
39236c6e
A		 2534 lck_mtx_unlock_always(c_list_lock);
2535
2536 c_seg = (c_segment_t)zalloc(compressor_segment_zone);
2537 bzero((char *)c_seg, sizeof(struct c_segment));
2538
3e170ce0 2539 c_seg->c_store.c_buffer = (int32_t *)C_SEG_BUFFER_ADDRESS(c_segno);
39236c6e
A		 2540
2541#if __i386__ || __x86_64__
2542 lck_mtx_init(&c_seg->c_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
2543#else /* __i386__ || __x86_64__ */
2544 lck_spin_init(&c_seg->c_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
2545#endif /* __i386__ || __x86_64__ */
2546
3e170ce0
A		 2547 c_seg->c_state = C_IS_EMPTY;
2548 c_seg->c_firstemptyslot = C_SLOT_MAX_INDEX;
39236c6e 2549 c_seg->c_mysegno = c_segno;
39236c6e
A		 2550
2551 lck_mtx_lock_spin_always(c_list_lock);
3e170ce0
A		 2552 c_empty_count++;
2553 c_seg_switch_state(c_seg, C_IS_FILLING, FALSE);
39236c6e 2554 c_segments[c_segno].c_seg = c_seg;
39236c6e
A		 2555 lck_mtx_unlock_always(c_list_lock);
2556
39236c6e 2557 *current_chead = c_seg;
39236c6e 2558 }
3e170ce0
A		 2559 c_seg_alloc_nextslot(c_seg);
2560
2561 size_to_populate = C_SEG_ALLOCSIZE - C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset);
2562
2563 if (size_to_populate) {
2564
2565 min_needed = PAGE_SIZE + (C_SEG_ALLOCSIZE - C_SEG_BUFSIZE);
39236c6e 2566
3e170ce0 2567 if (C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset - c_seg->c_nextoffset) < (unsigned) min_needed) {
39236c6e 2568
3e170ce0
A		 2569 if (size_to_populate > C_SEG_MAX_POPULATE_SIZE)
2570 size_to_populate = C_SEG_MAX_POPULATE_SIZE;
39236c6e 2571
3e170ce0
A		 2572 kernel_memory_populate(kernel_map,
2573 (vm_offset_t) &c_seg->c_store.c_buffer[c_seg->c_populated_offset],
2574 size_to_populate,
2575 KMA_COMPRESSOR,
2576 VM_KERN_MEMORY_COMPRESSOR);
2577 } else
2578 size_to_populate = 0;
39236c6e 2579 }
39236c6e
A		 2580 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2581
2582 lck_mtx_lock_spin_always(&c_seg->c_lock);
2583
3e170ce0
A		 2584 if (size_to_populate)
2585 c_seg->c_populated_offset += C_SEG_BYTES_TO_OFFSET(size_to_populate);
2586
39236c6e
A		 2587 return (c_seg);
2588}
2589
2590
39236c6e
A		 2591static void
2592c_current_seg_filled(c_segment_t c_seg, c_segment_t *current_chead)
2593{
2594 uint32_t unused_bytes;
2595 uint32_t offset_to_depopulate;
3e170ce0
A		 2596 int new_state = C_ON_AGE_Q;
2597 clock_sec_t sec;
2598 clock_nsec_t nsec;
39236c6e
A		 2599
2600 unused_bytes = trunc_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset - c_seg->c_nextoffset));
2601
2602 if (unused_bytes) {
2603
2604 offset_to_depopulate = C_SEG_BYTES_TO_OFFSET(round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_nextoffset)));
2605
2606 /*
2607 * release the extra physical page(s) at the end of the segment
2608 */
2609 lck_mtx_unlock_always(&c_seg->c_lock);
2610
2611 kernel_memory_depopulate(
2612 kernel_map,
2613 (vm_offset_t) &c_seg->c_store.c_buffer[offset_to_depopulate],
2614 unused_bytes,
2615 KMA_COMPRESSOR);
2616
2617 lck_mtx_lock_spin_always(&c_seg->c_lock);
2618
2619 c_seg->c_populated_offset = offset_to_depopulate;
2620 }
3e170ce0 2621 assert(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset) <= C_SEG_BUFSIZE);
39236c6e 2622
3e170ce0
A		 2623#if CONFIG_FREEZE
2624 if (current_chead == (c_segment_t*)&freezer_chead && DEFAULT_FREEZER_COMPRESSED_PAGER_IS_SWAPBACKED &&
2625 c_freezer_swapout_count < VM_MAX_FREEZER_CSEG_SWAP_COUNT) {
2626 new_state = C_ON_SWAPOUT_Q;
2627 }
2628#endif /* CONFIG_FREEZE */
39236c6e 2629
3e170ce0
A		 2630 clock_get_system_nanotime(&sec, &nsec);
2631 c_seg->c_creation_ts = (uint32_t)sec;
2632
2633 lck_mtx_lock_spin_always(c_list_lock);
2634
2635#if CONFIG_FREEZE
2636 if (c_seg->c_state == C_ON_SWAPOUT_Q)
2637 c_freezer_swapout_count++;
2638#endif /* CONFIG_FREEZE */
2639
2640 c_seg->c_generation_id = c_generation_id++;
2641 c_seg_switch_state(c_seg, new_state, FALSE);
2642
2643 lck_mtx_unlock_always(c_list_lock);
2644
2645#if CONFIG_FREEZE
2646 if (c_seg->c_state == C_ON_SWAPOUT_Q)
2647 thread_wakeup((event_t)&c_swapout_list_head);
2648#endif /* CONFIG_FREEZE */
2649
2650 if (c_seg->c_state == C_ON_AGE_Q && C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE)
2651 c_seg_need_delayed_compaction(c_seg);
39236c6e
A		 2652
2653 *current_chead = NULL;
2654}
2655
39236c6e
A		 2656/*
2657 * returns with c_seg locked
2658 */
2659void
3e170ce0 2660c_seg_swapin_requeue(c_segment_t c_seg, boolean_t has_data)
39236c6e
A		 2661{
2662 clock_sec_t sec;
2663 clock_nsec_t nsec;
2664
2665 clock_get_system_nanotime(&sec, &nsec);
2666
2667 lck_mtx_lock_spin_always(c_list_lock);
2668 lck_mtx_lock_spin_always(&c_seg->c_lock);
2669
3e170ce0
A		 2670 c_seg->c_busy_swapping = 0;
2671
2672 if (c_seg->c_overage_swap == TRUE) {
2673 c_overage_swapped_count--;
2674 c_seg->c_overage_swap = FALSE;
2675 }
2676 if (has_data == TRUE) {
2677 c_seg_switch_state(c_seg, C_ON_SWAPPEDIN_Q, FALSE);
39236c6e 2678 } else {
3e170ce0
A		 2679 c_seg->c_store.c_buffer = (int32_t*) NULL;
2680 c_seg->c_populated_offset = C_SEG_BYTES_TO_OFFSET(0);
39236c6e 2681
3e170ce0 2682 c_seg_switch_state(c_seg, C_ON_BAD_Q, FALSE);
39236c6e 2683 }
39236c6e 2684 c_seg->c_swappedin_ts = (uint32_t)sec;
39236c6e
A		 2685
2686 lck_mtx_unlock_always(c_list_lock);
2687}
2688
2689
2690
2691/*
2692 * c_seg has to be locked and is returned locked.
2693 * PAGE_REPLACMENT_DISALLOWED has to be TRUE on entry and is returned TRUE
2694 */
2695
2696void
2697c_seg_swapin(c_segment_t c_seg, boolean_t force_minor_compaction)
2698{
2699 vm_offset_t addr = 0;
2700 uint32_t io_size = 0;
2701 uint64_t f_offset;
2702
3e170ce0
A		 2703 assert(C_SEG_IS_ONDISK(c_seg));
2704
39236c6e 2705#if !CHECKSUM_THE_SWAP
3e170ce0 2706 c_seg_trim_tail(c_seg);
39236c6e
A		 2707#endif
2708 io_size = round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset));
2709 f_offset = c_seg->c_store.c_swap_handle;
fe8ab488
A		 2710
2711 C_SEG_BUSY(c_seg);
3e170ce0 2712 c_seg->c_busy_swapping = 1;
ecc0ceb4
A		 2713
2714 /*
2715 * This thread is likely going to block for I/O.
2716 * Make sure it is ready to run when the I/O completes because
2717 * it needs to clear the busy bit on the c_seg so that other
2718 * waiting threads can make progress too. To do that, boost
2719 * the rwlock_count so that the priority is boosted.
2720 */
2721 set_thread_rwlock_boost();
39236c6e 2722 lck_mtx_unlock_always(&c_seg->c_lock);
39236c6e 2723
3e170ce0 2724 PAGE_REPLACEMENT_DISALLOWED(FALSE);
39236c6e 2725
3e170ce0 2726 addr = (vm_offset_t)C_SEG_BUFFER_ADDRESS(c_seg->c_mysegno);
39236c6e 2727
3e170ce0 2728 kernel_memory_populate(kernel_map, addr, io_size, KMA_COMPRESSOR, VM_KERN_MEMORY_COMPRESSOR);
39236c6e 2729
3e170ce0
A		 2730 if (vm_swap_get(addr, f_offset, io_size) != KERN_SUCCESS) {
2731 PAGE_REPLACEMENT_DISALLOWED(TRUE);
39236c6e 2732
3e170ce0 2733 kernel_memory_depopulate(kernel_map, addr, io_size, KMA_COMPRESSOR);
39236c6e 2734
3e170ce0
A		 2735 c_seg_swapin_requeue(c_seg, FALSE);
2736 } else {
2737 c_seg->c_store.c_buffer = (int32_t*) addr;
fe8ab488 2738#if ENCRYPTED_SWAP
3e170ce0 2739 vm_swap_decrypt(c_seg);
fe8ab488 2740#endif /* ENCRYPTED_SWAP */
39236c6e
A		 2741
2742#if CHECKSUM_THE_SWAP
3e170ce0
A		 2743 if (c_seg->cseg_swap_size != io_size)
2744 panic("swapin size doesn't match swapout size");
39236c6e 2745
3e170ce0
A		 2746 if (c_seg->cseg_hash != hash_string((char*) c_seg->c_store.c_buffer, (int)io_size)) {
2747 panic("c_seg_swapin - Swap hash mismatch\n");
2748 }
39236c6e
A		 2749#endif /* CHECKSUM_THE_SWAP */
2750
3e170ce0 2751 PAGE_REPLACEMENT_DISALLOWED(TRUE);
39236c6e 2752
3e170ce0
A		 2753 if (force_minor_compaction == TRUE) {
2754 lck_mtx_lock_spin_always(&c_seg->c_lock);
39236c6e 2755
3e170ce0
A		 2756 c_seg_minor_compaction_and_unlock(c_seg, FALSE);
2757 }
2758 OSAddAtomic64(c_seg->c_bytes_used, &compressor_bytes_used);
2759
2760 c_seg_swapin_requeue(c_seg, TRUE);
2761 }
2762 C_SEG_WAKEUP_DONE(c_seg);
ecc0ceb4
A		 2763
2764 /*
2765 * Drop the rwlock_count so that the thread priority
2766 * is returned back to where it is supposed to be.
2767 */
2768 clear_thread_rwlock_boost();
3e170ce0
A		 2769}
2770
2771
2772static void
2773c_segment_sv_hash_drop_ref(int hash_indx)
2774{
2775 struct c_sv_hash_entry o_sv_he, n_sv_he;
2776
2777 while (1) {
2778
2779 o_sv_he.he_record = c_segment_sv_hash_table[hash_indx].he_record;
2780
2781 n_sv_he.he_ref = o_sv_he.he_ref - 1;
2782 n_sv_he.he_data = o_sv_he.he_data;
2783
2784 if (OSCompareAndSwap64((UInt64)o_sv_he.he_record, (UInt64)n_sv_he.he_record, (UInt64 *) &c_segment_sv_hash_table[hash_indx].he_record) == TRUE) {
2785 if (n_sv_he.he_ref == 0)
2786 OSAddAtomic(-1, &c_segment_svp_in_hash);
2787 break;
2788 }
2789 }
2790}
2791
2792
2793static int
2794c_segment_sv_hash_insert(uint32_t data)
2795{
2796 int hash_sindx;
2797 int misses;
2798 struct c_sv_hash_entry o_sv_he, n_sv_he;
2799 boolean_t got_ref = FALSE;
2800
2801 if (data == 0)
2802 OSAddAtomic(1, &c_segment_svp_zero_compressions);
2803 else
2804 OSAddAtomic(1, &c_segment_svp_nonzero_compressions);
2805
2806 hash_sindx = data & C_SV_HASH_MASK;
2807
2808 for (misses = 0; misses < C_SV_HASH_MAX_MISS; misses++)
2809 {
2810 o_sv_he.he_record = c_segment_sv_hash_table[hash_sindx].he_record;
2811
2812 while (o_sv_he.he_data == data || o_sv_he.he_ref == 0) {
2813 n_sv_he.he_ref = o_sv_he.he_ref + 1;
2814 n_sv_he.he_data = data;
2815
2816 if (OSCompareAndSwap64((UInt64)o_sv_he.he_record, (UInt64)n_sv_he.he_record, (UInt64 *) &c_segment_sv_hash_table[hash_sindx].he_record) == TRUE) {
2817 if (n_sv_he.he_ref == 1)
2818 OSAddAtomic(1, &c_segment_svp_in_hash);
2819 got_ref = TRUE;
2820 break;
39236c6e 2821 }
3e170ce0 2822 o_sv_he.he_record = c_segment_sv_hash_table[hash_sindx].he_record;
39236c6e 2823 }
3e170ce0
A		 2824 if (got_ref == TRUE)
2825 break;
2826 hash_sindx++;
2827
2828 if (hash_sindx == C_SV_HASH_SIZE)
2829 hash_sindx = 0;
39236c6e 2830 }
3e170ce0
A		 2831 if (got_ref == FALSE)
2832 return(-1);
39236c6e 2833
3e170ce0
A		 2834 return (hash_sindx);
2835}
2836
2837
2838#if RECORD_THE_COMPRESSED_DATA
2839
2840static void
2841c_compressed_record_data(char *src, int c_size)
2842{
2843 if ((c_compressed_record_cptr + c_size + 4) >= c_compressed_record_ebuf)
2844 panic("c_compressed_record_cptr >= c_compressed_record_ebuf");
2845
2846 *(int *)((void *)c_compressed_record_cptr) = c_size;
2847
2848 c_compressed_record_cptr += 4;
2849
2850 memcpy(c_compressed_record_cptr, src, c_size);
2851 c_compressed_record_cptr += c_size;
39236c6e 2852}
3e170ce0 2853#endif
39236c6e
A		 2854
2855
2856static int
2857c_compress_page(char *src, c_slot_mapping_t slot_ptr, c_segment_t *current_chead, char *scratch_buf)
2858{
2859 int c_size;
04b8595b 2860 int c_rounded_size = 0;
39236c6e
A		 2861 int max_csize;
2862 c_slot_t cs;
2863 c_segment_t c_seg;
2864
2865 KERNEL_DEBUG(0xe0400000 | DBG_FUNC_START, *current_chead, 0, 0, 0, 0);
2866retry:
2867 if ((c_seg = c_seg_allocate(current_chead)) == NULL)
2868 return (1);
2869 /*
2870 * returns with c_seg lock held
3e170ce0
A		 2871 * and PAGE_REPLACEMENT_DISALLOWED(TRUE)...
2872 * c_nextslot has been allocated and
2873 * c_store.c_buffer populated
39236c6e 2874 */
3e170ce0
A		 2875 assert(c_seg->c_state == C_IS_FILLING);
2876
39236c6e
A		 2877 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_seg->c_nextslot);
2878
2879 cs->c_packed_ptr = C_SLOT_PACK_PTR(slot_ptr);
fe8ab488
A		 2880 assert(slot_ptr == (c_slot_mapping_t)C_SLOT_UNPACK_PTR(cs));
2881
39236c6e
A		 2882 cs->c_offset = c_seg->c_nextoffset;
2883
2884 max_csize = C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES((int32_t)cs->c_offset);
2885
2886 if (max_csize > PAGE_SIZE)
2887 max_csize = PAGE_SIZE;
2888
39236c6e
A		 2889#if CHECKSUM_THE_DATA
2890 cs->c_hash_data = hash_string(src, PAGE_SIZE);
2891#endif
39236c6e 2892
3e170ce0 2893 c_size = WKdm_compress_new((const WK_word *)(uintptr_t)src, (WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
fe8ab488 2894 (WK_word *)(uintptr_t)scratch_buf, max_csize - 4);
39236c6e
A		 2895 assert(c_size <= (max_csize - 4) && c_size >= -1);
2896
2897 if (c_size == -1) {
2898
2899 if (max_csize < PAGE_SIZE) {
2900 c_current_seg_filled(c_seg, current_chead);
3e170ce0 2901 assert(*current_chead == NULL);
39236c6e 2902
3e170ce0 2903 lck_mtx_unlock_always(&c_seg->c_lock);
39236c6e 2904
3e170ce0 2905 PAGE_REPLACEMENT_DISALLOWED(FALSE);
39236c6e
A		 2906 goto retry;
2907 }
2908 c_size = PAGE_SIZE;
2909
2910 memcpy(&c_seg->c_store.c_buffer[cs->c_offset], src, c_size);
3e170ce0
A		 2911
2912 OSAddAtomic(1, &c_segment_noncompressible_pages);
2913
2914 } else if (c_size == 0) {
2915 int hash_index;
2916
2917 /*
2918 * special case - this is a page completely full of a single 32 bit value
2919 */
2920 hash_index = c_segment_sv_hash_insert(*(uint32_t *)(uintptr_t)src);
2921
2922 if (hash_index != -1) {
2923 slot_ptr->s_cindx = hash_index;
2924 slot_ptr->s_cseg = C_SV_CSEG_ID;
2925
2926 OSAddAtomic(1, &c_segment_svp_hash_succeeded);
2927#if RECORD_THE_COMPRESSED_DATA
2928 c_compressed_record_data(src, 4);
2929#endif
2930 goto sv_compression;
2931 }
2932 c_size = 4;
2933
2934 memcpy(&c_seg->c_store.c_buffer[cs->c_offset], src, c_size);
2935
2936 OSAddAtomic(1, &c_segment_svp_hash_failed);
39236c6e 2937 }
3e170ce0
A		 2938
2939#if RECORD_THE_COMPRESSED_DATA
2940 c_compressed_record_data((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size);
2941#endif
2942
39236c6e
A		 2943#if CHECKSUM_THE_COMPRESSED_DATA
2944 cs->c_hash_compressed_data = hash_string((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size);
2945#endif
2946 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
2947
2948 PACK_C_SIZE(cs, c_size);
2949 c_seg->c_bytes_used += c_rounded_size;
2950 c_seg->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
2951
2952 slot_ptr->s_cindx = c_seg->c_nextslot++;
2953 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
2954 slot_ptr->s_cseg = c_seg->c_mysegno + 1;
2955
3e170ce0
A		 2956sv_compression:
2957 if (c_seg->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg->c_nextslot >= C_SLOT_MAX_INDEX) {
39236c6e 2958 c_current_seg_filled(c_seg, current_chead);
3e170ce0
A		 2959 assert(*current_chead == NULL);
2960 }
2961 lck_mtx_unlock_always(&c_seg->c_lock);
39236c6e
A		 2962
2963 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2964
3e170ce0
A		 2965#if RECORD_THE_COMPRESSED_DATA
2966 if ((c_compressed_record_cptr - c_compressed_record_sbuf) >= C_SEG_ALLOCSIZE) {
2967 c_compressed_record_write(c_compressed_record_sbuf, (int)(c_compressed_record_cptr - c_compressed_record_sbuf));
2968 c_compressed_record_cptr = c_compressed_record_sbuf;
2969 }
2970#endif
2971 if (c_size) {
2972 OSAddAtomic64(c_size, &c_segment_compressed_bytes);
2973 OSAddAtomic64(c_rounded_size, &compressor_bytes_used);
2974 }
39236c6e 2975 OSAddAtomic64(PAGE_SIZE, &c_segment_input_bytes);
39236c6e
A		 2976
2977 OSAddAtomic(1, &c_segment_pages_compressed);
2978 OSAddAtomic(1, &sample_period_compression_count);
2979
2980 KERNEL_DEBUG(0xe0400000 | DBG_FUNC_END, *current_chead, c_size, c_segment_input_bytes, c_segment_compressed_bytes, 0);
2981
39236c6e
A		 2982 return (0);
2983}
2984
2985
2986static int
2987c_decompress_page(char *dst, volatile c_slot_mapping_t slot_ptr, int flags, int *zeroslot)
2988{
2989 c_slot_t cs;
2990 c_segment_t c_seg;
2991 int c_indx;
2992 int c_rounded_size;
2993 uint32_t c_size;
2994 int retval = 0;
39236c6e
A		 2995 boolean_t need_unlock = TRUE;
2996 boolean_t consider_defragmenting = FALSE;
3e170ce0
A		 2997 boolean_t kdp_mode = FALSE;
2998
2999 if (flags & C_KDP) {
3000 if (not_in_kdp) {
3001 panic("C_KDP passed to decompress page from outside of debugger context");
3002 }
3003
3004 assert((flags & C_KEEP) == C_KEEP);
3005 assert((flags & C_DONT_BLOCK) == C_DONT_BLOCK);
3006
3007 if ((flags & (C_DONT_BLOCK | C_KEEP)) != (C_DONT_BLOCK | C_KEEP)) {
3008 return (-2);
3009 }
3010
3011 kdp_mode = TRUE;
3012 }
39236c6e
A		 3013
3014ReTry:
3e170ce0
A		 3015 if (!kdp_mode) {
3016 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3017 } else {
3018 if (kdp_lck_rw_lock_is_acquired_exclusive(&c_master_lock)) {
3019 return (-2);
3020 }
3021 }
fe8ab488 3022
39236c6e 3023#if HIBERNATION
fe8ab488
A		 3024 /*
3025 * if hibernation is enabled, it indicates (via a call
3026 * to 'vm_decompressor_lock' that no further
3027 * decompressions are allowed once it reaches
3028 * the point of flushing all of the currently dirty
3029 * anonymous memory through the compressor and out
3030 * to disk... in this state we allow freeing of compressed
3031 * pages and must honor the C_DONT_BLOCK case
3032 */
3033 if (dst && decompressions_blocked == TRUE) {
3034 if (flags & C_DONT_BLOCK) {
3035
3e170ce0
A		 3036 if (!kdp_mode) {
3037 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3038 }
fe8ab488
A		 3039
3040 *zeroslot = 0;
3041 return (-2);
39236c6e 3042 }
fe8ab488
A		 3043 /*
3044 * it's safe to atomically assert and block behind the
3045 * lock held in shared mode because "decompressions_blocked" is
3046 * only set and cleared and the thread_wakeup done when the lock
3047 * is held exclusively
3048 */
3049 assert_wait((event_t)&decompressions_blocked, THREAD_UNINT);
3050
3051 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3052
3053 thread_block(THREAD_CONTINUE_NULL);
3054
3055 goto ReTry;
39236c6e
A		 3056 }
3057#endif
39236c6e
A		 3058 /* s_cseg is actually "segno+1" */
3059 c_seg = c_segments[slot_ptr->s_cseg - 1].c_seg;
3060
3e170ce0
A		 3061 if (!kdp_mode) {
3062 lck_mtx_lock_spin_always(&c_seg->c_lock);
3063 } else {
3064 if (kdp_lck_mtx_lock_spin_is_acquired(&c_seg->c_lock)) {
3065 return (-2);
3066 }
3067 }
39236c6e 3068
3e170ce0 3069 assert(c_seg->c_state != C_IS_EMPTY && c_seg->c_state != C_IS_FREE);
39236c6e 3070
3e170ce0
A		 3071 if (flags & C_DONT_BLOCK) {
3072 if (c_seg->c_busy || (C_SEG_IS_ONDISK(c_seg) && dst)) {
39236c6e
A		 3073 *zeroslot = 0;
3074
3e170ce0 3075 retval = -2;
39236c6e
A		 3076 goto done;
3077 }
3078 }
3079 if (c_seg->c_busy) {
3080
3081 PAGE_REPLACEMENT_DISALLOWED(FALSE);
fe8ab488 3082
39236c6e
A		 3083 c_seg_wait_on_busy(c_seg);
3084
3085 goto ReTry;
3086 }
3087 c_indx = slot_ptr->s_cindx;
3088
3089 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
3090
3091 c_size = UNPACK_C_SIZE(cs);
3092
3093 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
3094
3095 if (dst) {
3096 uint32_t age_of_cseg;
3097 clock_sec_t cur_ts_sec;
3098 clock_nsec_t cur_ts_nsec;
3099
3e170ce0
A		 3100 if (C_SEG_IS_ONDISK(c_seg)) {
3101 assert(kdp_mode == FALSE);
39236c6e 3102 c_seg_swapin(c_seg, FALSE);
3e170ce0
A		 3103
3104 retval = 1;
39236c6e 3105 }
3e170ce0
A		 3106 if (c_seg->c_state == C_ON_BAD_Q) {
3107 assert(c_seg->c_store.c_buffer == NULL);
3108
3109 retval = -1;
39236c6e
A		 3110 goto c_seg_invalid_data;
3111 }
3112#if CHECKSUM_THE_COMPRESSED_DATA
3113 if (cs->c_hash_compressed_data != hash_string((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size))
3114 panic("compressed data doesn't match original");
3115#endif
3116 if (c_rounded_size == PAGE_SIZE) {
3117 /*
3118 * page wasn't compressible... just copy it out
3119 */
3120 memcpy(dst, &c_seg->c_store.c_buffer[cs->c_offset], PAGE_SIZE);
3e170ce0
A		 3121 } else if (c_size == 4) {
3122 int32_t data;
3123 int32_t *dptr;
3124
3125 /*
3126 * page was populated with a single value
3127 * that didn't fit into our fast hash
3128 * so we packed it in as a single non-compressed value
3129 * that we need to populate the page with
3130 */
3131 dptr = (int32_t *)(uintptr_t)dst;
3132 data = *(int32_t *)(&c_seg->c_store.c_buffer[cs->c_offset]);
3133#if __x86_64__
3134 memset_word(dptr, data, PAGE_SIZE / sizeof(int32_t));
3135#else
3136 {
3137 int i;
3138
3139 for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++)
3140 *dptr++ = data;
3141 }
3142#endif
39236c6e
A		 3143 } else {
3144 uint32_t my_cpu_no;
3145 char *scratch_buf;
3146
3e170ce0
A		 3147 if (!kdp_mode) {
3148 /*
3149 * we're behind the c_seg lock held in spin mode
3150 * which means pre-emption is disabled... therefore
3151 * the following sequence is atomic and safe
3152 */
3153 my_cpu_no = cpu_number();
39236c6e 3154
3e170ce0 3155 assert(my_cpu_no < compressor_cpus);
39236c6e 3156
3e170ce0
A		 3157 scratch_buf = &compressor_scratch_bufs[my_cpu_no * WKdm_SCRATCH_BUF_SIZE];
3158 } else {
3159 scratch_buf = kdp_compressor_scratch_buf;
3160 }
39236c6e
A		 3161 WKdm_decompress_new((WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
3162 (WK_word *)(uintptr_t)dst, (WK_word *)(uintptr_t)scratch_buf, c_size);
3163 }
3164
3165#if CHECKSUM_THE_DATA
3166 if (cs->c_hash_data != hash_string(dst, PAGE_SIZE))
3167 panic("decompressed data doesn't match original");
3168#endif
3e170ce0 3169 if (c_seg->c_swappedin_ts == 0 && !kdp_mode) {
39236c6e
A		 3170
3171 clock_get_system_nanotime(&cur_ts_sec, &cur_ts_nsec);
3172
3173 age_of_cseg = (uint32_t)cur_ts_sec - c_seg->c_creation_ts;
3174
3175 if (age_of_cseg < DECOMPRESSION_SAMPLE_MAX_AGE)
3176 OSAddAtomic(1, &age_of_decompressions_during_sample_period[age_of_cseg]);
3177 else
3178 OSAddAtomic(1, &overage_decompressions_during_sample_period);
3179
3180 OSAddAtomic(1, &sample_period_decompression_count);
3181 }
39236c6e
A		 3182 }
3183c_seg_invalid_data:
3184
39236c6e
A		 3185 if (flags & C_KEEP) {
3186 *zeroslot = 0;
3187 goto done;
3188 }
3e170ce0
A		 3189
3190 assert(kdp_mode == FALSE);
39236c6e
A		 3191 c_seg->c_bytes_unused += c_rounded_size;
3192 c_seg->c_bytes_used -= c_rounded_size;
3193 PACK_C_SIZE(cs, 0);
3194
3195 if (c_indx < c_seg->c_firstemptyslot)
3196 c_seg->c_firstemptyslot = c_indx;
3197
3198 OSAddAtomic(-1, &c_segment_pages_compressed);
3199
3e170ce0 3200 if (c_seg->c_state != C_ON_BAD_Q && !(C_SEG_IS_ONDISK(c_seg))) {
39236c6e 3201 /*
3e170ce0 3202 * C_SEG_IS_ONDISK == TRUE can occur when we're doing a
39236c6e
A		 3203 * free of a compressed page (i.e. dst == NULL)
3204 */
3205 OSAddAtomic64(-c_rounded_size, &compressor_bytes_used);
3206 }
3e170ce0 3207 if (c_seg->c_state != C_IS_FILLING) {
39236c6e 3208 if (c_seg->c_bytes_used == 0) {
3e170ce0 3209 if ( !(C_SEG_IS_ONDISK(c_seg))) {
8a3053a0
A		 3210 int pages_populated;
3211
3212 pages_populated = (round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE;
3213 c_seg->c_populated_offset = C_SEG_BYTES_TO_OFFSET(0);
3214
3215 if (pages_populated) {
3e170ce0
A		 3216
3217 assert(c_seg->c_state != C_ON_BAD_Q);
8a3053a0
A		 3218 assert(c_seg->c_store.c_buffer != NULL);
3219
fe8ab488 3220 C_SEG_BUSY(c_seg);
8a3053a0
A		 3221 lck_mtx_unlock_always(&c_seg->c_lock);
3222
3223 kernel_memory_depopulate(kernel_map, (vm_offset_t) c_seg->c_store.c_buffer, pages_populated * PAGE_SIZE, KMA_COMPRESSOR);
3224
3225 lck_mtx_lock_spin_always(&c_seg->c_lock);
3226 C_SEG_WAKEUP_DONE(c_seg);
3227 }
3e170ce0 3228 if (!c_seg->c_on_minorcompact_q)
8a3053a0
A		 3229 c_seg_need_delayed_compaction(c_seg);
3230 } else
3e170ce0 3231 assert(c_seg->c_state == C_ON_SWAPPEDOUTSPARSE_Q);
8a3053a0 3232
39236c6e
A		 3233 } else if (c_seg->c_on_minorcompact_q) {
3234
3e170ce0
A		 3235 assert(c_seg->c_state != C_ON_BAD_Q);
3236
3237 if (C_SEG_SHOULD_MINORCOMPACT(c_seg)) {
39236c6e
A		 3238 c_seg_try_minor_compaction_and_unlock(c_seg);
3239 need_unlock = FALSE;
3240 }
3e170ce0 3241 } else if ( !(C_SEG_IS_ONDISK(c_seg))) {
39236c6e 3242
3e170ce0 3243 if (c_seg->c_state != C_ON_BAD_Q && c_seg->c_state != C_ON_SWAPOUT_Q && C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE) {
39236c6e
A		 3244 c_seg_need_delayed_compaction(c_seg);
3245 }
3e170ce0 3246 } else if (c_seg->c_state != C_ON_SWAPPEDOUTSPARSE_Q && C_SEG_ONDISK_IS_SPARSE(c_seg)) {
39236c6e
A		 3247
3248 c_seg_move_to_sparse_list(c_seg);
3249 consider_defragmenting = TRUE;
3250 }
3251 }
3252done:
3e170ce0
A		 3253 if (kdp_mode) {
3254 return retval;
3255 }
3256
39236c6e
A		 3257 if (need_unlock == TRUE)
3258 lck_mtx_unlock_always(&c_seg->c_lock);
3259
3260 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3261
3262 if (consider_defragmenting == TRUE)
3263 vm_swap_consider_defragmenting();
fe8ab488
A		 3264
3265
39236c6e
A		 3266 return (retval);
3267}
3268
3269
3270int
3271vm_compressor_get(ppnum_t pn, int *slot, int flags)
3272{
3e170ce0 3273 c_slot_mapping_t slot_ptr;
39236c6e
A		 3274 char *dst;
3275 int zeroslot = 1;
3276 int retval;
3277
3278#if __x86_64__
3279 dst = PHYSMAP_PTOV((uint64_t)pn << (uint64_t)PAGE_SHIFT);
3280#else
3281#error "unsupported architecture"
3282#endif
3e170ce0
A		 3283 slot_ptr = (c_slot_mapping_t)slot;
3284
3285 if (slot_ptr->s_cseg == C_SV_CSEG_ID) {
3286 int32_t data;
3287 int32_t *dptr;
3288
3289 /*
3290 * page was populated with a single value
3291 * that found a home in our hash table
3292 * grab that value from the hash and populate the page
3293 * that we need to populate the page with
3294 */
3295 dptr = (int32_t *)(uintptr_t)dst;
3296 data = c_segment_sv_hash_table[slot_ptr->s_cindx].he_data;
3297#if __x86_64__
3298 memset_word(dptr, data, PAGE_SIZE / sizeof(int32_t));
3299#else
3300 {
3301 int i;
3302
3303 for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++)
3304 *dptr++ = data;
3305 }
3306#endif
3307 c_segment_sv_hash_drop_ref(slot_ptr->s_cindx);
39236c6e 3308
3e170ce0
A		 3309 if ( !(flags & C_KEEP)) {
3310 OSAddAtomic(-1, &c_segment_pages_compressed);
3311 *slot = 0;
3312 }
3313 if (data)
3314 OSAddAtomic(1, &c_segment_svp_nonzero_decompressions);
3315 else
3316 OSAddAtomic(1, &c_segment_svp_zero_decompressions);
3317
3318 return (0);
3319 }
3320
3321 retval = c_decompress_page(dst, slot_ptr, flags, &zeroslot);
39236c6e
A		 3322
3323 /*
3324 * zeroslot will be set to 0 by c_decompress_page if (flags & C_KEEP)
3e170ce0 3325 * or (flags & C_DONT_BLOCK) and we found 'c_busy' or 'C_SEG_IS_ONDISK' to be TRUE
39236c6e
A		 3326 */
3327 if (zeroslot) {
39236c6e
A		 3328 *slot = 0;
3329 }
3330 /*
3331 * returns 0 if we successfully decompressed a page from a segment already in memory
3332 * returns 1 if we had to first swap in the segment, before successfully decompressing the page
3333 * returns -1 if we encountered an error swapping in the segment - decompression failed
3e170ce0 3334 * returns -2 if (flags & C_DONT_BLOCK) and we found 'c_busy' or 'C_SEG_IS_ONDISK' to be true
39236c6e
A		 3335 */
3336 return (retval);
3337}
3338
3339
fe8ab488
A		 3340int
3341vm_compressor_free(int *slot, int flags)
39236c6e 3342{
3e170ce0 3343 c_slot_mapping_t slot_ptr;
39236c6e 3344 int zeroslot = 1;
fe8ab488 3345 int retval;
39236c6e 3346
fe8ab488 3347 assert(flags == 0 || flags == C_DONT_BLOCK);
39236c6e 3348
3e170ce0
A		 3349 slot_ptr = (c_slot_mapping_t)slot;
3350
3351 if (slot_ptr->s_cseg == C_SV_CSEG_ID) {
3352
3353 c_segment_sv_hash_drop_ref(slot_ptr->s_cindx);
3354 OSAddAtomic(-1, &c_segment_pages_compressed);
3355
3356 *slot = 0;
3357 return (0);
3358 }
3359 retval = c_decompress_page(NULL, slot_ptr, flags, &zeroslot);
fe8ab488
A		 3360 /*
3361 * returns 0 if we successfully freed the specified compressed page
3362 * returns -2 if (flags & C_DONT_BLOCK) and we found 'c_busy' set
3363 */
3364
3365 if (retval == 0)
3366 *slot = 0;
3e170ce0
A		 3367 else
3368 assert(retval == -2);
fe8ab488
A		 3369
3370 return (retval);
39236c6e
A		 3371}
3372
3373
3374int
3375vm_compressor_put(ppnum_t pn, int *slot, void **current_chead, char *scratch_buf)
3376{
3377 char *src;
3378 int retval;
3379
39236c6e
A		 3380#if __x86_64__
3381 src = PHYSMAP_PTOV((uint64_t)pn << (uint64_t)PAGE_SHIFT);
3382#else
3383#error "unsupported architecture"
3384#endif
3385 retval = c_compress_page(src, (c_slot_mapping_t)slot, (c_segment_t *)current_chead, scratch_buf);
3386
3387 return (retval);
3388}
fe8ab488
A		 3389
3390void
3391vm_compressor_transfer(
3392 int *dst_slot_p,
3393 int *src_slot_p)
3394{
3395 c_slot_mapping_t dst_slot, src_slot;
3396 c_segment_t c_seg;
3397 int c_indx;
3398 c_slot_t cs;
3399
fe8ab488
A		 3400 src_slot = (c_slot_mapping_t) src_slot_p;
3401
3e170ce0
A		 3402 if (src_slot->s_cseg == C_SV_CSEG_ID) {
3403 *dst_slot_p = *src_slot_p;
3404 *src_slot_p = 0;
3405 return;
3406 }
3407 dst_slot = (c_slot_mapping_t) dst_slot_p;
fe8ab488
A		 3408Retry:
3409 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3410 /* get segment for src_slot */
3411 c_seg = c_segments[src_slot->s_cseg -1].c_seg;
3412 /* lock segment */
3413 lck_mtx_lock_spin_always(&c_seg->c_lock);
3414 /* wait if it's busy */
3e170ce0 3415 if (c_seg->c_busy && !c_seg->c_busy_swapping) {
fe8ab488
A		 3416 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3417 c_seg_wait_on_busy(c_seg);
3418 goto Retry;
3419 }
3420 /* find the c_slot */
3421 c_indx = src_slot->s_cindx;
3422 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
3423 /* point the c_slot back to dst_slot instead of src_slot */
3424 cs->c_packed_ptr = C_SLOT_PACK_PTR(dst_slot);
3425 /* transfer */
3426 *dst_slot_p = *src_slot_p;
3427 *src_slot_p = 0;
3428 lck_mtx_unlock_always(&c_seg->c_lock);
3429 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3430}
3e170ce0
A		 3431
3432#if CONFIG_FREEZE
3433
3434int freezer_finished_filling = 0;
3435
3436void
3437vm_compressor_finished_filling(
3438 void **current_chead)
3439{
3440 c_segment_t c_seg;
3441
3442 if ((c_seg = *(c_segment_t *)current_chead) == NULL)
3443 return;
3444
3445 assert(c_seg->c_state == C_IS_FILLING);
3446
3447 lck_mtx_lock_spin_always(&c_seg->c_lock);
3448
3449 c_current_seg_filled(c_seg, (c_segment_t *)current_chead);
3450
3451 lck_mtx_unlock_always(&c_seg->c_lock);
3452
3453 freezer_finished_filling++;
3454}
3455
3456
3457/*
3458 * This routine is used to transfer the compressed chunks from
3459 * the c_seg/cindx pointed to by slot_p into a new c_seg headed
3460 * by the current_chead and a new cindx within that c_seg.
3461 *
3462 * Currently, this routine is only used by the "freezer backed by
3463 * compressor with swap" mode to create a series of c_segs that
3464 * only contain compressed data belonging to one task. So, we
3465 * move a task's previously compressed data into a set of new
3466 * c_segs which will also hold the task's yet to be compressed data.
3467 */
3468
3469kern_return_t
3470vm_compressor_relocate(
3471 void **current_chead,
3472 int *slot_p)
3473{
3474 c_slot_mapping_t slot_ptr;
3475 c_slot_mapping_t src_slot;
3476 uint32_t c_rounded_size;
3477 uint32_t c_size;
3478 uint16_t dst_slot;
3479 c_slot_t c_dst;
3480 c_slot_t c_src;
3481 int c_indx;
3482 c_segment_t c_seg_dst = NULL;
3483 c_segment_t c_seg_src = NULL;
3484 kern_return_t kr = KERN_SUCCESS;
3485
3486
3487 src_slot = (c_slot_mapping_t) slot_p;
3488
3489 if (src_slot->s_cseg == C_SV_CSEG_ID) {
3490 /*
3491 * no need to relocate... this is a page full of a single
3492 * value which is hashed to a single entry not contained
3493 * in a c_segment_t
3494 */
3495 return (kr);
3496 }
3497
3498Relookup_dst:
3499 c_seg_dst = c_seg_allocate((c_segment_t *)current_chead);
3500 /*
3501 * returns with c_seg lock held
3502 * and PAGE_REPLACEMENT_DISALLOWED(TRUE)...
3503 * c_nextslot has been allocated and
3504 * c_store.c_buffer populated
3505 */
3506 if (c_seg_dst == NULL) {
3507 /*
3508 * Out of compression segments?
3509 */
3510 kr = KERN_RESOURCE_SHORTAGE;
3511 goto out;
3512 }
3513
3514 assert(c_seg_dst->c_busy == 0);
3515
3516 C_SEG_BUSY(c_seg_dst);
3517
3518 dst_slot = c_seg_dst->c_nextslot;
3519
3520 lck_mtx_unlock_always(&c_seg_dst->c_lock);
3521
3522Relookup_src:
3523 c_seg_src = c_segments[src_slot->s_cseg - 1].c_seg;
3524
3525 assert(c_seg_dst != c_seg_src);
3526
3527 lck_mtx_lock_spin_always(&c_seg_src->c_lock);
3528
3529 if (C_SEG_IS_ONDISK(c_seg_src)) {
3530
3531 /*
3532 * A "thaw" can mark a process as eligible for
3533 * another freeze cycle without bringing any of
3534 * its swapped out c_segs back from disk (because
3535 * that is done on-demand).
3536 *
3537 * If the src c_seg we find for our pre-compressed
3538 * data is already on-disk, then we are dealing
3539 * with an app's data that is already packed and
3540 * swapped out. Don't do anything.
3541 */
3542
3543 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3544
3545 lck_mtx_unlock_always(&c_seg_src->c_lock);
3546
3547 c_seg_src = NULL;
3548
3549 goto out;
3550 }
3551
3552 if (c_seg_src->c_busy) {
3553
3554 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3555 c_seg_wait_on_busy(c_seg_src);
3556
3557 c_seg_src = NULL;
3558
3559 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3560
3561 goto Relookup_src;
3562 }
3563
3564 C_SEG_BUSY(c_seg_src);
3565
3566 lck_mtx_unlock_always(&c_seg_src->c_lock);
3567
3568 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3569
3570 /* find the c_slot */
3571 c_indx = src_slot->s_cindx;
3572
3573 c_src = C_SEG_SLOT_FROM_INDEX(c_seg_src, c_indx);
3574
3575 c_size = UNPACK_C_SIZE(c_src);
3576
3577 assert(c_size);
3578
3579 if (c_size > (uint32_t)(C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES((int32_t)c_seg_dst->c_nextoffset))) {
3580 /*
3581 * This segment is full. We need a new one.
3582 */
3583
3584 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3585
3586 lck_mtx_lock_spin_always(&c_seg_src->c_lock);
3587 C_SEG_WAKEUP_DONE(c_seg_src);
3588 lck_mtx_unlock_always(&c_seg_src->c_lock);
3589
3590 c_seg_src = NULL;
3591
3592 lck_mtx_lock_spin_always(&c_seg_dst->c_lock);
3593
3594 assert(c_seg_dst->c_busy);
3595 assert(c_seg_dst->c_state == C_IS_FILLING);
3596 assert(!c_seg_dst->c_on_minorcompact_q);
3597
3598 c_current_seg_filled(c_seg_dst, (c_segment_t *)current_chead);
3599 assert(*current_chead == NULL);
3600
3601 C_SEG_WAKEUP_DONE(c_seg_dst);
3602
3603 lck_mtx_unlock_always(&c_seg_dst->c_lock);
3604
3605 c_seg_dst = NULL;
3606
3607 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3608
3609 goto Relookup_dst;
3610 }
3611
3612 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, c_seg_dst->c_nextslot);
3613
3614 memcpy(&c_seg_dst->c_store.c_buffer[c_seg_dst->c_nextoffset], &c_seg_src->c_store.c_buffer[c_src->c_offset], c_size);
3615
3616 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
3617
3618#if CHECKSUM_THE_DATA
3619 c_dst->c_hash_data = c_src->c_hash_data;
3620#endif
3621#if CHECKSUM_THE_COMPRESSED_DATA
3622 c_dst->c_hash_compressed_data = c_src->c_hash_compressed_data;
3623#endif
3624
3625 c_dst->c_size = c_src->c_size;
3626 c_dst->c_packed_ptr = c_src->c_packed_ptr;
3627 c_dst->c_offset = c_seg_dst->c_nextoffset;
3628
3629 if (c_seg_dst->c_firstemptyslot == c_seg_dst->c_nextslot)
3630 c_seg_dst->c_firstemptyslot++;
3631
3632 c_seg_dst->c_nextslot++;
3633 c_seg_dst->c_bytes_used += c_rounded_size;
3634 c_seg_dst->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
3635
3636
3637 PACK_C_SIZE(c_src, 0);
3638
3639 c_seg_src->c_bytes_used -= c_rounded_size;
3640 c_seg_src->c_bytes_unused += c_rounded_size;
3641
3642 if (c_indx < c_seg_src->c_firstemptyslot) {
3643 c_seg_src->c_firstemptyslot = c_indx;
3644 }
3645
3646 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, dst_slot);
3647
3648 PAGE_REPLACEMENT_ALLOWED(TRUE);
3649 slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
3650 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
3651 slot_ptr->s_cseg = c_seg_dst->c_mysegno + 1;
3652 slot_ptr->s_cindx = dst_slot;
3653
3654 PAGE_REPLACEMENT_ALLOWED(FALSE);
3655
3656out:
3657 if (c_seg_src) {
3658
3659 lck_mtx_lock_spin_always(&c_seg_src->c_lock);
3660
3661 C_SEG_WAKEUP_DONE(c_seg_src);
3662
3663 if (c_seg_src->c_bytes_used == 0 && c_seg_src->c_state != C_IS_FILLING) {
3664 if (!c_seg_src->c_on_minorcompact_q)
3665 c_seg_need_delayed_compaction(c_seg_src);
3666 }
3667
3668 lck_mtx_unlock_always(&c_seg_src->c_lock);
3669 }
3670
3671 if (c_seg_dst) {
3672
3673 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3674
3675 lck_mtx_lock_spin_always(&c_seg_dst->c_lock);
3676
3677 if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX_INDEX) {
3678 /*
3679 * Nearing or exceeded maximum slot and offset capacity.
3680 */
3681 assert(c_seg_dst->c_busy);
3682 assert(c_seg_dst->c_state == C_IS_FILLING);
3683 assert(!c_seg_dst->c_on_minorcompact_q);
3684
3685 c_current_seg_filled(c_seg_dst, (c_segment_t *)current_chead);
3686 assert(*current_chead == NULL);
3687 }
3688
3689 C_SEG_WAKEUP_DONE(c_seg_dst);
3690
3691 lck_mtx_unlock_always(&c_seg_dst->c_lock);
3692
3693 c_seg_dst = NULL;
3694
3695 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3696 }
3697
3698 return kr;
3699}
3700#endif /* CONFIG_FREEZE */
mirror of XNU