]> git.saurik.com Git - apple/xnu.git/blame - osfmk/vm/vm_compressor.c
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-3247.1.106.tar.gz
[apple/xnu.git] / osfmk / vm / vm_compressor.c
CommitLineData
39236c6e
A		 1/*
2 * Copyright (c) 2000-2013 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <vm/vm_compressor.h>
fe8ab488
A		 30
31#if CONFIG_PHANTOM_CACHE
32#include <vm/vm_phantom_cache.h>
33#endif
34
39236c6e
A		 35#include <vm/vm_map.h>
36#include <vm/vm_pageout.h>
37#include <vm/memory_object.h>
38#include <mach/mach_host.h> /* for host_info() */
39#include <kern/ledger.h>
40
3e170ce0
A		 41#include <i386/misc_protos.h>
42
39236c6e
A		 43#include <default_pager/default_pager_alerts.h>
44#include <default_pager/default_pager_object_server.h>
45
46#include <IOKit/IOHibernatePrivate.h>
47
48/*
49 * vm_compressor_mode has a heirarchy of control to set its value.
50 * boot-args are checked first, then device-tree, and finally
51 * the default value that is defined below. See vm_fault_init() for
52 * the boot-arg & device-tree code.
53 */
54
39236c6e
A		 55
56int vm_compressor_mode = VM_PAGER_COMPRESSOR_WITH_SWAP;
57int vm_scale = 16;
58
59
04b8595b 60int vm_compressor_is_active = 0;
39236c6e 61int vm_compression_limit = 0;
3e170ce0 62int vm_compressor_available = 0;
39236c6e
A		 63
64extern boolean_t vm_swap_up;
65extern void vm_pageout_io_throttle(void);
3e170ce0 66extern int not_in_kdp;
39236c6e
A		 67
68#if CHECKSUM_THE_DATA || CHECKSUM_THE_SWAP || CHECKSUM_THE_COMPRESSED_DATA
69extern unsigned int hash_string(char *cp, int len);
70#endif
71
3e170ce0
A		 72#define UNPACK_C_SIZE(cs) ((cs->c_size == (PAGE_SIZE-1)) ? PAGE_SIZE : cs->c_size)
73#define PACK_C_SIZE(cs, size) (cs->c_size = ((size == PAGE_SIZE) ? PAGE_SIZE - 1 : size))
fe8ab488 74
39236c6e 75
3e170ce0
A		 76struct c_sv_hash_entry {
77 union {
78 struct {
79 uint32_t c_sv_he_ref;
80 uint32_t c_sv_he_data;
81 } c_sv_he;
82 uint64_t c_sv_he_record;
83
84 } c_sv_he_un;
39236c6e
A		 85};
86
3e170ce0
A		 87#define he_ref c_sv_he_un.c_sv_he.c_sv_he_ref
88#define he_data c_sv_he_un.c_sv_he.c_sv_he_data
89#define he_record c_sv_he_un.c_sv_he_record
90
91#define C_SV_HASH_MAX_MISS 32
92#define C_SV_HASH_SIZE ((1 << 10))
93#define C_SV_HASH_MASK ((1 << 10) - 1)
94#define C_SV_CSEG_ID ((1 << 22) - 1)
39236c6e
A		 95
96
97struct c_slot_mapping {
98 uint32_t s_cseg:22, /* segment number + 1 */
99 s_cindx:10; /* index in the segment */
100};
3e170ce0 101#define C_SLOT_MAX_INDEX (1 << 10)
39236c6e
A		 102
103typedef struct c_slot_mapping *c_slot_mapping_t;
104
105
106union c_segu {
107 c_segment_t c_seg;
108 uint32_t c_segno;
109};
110
111
112
113#define C_SLOT_PACK_PTR(ptr) (((uintptr_t)ptr - (uintptr_t) VM_MIN_KERNEL_AND_KEXT_ADDRESS) >> 2)
114#define C_SLOT_UNPACK_PTR(cslot) ((uintptr_t)(cslot->c_packed_ptr << 2) + (uintptr_t) VM_MIN_KERNEL_AND_KEXT_ADDRESS)
115
116
117uint32_t c_segment_count = 0;
3e170ce0 118uint32_t c_segment_count_max = 0;
39236c6e
A		 119
120uint64_t c_generation_id = 0;
121uint64_t c_generation_id_flush_barrier;
122
123
124#define HIBERNATE_FLUSHING_SECS_TO_COMPLETE 120
125
126boolean_t hibernate_no_swapspace = FALSE;
127clock_sec_t hibernate_flushing_deadline = 0;
128
129
3e170ce0
A		 130#if RECORD_THE_COMPRESSED_DATA
131char *c_compressed_record_sbuf;
132char *c_compressed_record_ebuf;
133char *c_compressed_record_cptr;
39236c6e
A		 134#endif
135
3e170ce0 136
39236c6e
A		 137queue_head_t c_age_list_head;
138queue_head_t c_swapout_list_head;
139queue_head_t c_swappedin_list_head;
140queue_head_t c_swappedout_list_head;
141queue_head_t c_swappedout_sparse_list_head;
3e170ce0
A		 142queue_head_t c_major_list_head;
143queue_head_t c_filling_list_head;
144queue_head_t c_bad_list_head;
39236c6e
A		 145
146uint32_t c_age_count = 0;
147uint32_t c_swapout_count = 0;
148uint32_t c_swappedin_count = 0;
149uint32_t c_swappedout_count = 0;
150uint32_t c_swappedout_sparse_count = 0;
3e170ce0
A		 151uint32_t c_major_count = 0;
152uint32_t c_filling_count = 0;
153uint32_t c_empty_count = 0;
154uint32_t c_bad_count = 0;
155
39236c6e
A		 156
157queue_head_t c_minor_list_head;
158uint32_t c_minor_count = 0;
159
3e170ce0
A		 160int c_overage_swapped_count = 0;
161int c_overage_swapped_limit = 0;
162
163int c_seg_fixed_array_len;
39236c6e 164union c_segu *c_segments;
3e170ce0
A		 165vm_offset_t c_buffers;
166vm_size_t c_buffers_size;
39236c6e
A		 167caddr_t c_segments_next_page;
168boolean_t c_segments_busy;
169uint32_t c_segments_available;
170uint32_t c_segments_limit;
fe8ab488 171uint32_t c_segments_nearing_limit;
3e170ce0
A		 172
173uint32_t c_segment_svp_in_hash;
174uint32_t c_segment_svp_hash_succeeded;
175uint32_t c_segment_svp_hash_failed;
176uint32_t c_segment_svp_zero_compressions;
177uint32_t c_segment_svp_nonzero_compressions;
178uint32_t c_segment_svp_zero_decompressions;
179uint32_t c_segment_svp_nonzero_decompressions;
180
181uint32_t c_segment_noncompressible_pages;
182
39236c6e
A		 183uint32_t c_segment_pages_compressed;
184uint32_t c_segment_pages_compressed_limit;
fe8ab488 185uint32_t c_segment_pages_compressed_nearing_limit;
39236c6e
A		 186uint32_t c_free_segno_head = (uint32_t)-1;
187
188uint32_t vm_compressor_minorcompact_threshold_divisor = 10;
189uint32_t vm_compressor_majorcompact_threshold_divisor = 10;
190uint32_t vm_compressor_unthrottle_threshold_divisor = 10;
191uint32_t vm_compressor_catchup_threshold_divisor = 10;
192
193#define C_SEGMENTS_PER_PAGE (PAGE_SIZE / sizeof(union c_segu))
194
195
196lck_grp_attr_t vm_compressor_lck_grp_attr;
197lck_attr_t vm_compressor_lck_attr;
198lck_grp_t vm_compressor_lck_grp;
199
39236c6e
A		 200#if __i386__ || __x86_64__
201lck_mtx_t *c_list_lock;
202#else /* __i386__ || __x86_64__ */
203lck_spin_t *c_list_lock;
204#endif /* __i386__ || __x86_64__ */
205
206lck_rw_t c_master_lock;
fe8ab488 207boolean_t decompressions_blocked = FALSE;
39236c6e
A		 208
209zone_t compressor_segment_zone;
210int c_compressor_swap_trigger = 0;
211
212uint32_t compressor_cpus;
213char *compressor_scratch_bufs;
3e170ce0
A		 214char *kdp_compressor_scratch_buf;
215char *kdp_compressor_decompressed_page;
216addr64_t kdp_compressor_decompressed_page_paddr;
217ppnum_t kdp_compressor_decompressed_page_ppnum;
39236c6e
A		 218
219clock_sec_t start_of_sample_period_sec = 0;
220clock_nsec_t start_of_sample_period_nsec = 0;
221clock_sec_t start_of_eval_period_sec = 0;
222clock_nsec_t start_of_eval_period_nsec = 0;
223uint32_t sample_period_decompression_count = 0;
224uint32_t sample_period_compression_count = 0;
225uint32_t last_eval_decompression_count = 0;
226uint32_t last_eval_compression_count = 0;
227
228#define DECOMPRESSION_SAMPLE_MAX_AGE (60 * 30)
229
3e170ce0
A		 230boolean_t vm_swapout_ripe_segments = FALSE;
231uint32_t vm_ripe_target_age = (60 * 60 * 48);
232
39236c6e
A		 233uint32_t swapout_target_age = 0;
234uint32_t age_of_decompressions_during_sample_period[DECOMPRESSION_SAMPLE_MAX_AGE];
235uint32_t overage_decompressions_during_sample_period = 0;
236
237void do_fastwake_warmup(void);
238boolean_t fastwake_warmup = FALSE;
239boolean_t fastwake_recording_in_progress = FALSE;
240clock_sec_t dont_trim_until_ts = 0;
241
242uint64_t c_segment_warmup_count;
243uint64_t first_c_segment_to_warm_generation_id = 0;
244uint64_t last_c_segment_to_warm_generation_id = 0;
245boolean_t hibernate_flushing = FALSE;
246
fe8ab488
A		 247int64_t c_segment_input_bytes __attribute__((aligned(8))) = 0;
248int64_t c_segment_compressed_bytes __attribute__((aligned(8))) = 0;
249int64_t compressor_bytes_used __attribute__((aligned(8))) = 0;
3e170ce0
A		 250
251
252struct c_sv_hash_entry c_segment_sv_hash_table[C_SV_HASH_SIZE] __attribute__ ((aligned (8)));
253
39236c6e
A		 254
255static boolean_t compressor_needs_to_swap(void);
256static void vm_compressor_swap_trigger_thread(void);
257static void vm_compressor_do_delayed_compactions(boolean_t);
258static void vm_compressor_compact_and_swap(boolean_t);
259static void vm_compressor_age_swapped_in_segments(boolean_t);
39236c6e 260
3e170ce0
A		 261static void vm_compressor_take_paging_space_action(void);
262
39236c6e
A		 263boolean_t vm_compressor_low_on_space(void);
264
265void compute_swapout_target_age(void);
266
267boolean_t c_seg_major_compact(c_segment_t, c_segment_t);
268boolean_t c_seg_major_compact_ok(c_segment_t, c_segment_t);
269
270int c_seg_minor_compaction_and_unlock(c_segment_t, boolean_t);
271int c_seg_do_minor_compaction_and_unlock(c_segment_t, boolean_t, boolean_t, boolean_t);
272void c_seg_try_minor_compaction_and_unlock(c_segment_t c_seg);
273void c_seg_need_delayed_compaction(c_segment_t);
274
275void c_seg_move_to_sparse_list(c_segment_t);
276void c_seg_insert_into_q(queue_head_t *, c_segment_t);
277
39236c6e 278uint64_t vm_available_memory(void);
fe8ab488 279uint64_t vm_compressor_pages_compressed(void);
39236c6e
A		 280
281extern unsigned int dp_pages_free, dp_pages_reserve;
282
283uint64_t
284vm_available_memory(void)
285{
286 return (((uint64_t)AVAILABLE_NON_COMPRESSED_MEMORY) * PAGE_SIZE_64);
287}
288
289
fe8ab488
A		 290uint64_t
291vm_compressor_pages_compressed(void)
292{
293 return (c_segment_pages_compressed * PAGE_SIZE_64);
294}
295
296
39236c6e
A		 297boolean_t
298vm_compression_available(void)
299{
300 if ( !(COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE))
301 return (FALSE);
302
303 if (c_segments_available >= c_segments_limit || c_segment_pages_compressed >= c_segment_pages_compressed_limit)
304 return (FALSE);
305
306 return (TRUE);
307}
308
309
310boolean_t
311vm_compressor_low_on_space(void)
312{
fe8ab488
A		 313 if ((c_segment_pages_compressed > c_segment_pages_compressed_nearing_limit) ||
314 (c_segment_count > c_segments_nearing_limit))
39236c6e
A		 315 return (TRUE);
316
317 return (FALSE);
318}
319
320
321int
fe8ab488 322vm_wants_task_throttled(task_t task)
39236c6e 323{
fe8ab488
A		 324 if (task == kernel_task)
325 return (0);
326
39236c6e 327 if (vm_compressor_mode == COMPRESSED_PAGER_IS_ACTIVE || vm_compressor_mode == DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE) {
fe8ab488
A		 328 if ((vm_compressor_low_on_space() || HARD_THROTTLE_LIMIT_REACHED()) &&
329 (unsigned int)pmap_compressed(task->map->pmap) > (c_segment_pages_compressed / 4))
39236c6e
A		 330 return (1);
331 } else {
fe8ab488
A		 332 if (((dp_pages_free + dp_pages_reserve < 2000) && VM_DYNAMIC_PAGING_ENABLED(memory_manager_default)) &&
333 get_task_resident_size(task) > (((AVAILABLE_NON_COMPRESSED_MEMORY) * PAGE_SIZE) / 5))
39236c6e
A		 334 return (1);
335 }
336 return (0);
337}
338
339
3e170ce0
A		 340
341static uint32_t no_paging_space_action_in_progress = 0;
342extern void memorystatus_send_low_swap_note(void);
343
344static void
345vm_compressor_take_paging_space_action(void)
346{
347 if (no_paging_space_action_in_progress == 0) {
348
349 if (OSCompareAndSwap(0, 1, (UInt32 *)&no_paging_space_action_in_progress)) {
350
351 if (no_paging_space_action()) {
352 memorystatus_send_low_swap_note();
353 }
354
355 no_paging_space_action_in_progress = 0;
356 }
357 }
358}
359
360
361
39236c6e
A		 362void
363vm_compressor_init_locks(void)
364{
365 lck_grp_attr_setdefault(&vm_compressor_lck_grp_attr);
366 lck_grp_init(&vm_compressor_lck_grp, "vm_compressor", &vm_compressor_lck_grp_attr);
367 lck_attr_setdefault(&vm_compressor_lck_attr);
368
369 lck_rw_init(&c_master_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
39236c6e
A		 370}
371
372
373void
374vm_decompressor_lock(void)
375{
fe8ab488
A		 376 PAGE_REPLACEMENT_ALLOWED(TRUE);
377
378 decompressions_blocked = TRUE;
379
380 PAGE_REPLACEMENT_ALLOWED(FALSE);
39236c6e
A		 381}
382
383void
384vm_decompressor_unlock(void)
385{
fe8ab488
A		 386 PAGE_REPLACEMENT_ALLOWED(TRUE);
387
388 decompressions_blocked = FALSE;
389
390 PAGE_REPLACEMENT_ALLOWED(FALSE);
39236c6e 391
fe8ab488 392 thread_wakeup((event_t)&decompressions_blocked);
39236c6e
A		 393}
394
395
396
397void
398vm_compressor_init(void)
399{
400 thread_t thread;
fe8ab488
A		 401 struct c_slot cs_dummy;
402 c_slot_t cs = &cs_dummy;
3e170ce0
A		 403 int c_segment_min_size;
404 int c_segment_padded_size;
fe8ab488
A		 405
406 /*
407 * ensure that any pointer that gets created from
408 * the vm_page zone can be packed properly
409 */
410 cs->c_packed_ptr = C_SLOT_PACK_PTR(zone_map_min_address);
411
412 if (C_SLOT_UNPACK_PTR(cs) != (uintptr_t)zone_map_min_address)
413 panic("C_SLOT_UNPACK_PTR failed on zone_map_min_address - %p", (void *)zone_map_min_address);
414
415 cs->c_packed_ptr = C_SLOT_PACK_PTR(zone_map_max_address);
416
417 if (C_SLOT_UNPACK_PTR(cs) != (uintptr_t)zone_map_max_address)
418 panic("C_SLOT_UNPACK_PTR failed on zone_map_max_address - %p", (void *)zone_map_max_address);
419
39236c6e
A		 420
421 assert((C_SEGMENTS_PER_PAGE * sizeof(union c_segu)) == PAGE_SIZE);
422
423 PE_parse_boot_argn("vm_compression_limit", &vm_compression_limit, sizeof (vm_compression_limit));
424
425 if (max_mem <= (3ULL * 1024ULL * 1024ULL * 1024ULL)) {
426 vm_compressor_minorcompact_threshold_divisor = 11;
427 vm_compressor_majorcompact_threshold_divisor = 13;
428 vm_compressor_unthrottle_threshold_divisor = 20;
429 vm_compressor_catchup_threshold_divisor = 35;
430 } else {
431 vm_compressor_minorcompact_threshold_divisor = 20;
432 vm_compressor_majorcompact_threshold_divisor = 25;
433 vm_compressor_unthrottle_threshold_divisor = 35;
434 vm_compressor_catchup_threshold_divisor = 50;
435 }
436 /*
437 * vm_page_init_lck_grp is now responsible for calling vm_compressor_init_locks
438 * c_master_lock needs to be available early so that "vm_page_find_contiguous" can
439 * use PAGE_REPLACEMENT_ALLOWED to coordinate with the compressor.
440 */
441
442#if __i386__ || __x86_64__
443 c_list_lock = lck_mtx_alloc_init(&vm_compressor_lck_grp, &vm_compressor_lck_attr);
444#else /* __i386__ || __x86_64__ */
445 c_list_lock = lck_spin_alloc_init(&vm_compressor_lck_grp, &vm_compressor_lck_attr);
446#endif /* __i386__ || __x86_64__ */
447
3e170ce0 448
39236c6e 449 queue_init(&c_bad_list_head);
39236c6e
A		 450 queue_init(&c_age_list_head);
451 queue_init(&c_minor_list_head);
3e170ce0
A		 452 queue_init(&c_major_list_head);
453 queue_init(&c_filling_list_head);
39236c6e
A		 454 queue_init(&c_swapout_list_head);
455 queue_init(&c_swappedin_list_head);
456 queue_init(&c_swappedout_list_head);
457 queue_init(&c_swappedout_sparse_list_head);
458
3e170ce0
A		 459 c_segment_min_size = sizeof(struct c_segment) + (C_SEG_SLOT_VAR_ARRAY_MIN_LEN * sizeof(struct c_slot));
460
461 for (c_segment_padded_size = 128; c_segment_padded_size < c_segment_min_size; c_segment_padded_size = c_segment_padded_size << 1);
462
463 compressor_segment_zone = zinit(c_segment_padded_size, 128000 * c_segment_padded_size, PAGE_SIZE, "compressor_segment");
39236c6e
A		 464 zone_change(compressor_segment_zone, Z_CALLERACCT, FALSE);
465 zone_change(compressor_segment_zone, Z_NOENCRYPT, TRUE);
466
3e170ce0 467 c_seg_fixed_array_len = (c_segment_padded_size - sizeof(struct c_segment)) / sizeof(struct c_slot);
39236c6e
A		 468
469 c_free_segno_head = -1;
470 c_segments_available = 0;
471
472 if (vm_compression_limit == 0) {
473 c_segment_pages_compressed_limit = (uint32_t)((max_mem / PAGE_SIZE)) * vm_scale;
474
475#define OLD_SWAP_LIMIT (1024 * 1024 * 16)
476#define MAX_SWAP_LIMIT (1024 * 1024 * 128)
477
478 if (c_segment_pages_compressed_limit > (OLD_SWAP_LIMIT))
479 c_segment_pages_compressed_limit = OLD_SWAP_LIMIT;
480
481 if (c_segment_pages_compressed_limit < (uint32_t)(max_mem / PAGE_SIZE_64))
482 c_segment_pages_compressed_limit = (uint32_t)(max_mem / PAGE_SIZE_64);
483 } else {
484 if (vm_compression_limit < MAX_SWAP_LIMIT)
485 c_segment_pages_compressed_limit = vm_compression_limit;
486 else
487 c_segment_pages_compressed_limit = MAX_SWAP_LIMIT;
488 }
489 if ((c_segments_limit = c_segment_pages_compressed_limit / (C_SEG_BUFSIZE / PAGE_SIZE)) > C_SEG_MAX_LIMIT)
490 c_segments_limit = C_SEG_MAX_LIMIT;
491
fe8ab488
A		 492 c_segment_pages_compressed_nearing_limit = (c_segment_pages_compressed_limit * 98) / 100;
493 c_segments_nearing_limit = (c_segments_limit * 98) / 100;
494
39236c6e
A		 495 c_segments_busy = FALSE;
496
3e170ce0
A		 497 if (kernel_memory_allocate(kernel_map, (vm_offset_t *)(&c_segments), (sizeof(union c_segu) * c_segments_limit), 0, KMA_KOBJECT | KMA_VAONLY | KMA_PERMANENT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS)
498 panic("vm_compressor_init: kernel_memory_allocate failed - c_segments\n");
499 c_buffers_size = (vm_size_t)C_SEG_ALLOCSIZE * (vm_size_t)c_segments_limit;
500 if (kernel_memory_allocate(kernel_map, &c_buffers, c_buffers_size, 0, KMA_COMPRESSOR | KMA_VAONLY | KMA_PERMANENT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS)
501 panic("vm_compressor_init: kernel_memory_allocate failed - c_buffers\n");
39236c6e
A		 502
503 c_segments_next_page = (caddr_t)c_segments;
504
505 {
506 host_basic_info_data_t hinfo;
507 mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
508
509#define BSD_HOST 1
510 host_info((host_t)BSD_HOST, HOST_BASIC_INFO, (host_info_t)&hinfo, &count);
511
512 compressor_cpus = hinfo.max_cpus;
513
3e170ce0
A		 514 compressor_scratch_bufs = kalloc_tag(compressor_cpus * WKdm_SCRATCH_BUF_SIZE, VM_KERN_MEMORY_COMPRESSOR);
515
516 kdp_compressor_scratch_buf = kalloc_tag(WKdm_SCRATCH_BUF_SIZE, VM_KERN_MEMORY_COMPRESSOR);
517 kdp_compressor_decompressed_page = kalloc_tag(PAGE_SIZE, VM_KERN_MEMORY_COMPRESSOR);
518 kdp_compressor_decompressed_page_paddr = kvtophys((vm_offset_t)kdp_compressor_decompressed_page);
519 kdp_compressor_decompressed_page_ppnum = (ppnum_t) atop(kdp_compressor_decompressed_page_paddr);
39236c6e 520 }
3e170ce0
A		 521#if CONFIG_FREEZE
522 freezer_compressor_scratch_buf = kalloc_tag(WKdm_SCRATCH_BUF_SIZE, VM_KERN_MEMORY_COMPRESSOR);
523#endif
524
525#if RECORD_THE_COMPRESSED_DATA
526 if (kernel_memory_allocate(kernel_map, (vm_offset_t *)&c_compressed_record_sbuf, (vm_size_t)C_SEG_ALLOCSIZE + (PAGE_SIZE * 2), 0, KMA_KOBJECT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS)
527 panic("vm_compressor_init: kernel_memory_allocate failed - c_compressed_record_sbuf\n");
528
529 c_compressed_record_cptr = c_compressed_record_sbuf;
530 c_compressed_record_ebuf = c_compressed_record_sbuf + C_SEG_ALLOCSIZE + (PAGE_SIZE * 2);
531#endif
39236c6e
A		 532
533 if (kernel_thread_start_priority((thread_continue_t)vm_compressor_swap_trigger_thread, NULL,
534 BASEPRI_PREEMPT - 1, &thread) != KERN_SUCCESS) {
535 panic("vm_compressor_swap_trigger_thread: create failed");
536 }
39236c6e
A		 537 thread_deallocate(thread);
538
539 assert(default_pager_init_flag == 0);
540
541 if (vm_pageout_internal_start() != KERN_SUCCESS) {
542 panic("vm_compressor_init: Failed to start the internal pageout thread.\n");
543 }
3e170ce0 544 if (COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE)
fe8ab488 545 vm_compressor_swap_init();
fe8ab488 546
04b8595b
A		 547 if (COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_SWAPBACKED)
548 vm_compressor_is_active = 1;
549
39236c6e
A		 550#if CONFIG_FREEZE
551 memorystatus_freeze_enabled = TRUE;
552#endif /* CONFIG_FREEZE */
553
554 default_pager_init_flag = 1;
3e170ce0 555 vm_compressor_available = 1;
39236c6e
A		 556
557 vm_page_reactivate_all_throttled();
558}
559
560
561#if VALIDATE_C_SEGMENTS
562
563static void
564c_seg_validate(c_segment_t c_seg, boolean_t must_be_compact)
565{
566 int c_indx;
567 int32_t bytes_used;
568 int32_t bytes_unused;
569 uint32_t c_rounded_size;
570 uint32_t c_size;
571 c_slot_t cs;
572
573 if (c_seg->c_firstemptyslot < c_seg->c_nextslot) {
574 c_indx = c_seg->c_firstemptyslot;
575 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
576
577 if (cs == NULL)
578 panic("c_seg_validate: no slot backing c_firstemptyslot");
579
580 if (cs->c_size)
581 panic("c_seg_validate: c_firstemptyslot has non-zero size (%d)\n", cs->c_size);
582 }
583 bytes_used = 0;
584 bytes_unused = 0;
585
586 for (c_indx = 0; c_indx < c_seg->c_nextslot; c_indx++) {
587
588 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
589
590 c_size = UNPACK_C_SIZE(cs);
591
592 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
593
594 bytes_used += c_rounded_size;
595
596#if CHECKSUM_THE_COMPRESSED_DATA
597 if (c_size && cs->c_hash_compressed_data != hash_string((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size))
598 panic("compressed data doesn't match original");
599#endif
600 }
601
602 if (bytes_used != c_seg->c_bytes_used)
603 panic("c_seg_validate: bytes_used mismatch - found %d, segment has %d\n", bytes_used, c_seg->c_bytes_used);
604
605 if (c_seg->c_bytes_used > C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset))
606 panic("c_seg_validate: c_bytes_used > c_nextoffset - c_nextoffset = %d, c_bytes_used = %d\n",
607 (int32_t)C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset), c_seg->c_bytes_used);
608
609 if (must_be_compact) {
610 if (c_seg->c_bytes_used != C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset))
611 panic("c_seg_validate: c_bytes_used doesn't match c_nextoffset - c_nextoffset = %d, c_bytes_used = %d\n",
612 (int32_t)C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset), c_seg->c_bytes_used);
613 }
614}
615
616#endif
617
618
619void
620c_seg_need_delayed_compaction(c_segment_t c_seg)
621{
622 boolean_t clear_busy = FALSE;
623
624 if ( !lck_mtx_try_lock_spin_always(c_list_lock)) {
fe8ab488 625 C_SEG_BUSY(c_seg);
39236c6e
A		 626
627 lck_mtx_unlock_always(&c_seg->c_lock);
628 lck_mtx_lock_spin_always(c_list_lock);
629 lck_mtx_lock_spin_always(&c_seg->c_lock);
630
631 clear_busy = TRUE;
632 }
3e170ce0
A		 633 assert(c_seg->c_state != C_IS_FILLING);
634
635 if (!c_seg->c_on_minorcompact_q && !(C_SEG_IS_ONDISK(c_seg))) {
39236c6e
A		 636 queue_enter(&c_minor_list_head, c_seg, c_segment_t, c_list);
637 c_seg->c_on_minorcompact_q = 1;
638 c_minor_count++;
639 }
640 lck_mtx_unlock_always(c_list_lock);
641
642 if (clear_busy == TRUE)
643 C_SEG_WAKEUP_DONE(c_seg);
644}
645
646
647unsigned int c_seg_moved_to_sparse_list = 0;
648
649void
650c_seg_move_to_sparse_list(c_segment_t c_seg)
651{
652 boolean_t clear_busy = FALSE;
653
654 if ( !lck_mtx_try_lock_spin_always(c_list_lock)) {
fe8ab488 655 C_SEG_BUSY(c_seg);
39236c6e
A		 656
657 lck_mtx_unlock_always(&c_seg->c_lock);
658 lck_mtx_lock_spin_always(c_list_lock);
659 lck_mtx_lock_spin_always(&c_seg->c_lock);
660
661 clear_busy = TRUE;
662 }
3e170ce0 663 c_seg_switch_state(c_seg, C_ON_SWAPPEDOUTSPARSE_Q, FALSE);
39236c6e
A		 664
665 c_seg_moved_to_sparse_list++;
666
667 lck_mtx_unlock_always(c_list_lock);
668
669 if (clear_busy == TRUE)
670 C_SEG_WAKEUP_DONE(c_seg);
671}
672
673
674void
675c_seg_insert_into_q(queue_head_t *qhead, c_segment_t c_seg)
676{
677 c_segment_t c_seg_next;
678
679 if (queue_empty(qhead)) {
680 queue_enter(qhead, c_seg, c_segment_t, c_age_list);
681 } else {
682 c_seg_next = (c_segment_t)queue_first(qhead);
683
684 while (TRUE) {
685
686 if (c_seg->c_generation_id < c_seg_next->c_generation_id) {
687 queue_insert_before(qhead, c_seg, c_seg_next, c_segment_t, c_age_list);
688 break;
689 }
690 c_seg_next = (c_segment_t) queue_next(&c_seg_next->c_age_list);
691
692 if (queue_end(qhead, (queue_entry_t) c_seg_next)) {
693 queue_enter(qhead, c_seg, c_segment_t, c_age_list);
694 break;
695 }
696 }
697 }
698}
699
700
701int try_minor_compaction_failed = 0;
702int try_minor_compaction_succeeded = 0;
703
704void
705c_seg_try_minor_compaction_and_unlock(c_segment_t c_seg)
706{
707
708 assert(c_seg->c_on_minorcompact_q);
709 /*
710 * c_seg is currently on the delayed minor compaction
711 * queue and we have c_seg locked... if we can get the
712 * c_list_lock w/o blocking (if we blocked we could deadlock
713 * because the lock order is c_list_lock then c_seg's lock)
714 * we'll pull it from the delayed list and free it directly
715 */
716 if ( !lck_mtx_try_lock_spin_always(c_list_lock)) {
717 /*
718 * c_list_lock is held, we need to bail
719 */
720 try_minor_compaction_failed++;
721
722 lck_mtx_unlock_always(&c_seg->c_lock);
723 } else {
724 try_minor_compaction_succeeded++;
725
fe8ab488 726 C_SEG_BUSY(c_seg);
39236c6e
A		 727 c_seg_do_minor_compaction_and_unlock(c_seg, TRUE, FALSE, FALSE);
728 }
729}
730
731
732int
733c_seg_do_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy, boolean_t need_list_lock, boolean_t disallow_page_replacement)
734{
735 int c_seg_freed;
736
737 assert(c_seg->c_busy);
738
3e170ce0
A		 739 /*
740 * check for the case that can occur when we are not swapping
741 * and this segment has been major compacted in the past
742 * and moved to the majorcompact q to remove it from further
743 * consideration... if the occupancy falls too low we need
744 * to put it back on the age_q so that it will be considered
745 * in the next major compaction sweep... if we don't do this
746 * we will eventually run into the c_segments_limit
747 */
748 if (c_seg->c_state == C_ON_MAJORCOMPACT_Q && C_SEG_SHOULD_MAJORCOMPACT(c_seg)) {
749
750 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
751 }
39236c6e
A		 752 if (!c_seg->c_on_minorcompact_q) {
753 if (clear_busy == TRUE)
754 C_SEG_WAKEUP_DONE(c_seg);
755
756 lck_mtx_unlock_always(&c_seg->c_lock);
757
758 return (0);
759 }
760 queue_remove(&c_minor_list_head, c_seg, c_segment_t, c_list);
761 c_seg->c_on_minorcompact_q = 0;
762 c_minor_count--;
763
764 lck_mtx_unlock_always(c_list_lock);
765
766 if (disallow_page_replacement == TRUE) {
767 lck_mtx_unlock_always(&c_seg->c_lock);
768
769 PAGE_REPLACEMENT_DISALLOWED(TRUE);
770
771 lck_mtx_lock_spin_always(&c_seg->c_lock);
772 }
773 c_seg_freed = c_seg_minor_compaction_and_unlock(c_seg, clear_busy);
774
775 if (disallow_page_replacement == TRUE)
776 PAGE_REPLACEMENT_DISALLOWED(FALSE);
777
778 if (need_list_lock == TRUE)
779 lck_mtx_lock_spin_always(c_list_lock);
780
781 return (c_seg_freed);
782}
783
784
785void
786c_seg_wait_on_busy(c_segment_t c_seg)
787{
788 c_seg->c_wanted = 1;
789 assert_wait((event_t) (c_seg), THREAD_UNINT);
790
791 lck_mtx_unlock_always(&c_seg->c_lock);
792 thread_block(THREAD_CONTINUE_NULL);
793}
794
795
3e170ce0
A		 796void
797c_seg_switch_state(c_segment_t c_seg, int new_state, boolean_t insert_head)
798{
799 int old_state = c_seg->c_state;
39236c6e 800
3e170ce0
A		 801#if DEVELOPMENT || DEBUG
802#if __i386__ || __x86_64__
803 if (new_state != C_IS_FILLING)
804 lck_mtx_assert(&c_seg->c_lock, LCK_MTX_ASSERT_OWNED);
805 lck_mtx_assert(c_list_lock, LCK_MTX_ASSERT_OWNED);
806#endif
807#endif
808 switch (old_state) {
39236c6e 809
3e170ce0
A		 810 case C_IS_EMPTY:
811 assert(new_state == C_IS_FILLING || new_state == C_IS_FREE);
39236c6e 812
3e170ce0
A		 813 c_empty_count--;
814 break;
815
816 case C_IS_FILLING:
817 assert(new_state == C_ON_AGE_Q || new_state == C_ON_SWAPOUT_Q);
818
819 queue_remove(&c_filling_list_head, c_seg, c_segment_t, c_age_list);
820 c_filling_count--;
821 break;
822
823 case C_ON_AGE_Q:
824 assert(new_state == C_ON_SWAPOUT_Q || new_state == C_ON_MAJORCOMPACT_Q ||
825 new_state == C_IS_FREE);
826
827 queue_remove(&c_age_list_head, c_seg, c_segment_t, c_age_list);
828 c_age_count--;
829 break;
830
831 case C_ON_SWAPPEDIN_Q:
832 assert(new_state == C_ON_AGE_Q || new_state == C_IS_FREE);
833
834 queue_remove(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
835 c_swappedin_count--;
836 break;
837
838 case C_ON_SWAPOUT_Q:
839 assert(new_state == C_ON_SWAPPEDOUT_Q || new_state == C_ON_SWAPPEDOUTSPARSE_Q ||
840 new_state == C_ON_AGE_Q || new_state == C_IS_FREE || new_state == C_IS_EMPTY);
841
842 queue_remove(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
843 thread_wakeup((event_t)&compaction_swapper_running);
844 c_swapout_count--;
845 break;
846
847 case C_ON_SWAPPEDOUT_Q:
848 assert(new_state == C_ON_SWAPPEDIN_Q || new_state == C_ON_SWAPPEDOUTSPARSE_Q ||
849 new_state == C_ON_BAD_Q || new_state == C_IS_EMPTY || new_state == C_IS_FREE);
850
851 queue_remove(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
852 c_swappedout_count--;
853 break;
854
855 case C_ON_SWAPPEDOUTSPARSE_Q:
856 assert(new_state == C_ON_SWAPPEDIN_Q ||
857 new_state == C_ON_BAD_Q || new_state == C_IS_EMPTY || new_state == C_IS_FREE);
858
859 queue_remove(&c_swappedout_sparse_list_head, c_seg, c_segment_t, c_age_list);
860 c_swappedout_sparse_count--;
861 break;
862
863 case C_ON_MAJORCOMPACT_Q:
864 assert(new_state == C_ON_AGE_Q || new_state == C_IS_FREE);
865
866 queue_remove(&c_major_list_head, c_seg, c_segment_t, c_age_list);
867 c_major_count--;
868 break;
869
870 case C_ON_BAD_Q:
871 assert(new_state == C_IS_FREE);
872
873 queue_remove(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
874 c_bad_count--;
875 break;
876
877 default:
878 panic("c_seg %p has bad c_state = %d\n", c_seg, old_state);
39236c6e 879 }
39236c6e 880
3e170ce0
A		 881 switch(new_state) {
882 case C_IS_FREE:
883 assert(old_state != C_IS_FILLING);
884
885 break;
39236c6e 886
3e170ce0
A		 887 case C_IS_EMPTY:
888 assert(old_state == C_ON_SWAPOUT_Q || old_state == C_ON_SWAPPEDOUT_Q || old_state == C_ON_SWAPPEDOUTSPARSE_Q);
39236c6e 889
3e170ce0
A		 890 c_empty_count++;
891 break;
892
893 case C_IS_FILLING:
894 assert(old_state == C_IS_EMPTY);
895
896 queue_enter(&c_filling_list_head, c_seg, c_segment_t, c_age_list);
897 c_filling_count++;
898 break;
899
900 case C_ON_AGE_Q:
901 assert(old_state == C_IS_FILLING || old_state == C_ON_SWAPPEDIN_Q ||
902 old_state == C_ON_MAJORCOMPACT_Q || old_state == C_ON_SWAPOUT_Q);
903
904 if (old_state == C_IS_FILLING)
905 queue_enter(&c_age_list_head, c_seg, c_segment_t, c_age_list);
906 else
907 c_seg_insert_into_q(&c_age_list_head, c_seg);
908 c_age_count++;
909 break;
910
911 case C_ON_SWAPPEDIN_Q:
912 assert(c_seg->c_state == C_ON_SWAPPEDOUT_Q || c_seg->c_state == C_ON_SWAPPEDOUTSPARSE_Q);
913
914 if (insert_head == TRUE)
915 queue_enter_first(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
916 else
917 queue_enter(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
918 c_swappedin_count++;
919 break;
920
921 case C_ON_SWAPOUT_Q:
922 assert(old_state == C_ON_AGE_Q || old_state == C_IS_FILLING);
923
924 if (insert_head == TRUE)
925 queue_enter_first(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
926 else
927 queue_enter(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
928 c_swapout_count++;
929 break;
930
931 case C_ON_SWAPPEDOUT_Q:
932 assert(c_seg->c_state == C_ON_SWAPOUT_Q);
933
934 if (insert_head == TRUE)
935 queue_enter_first(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
936 else
937 queue_enter(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
938 c_swappedout_count++;
939 break;
940
941 case C_ON_SWAPPEDOUTSPARSE_Q:
942 assert(c_seg->c_state == C_ON_SWAPOUT_Q || c_seg->c_state == C_ON_SWAPPEDOUT_Q);
943
944 c_seg_insert_into_q(&c_swappedout_sparse_list_head, c_seg);
945 c_swappedout_sparse_count++;
946 break;
947
948 case C_ON_MAJORCOMPACT_Q:
949 assert(c_seg->c_state == C_ON_AGE_Q);
950
951 if (insert_head == TRUE)
952 queue_enter_first(&c_major_list_head, c_seg, c_segment_t, c_age_list);
953 else
954 queue_enter(&c_major_list_head, c_seg, c_segment_t, c_age_list);
955 c_major_count++;
956 break;
957
958 case C_ON_BAD_Q:
959 assert(c_seg->c_state == C_ON_SWAPPEDOUT_Q || c_seg->c_state == C_ON_SWAPPEDOUTSPARSE_Q);
960
961 if (insert_head == TRUE)
962 queue_enter_first(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
963 else
964 queue_enter(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
965 c_bad_count++;
966 break;
967
968 default:
969 panic("c_seg %p requesting bad c_state = %d\n", c_seg, new_state);
970 }
971 c_seg->c_state = new_state;
39236c6e
A		 972}
973
974
3e170ce0 975
39236c6e
A		 976void
977c_seg_free(c_segment_t c_seg)
978{
fe8ab488 979 assert(c_seg->c_busy);
39236c6e
A		 980
981 lck_mtx_unlock_always(&c_seg->c_lock);
982 lck_mtx_lock_spin_always(c_list_lock);
983 lck_mtx_lock_spin_always(&c_seg->c_lock);
984
985 c_seg_free_locked(c_seg);
986}
987
988
989void
990c_seg_free_locked(c_segment_t c_seg)
991{
3e170ce0 992 int segno;
04b8595b 993 int pages_populated = 0;
39236c6e 994 int32_t *c_buffer = NULL;
04b8595b 995 uint64_t c_swap_handle = 0;
39236c6e 996
3e170ce0 997 assert(c_seg->c_busy);
39236c6e 998 assert(!c_seg->c_on_minorcompact_q);
3e170ce0 999 assert(!c_seg->c_busy_swapping);
39236c6e 1000
3e170ce0
A		 1001 if (c_seg->c_overage_swap == TRUE) {
1002 c_overage_swapped_count--;
1003 c_seg->c_overage_swap = FALSE;
1004 }
1005 if ( !(C_SEG_IS_ONDISK(c_seg)))
1006 c_buffer = c_seg->c_store.c_buffer;
1007 else
1008 c_swap_handle = c_seg->c_store.c_swap_handle;
39236c6e 1009
3e170ce0 1010 c_seg_switch_state(c_seg, C_IS_FREE, FALSE);
39236c6e 1011
3e170ce0 1012 lck_mtx_unlock_always(c_list_lock);
39236c6e 1013
3e170ce0 1014 if (c_buffer) {
39236c6e 1015 pages_populated = (round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE;
39236c6e 1016 c_seg->c_store.c_buffer = NULL;
3e170ce0 1017 } else
39236c6e 1018 c_seg->c_store.c_swap_handle = (uint64_t)-1;
3e170ce0 1019
39236c6e
A		 1020 lck_mtx_unlock_always(&c_seg->c_lock);
1021
1022 if (c_buffer) {
8a3053a0
A		 1023 if (pages_populated)
1024 kernel_memory_depopulate(kernel_map, (vm_offset_t) c_buffer, pages_populated * PAGE_SIZE, KMA_COMPRESSOR);
39236c6e 1025
3e170ce0
A		 1026 } else if (c_swap_handle) {
1027 /*
1028 * Free swap space on disk.
1029 */
39236c6e 1030 vm_swap_free(c_swap_handle);
3e170ce0
A		 1031 }
1032 lck_mtx_lock_spin_always(&c_seg->c_lock);
1033
1034 C_SEG_WAKEUP_DONE(c_seg);
1035 lck_mtx_unlock_always(&c_seg->c_lock);
1036
1037 segno = c_seg->c_mysegno;
39236c6e 1038
3e170ce0
A		 1039 lck_mtx_lock_spin_always(c_list_lock);
1040 /*
1041 * because the c_buffer is now associated with the segno,
1042 * we can't put the segno back on the free list until
1043 * after we have depopulated the c_buffer range, or
1044 * we run the risk of depopulating a range that is
1045 * now being used in one of the compressor heads
1046 */
1047 c_segments[segno].c_segno = c_free_segno_head;
1048 c_free_segno_head = segno;
1049 c_segment_count--;
1050
1051 lck_mtx_unlock_always(c_list_lock);
39236c6e
A		 1052
1053#if __i386__ || __x86_64__
1054 lck_mtx_destroy(&c_seg->c_lock, &vm_compressor_lck_grp);
1055#else /* __i386__ || __x86_64__ */
1056 lck_spin_destroy(&c_seg->c_lock, &vm_compressor_lck_grp);
1057#endif /* __i386__ || __x86_64__ */
1058
3e170ce0
A		 1059 if (c_seg->c_slot_var_array_len)
1060 kfree(c_seg->c_slot_var_array, sizeof(struct c_slot) * c_seg->c_slot_var_array_len);
39236c6e 1061
39236c6e
A		 1062 zfree(compressor_segment_zone, c_seg);
1063}
1064
1065
1066int c_seg_trim_page_count = 0;
1067
1068void
1069c_seg_trim_tail(c_segment_t c_seg)
1070{
1071 c_slot_t cs;
1072 uint32_t c_size;
1073 uint32_t c_offset;
1074 uint32_t c_rounded_size;
1075 uint16_t current_nextslot;
1076 uint32_t current_populated_offset;
1077
1078 if (c_seg->c_bytes_used == 0)
1079 return;
1080 current_nextslot = c_seg->c_nextslot;
1081 current_populated_offset = c_seg->c_populated_offset;
1082
1083 while (c_seg->c_nextslot) {
1084
1085 cs = C_SEG_SLOT_FROM_INDEX(c_seg, (c_seg->c_nextslot - 1));
1086
1087 c_size = UNPACK_C_SIZE(cs);
1088
1089 if (c_size) {
1090 if (current_nextslot != c_seg->c_nextslot) {
1091 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
1092 c_offset = cs->c_offset + C_SEG_BYTES_TO_OFFSET(c_rounded_size);
1093
1094 c_seg->c_nextoffset = c_offset;
1095 c_seg->c_populated_offset = (c_offset + (C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1)) & ~(C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1);
1096
1097 if (c_seg->c_firstemptyslot > c_seg->c_nextslot)
1098 c_seg->c_firstemptyslot = c_seg->c_nextslot;
1099
1100 c_seg_trim_page_count += ((round_page_32(C_SEG_OFFSET_TO_BYTES(current_populated_offset)) -
1101 round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE);
1102 }
1103 break;
1104 }
1105 c_seg->c_nextslot--;
1106 }
1107 assert(c_seg->c_nextslot);
1108}
1109
1110
1111int
1112c_seg_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy)
1113{
1114 c_slot_mapping_t slot_ptr;
1115 uint32_t c_offset = 0;
1116 uint32_t old_populated_offset;
1117 uint32_t c_rounded_size;
1118 uint32_t c_size;
1119 int c_indx = 0;
1120 int i;
1121 c_slot_t c_dst;
1122 c_slot_t c_src;
1123 boolean_t need_unlock = TRUE;
1124
1125 assert(c_seg->c_busy);
1126
1127#if VALIDATE_C_SEGMENTS
1128 c_seg_validate(c_seg, FALSE);
1129#endif
1130 if (c_seg->c_bytes_used == 0) {
1131 c_seg_free(c_seg);
1132 return (1);
1133 }
1134 if (c_seg->c_firstemptyslot >= c_seg->c_nextslot || C_SEG_UNUSED_BYTES(c_seg) < PAGE_SIZE)
1135 goto done;
1136
1137#if VALIDATE_C_SEGMENTS
1138 c_seg->c_was_minor_compacted++;
1139#endif
1140 c_indx = c_seg->c_firstemptyslot;
1141 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
1142
1143 old_populated_offset = c_seg->c_populated_offset;
1144 c_offset = c_dst->c_offset;
1145
1146 for (i = c_indx + 1; i < c_seg->c_nextslot && c_offset < c_seg->c_nextoffset; i++) {
1147
1148 c_src = C_SEG_SLOT_FROM_INDEX(c_seg, i);
1149
1150 c_size = UNPACK_C_SIZE(c_src);
1151
1152 if (c_size == 0)
1153 continue;
1154
1155 memcpy(&c_seg->c_store.c_buffer[c_offset], &c_seg->c_store.c_buffer[c_src->c_offset], c_size);
1156
1157#if CHECKSUM_THE_DATA
1158 c_dst->c_hash_data = c_src->c_hash_data;
1159#endif
1160#if CHECKSUM_THE_COMPRESSED_DATA
1161 c_dst->c_hash_compressed_data = c_src->c_hash_compressed_data;
1162#endif
1163 c_dst->c_size = c_src->c_size;
1164 c_dst->c_packed_ptr = c_src->c_packed_ptr;
1165 c_dst->c_offset = c_offset;
1166
1167 slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
1168 slot_ptr->s_cindx = c_indx;
1169
1170 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
1171
1172 c_offset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
1173 PACK_C_SIZE(c_src, 0);
1174 c_indx++;
1175
1176 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
1177 }
1178 c_seg->c_firstemptyslot = c_indx;
1179 c_seg->c_nextslot = c_indx;
1180 c_seg->c_nextoffset = c_offset;
1181 c_seg->c_populated_offset = (c_offset + (C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1)) & ~(C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1);
1182 c_seg->c_bytes_unused = 0;
1183
1184#if VALIDATE_C_SEGMENTS
1185 c_seg_validate(c_seg, TRUE);
1186#endif
1187
1188 if (old_populated_offset > c_seg->c_populated_offset) {
1189 uint32_t gc_size;
1190 int32_t *gc_ptr;
1191
1192 gc_size = C_SEG_OFFSET_TO_BYTES(old_populated_offset - c_seg->c_populated_offset);
1193 gc_ptr = &c_seg->c_store.c_buffer[c_seg->c_populated_offset];
1194
1195 lck_mtx_unlock_always(&c_seg->c_lock);
1196
1197 kernel_memory_depopulate(kernel_map, (vm_offset_t)gc_ptr, gc_size, KMA_COMPRESSOR);
1198
1199 if (clear_busy == TRUE)
1200 lck_mtx_lock_spin_always(&c_seg->c_lock);
1201 else
1202 need_unlock = FALSE;
1203 }
1204done:
1205 if (need_unlock == TRUE) {
1206 if (clear_busy == TRUE)
1207 C_SEG_WAKEUP_DONE(c_seg);
1208
1209 lck_mtx_unlock_always(&c_seg->c_lock);
1210 }
1211 return (0);
1212}
1213
1214
3e170ce0
A		 1215static void
1216c_seg_alloc_nextslot(c_segment_t c_seg)
1217{
1218 struct c_slot *old_slot_array = NULL;
1219 struct c_slot *new_slot_array = NULL;
1220 int newlen;
1221 int oldlen;
1222
1223 if (c_seg->c_nextslot < c_seg_fixed_array_len)
1224 return;
1225
1226 if ((c_seg->c_nextslot - c_seg_fixed_array_len) >= c_seg->c_slot_var_array_len) {
1227
1228 oldlen = c_seg->c_slot_var_array_len;
1229 old_slot_array = c_seg->c_slot_var_array;
1230
1231 if (oldlen == 0)
1232 newlen = C_SEG_SLOT_VAR_ARRAY_MIN_LEN;
1233 else
1234 newlen = oldlen * 2;
1235
1236 new_slot_array = (struct c_slot *)kalloc(sizeof(struct c_slot) * newlen);
1237
1238 lck_mtx_lock_spin_always(&c_seg->c_lock);
1239
1240 if (old_slot_array)
1241 memcpy((char *)new_slot_array, (char *)old_slot_array, sizeof(struct c_slot) * oldlen);
1242
1243 c_seg->c_slot_var_array_len = newlen;
1244 c_seg->c_slot_var_array = new_slot_array;
1245
1246 lck_mtx_unlock_always(&c_seg->c_lock);
1247
1248 if (old_slot_array)
1249 kfree(old_slot_array, sizeof(struct c_slot) * oldlen);
1250 }
1251}
1252
1253
39236c6e
A		 1254
1255struct {
1256 uint64_t asked_permission;
1257 uint64_t compactions;
1258 uint64_t moved_slots;
1259 uint64_t moved_bytes;
1260 uint64_t wasted_space_in_swapouts;
1261 uint64_t count_of_swapouts;
3e170ce0 1262 uint64_t count_of_freed_segs;
39236c6e
A		 1263} c_seg_major_compact_stats;
1264
1265
04b8595b 1266#define C_MAJOR_COMPACTION_SIZE_APPROPRIATE ((C_SEG_BUFSIZE * 90) / 100)
39236c6e
A		 1267
1268
1269boolean_t
1270c_seg_major_compact_ok(
1271 c_segment_t c_seg_dst,
1272 c_segment_t c_seg_src)
1273{
1274
1275 c_seg_major_compact_stats.asked_permission++;
1276
39236c6e
A		 1277 if (c_seg_src->c_bytes_used >= C_MAJOR_COMPACTION_SIZE_APPROPRIATE &&
1278 c_seg_dst->c_bytes_used >= C_MAJOR_COMPACTION_SIZE_APPROPRIATE)
1279 return (FALSE);
1280
3e170ce0 1281 if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX_INDEX) {
39236c6e
A		 1282 /*
1283 * destination segment is full... can't compact
1284 */
1285 return (FALSE);
1286 }
1287
1288 return (TRUE);
1289}
1290
1291
1292boolean_t
1293c_seg_major_compact(
1294 c_segment_t c_seg_dst,
1295 c_segment_t c_seg_src)
1296{
1297 c_slot_mapping_t slot_ptr;
1298 uint32_t c_rounded_size;
1299 uint32_t c_size;
1300 uint16_t dst_slot;
1301 int i;
1302 c_slot_t c_dst;
1303 c_slot_t c_src;
39236c6e
A		 1304 boolean_t keep_compacting = TRUE;
1305
1306 /*
1307 * segments are not locked but they are both marked c_busy
1308 * which keeps c_decompress from working on them...
1309 * we can safely allocate new pages, move compressed data
1310 * from c_seg_src to c_seg_dst and update both c_segment's
1311 * state w/o holding the master lock
1312 */
1313
1314#if VALIDATE_C_SEGMENTS
1315 c_seg_dst->c_was_major_compacted++;
1316 c_seg_src->c_was_major_donor++;
1317#endif
1318 c_seg_major_compact_stats.compactions++;
1319
1320 dst_slot = c_seg_dst->c_nextslot;
1321
1322 for (i = 0; i < c_seg_src->c_nextslot; i++) {
1323
1324 c_src = C_SEG_SLOT_FROM_INDEX(c_seg_src, i);
1325
1326 c_size = UNPACK_C_SIZE(c_src);
1327
1328 if (c_size == 0) {
1329 /* BATCH: move what we have so far; */
1330 continue;
1331 }
1332
1333 if (C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset - c_seg_dst->c_nextoffset) < (unsigned) c_size) {
3e170ce0
A		 1334 int size_to_populate;
1335
39236c6e 1336 /* doesn't fit */
3e170ce0
A		 1337 size_to_populate = C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset);
1338
1339 if (size_to_populate == 0) {
39236c6e
A		 1340 /* can't fit */
1341 keep_compacting = FALSE;
1342 break;
1343 }
3e170ce0
A		 1344 if (size_to_populate > C_SEG_MAX_POPULATE_SIZE)
1345 size_to_populate = C_SEG_MAX_POPULATE_SIZE;
1346
39236c6e
A		 1347 kernel_memory_populate(kernel_map,
1348 (vm_offset_t) &c_seg_dst->c_store.c_buffer[c_seg_dst->c_populated_offset],
3e170ce0
A		 1349 size_to_populate,
1350 KMA_COMPRESSOR,
1351 VM_KERN_MEMORY_COMPRESSOR);
39236c6e 1352
3e170ce0 1353 c_seg_dst->c_populated_offset += C_SEG_BYTES_TO_OFFSET(size_to_populate);
39236c6e
A		 1354 assert(C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset) <= C_SEG_BUFSIZE);
1355 }
3e170ce0 1356 c_seg_alloc_nextslot(c_seg_dst);
39236c6e 1357
39236c6e
A		 1358 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, c_seg_dst->c_nextslot);
1359
1360 memcpy(&c_seg_dst->c_store.c_buffer[c_seg_dst->c_nextoffset], &c_seg_src->c_store.c_buffer[c_src->c_offset], c_size);
1361
1362 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
1363
1364 c_seg_major_compact_stats.moved_slots++;
1365 c_seg_major_compact_stats.moved_bytes += c_size;
1366
1367#if CHECKSUM_THE_DATA
1368 c_dst->c_hash_data = c_src->c_hash_data;
1369#endif
1370#if CHECKSUM_THE_COMPRESSED_DATA
1371 c_dst->c_hash_compressed_data = c_src->c_hash_compressed_data;
1372#endif
1373 c_dst->c_size = c_src->c_size;
1374 c_dst->c_packed_ptr = c_src->c_packed_ptr;
1375 c_dst->c_offset = c_seg_dst->c_nextoffset;
1376
1377 if (c_seg_dst->c_firstemptyslot == c_seg_dst->c_nextslot)
1378 c_seg_dst->c_firstemptyslot++;
1379 c_seg_dst->c_nextslot++;
1380 c_seg_dst->c_bytes_used += c_rounded_size;
1381 c_seg_dst->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
1382
1383 PACK_C_SIZE(c_src, 0);
1384
1385 c_seg_src->c_bytes_used -= c_rounded_size;
1386 c_seg_src->c_bytes_unused += c_rounded_size;
1387 c_seg_src->c_firstemptyslot = 0;
1388
3e170ce0 1389 if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX_INDEX) {
39236c6e
A		 1390 /* dest segment is now full */
1391 keep_compacting = FALSE;
1392 break;
1393 }
1394 }
1395 if (dst_slot < c_seg_dst->c_nextslot) {
1396
1397 PAGE_REPLACEMENT_ALLOWED(TRUE);
1398 /*
1399 * we've now locked out c_decompress from
1400 * converting the slot passed into it into
1401 * a c_segment_t which allows us to use
1402 * the backptr to change which c_segment and
1403 * index the slot points to
1404 */
1405 while (dst_slot < c_seg_dst->c_nextslot) {
1406
1407 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, dst_slot);
1408
1409 slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
1410 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
1411 slot_ptr->s_cseg = c_seg_dst->c_mysegno + 1;
1412 slot_ptr->s_cindx = dst_slot++;
1413 }
1414 PAGE_REPLACEMENT_ALLOWED(FALSE);
1415 }
1416 return (keep_compacting);
1417}
1418
1419
fe8ab488
A		 1420uint64_t
1421vm_compressor_compute_elapsed_msecs(clock_sec_t end_sec, clock_nsec_t end_nsec, clock_sec_t start_sec, clock_nsec_t start_nsec)
39236c6e
A		 1422{
1423 uint64_t end_msecs;
1424 uint64_t start_msecs;
1425
1426 end_msecs = (end_sec * 1000) + end_nsec / 1000000;
1427 start_msecs = (start_sec * 1000) + start_nsec / 1000000;
1428
1429 return (end_msecs - start_msecs);
1430}
1431
1432
1433
1434uint32_t compressor_eval_period_in_msecs = 250;
1435uint32_t compressor_sample_min_in_msecs = 500;
1436uint32_t compressor_sample_max_in_msecs = 10000;
1437uint32_t compressor_thrashing_threshold_per_10msecs = 50;
1438uint32_t compressor_thrashing_min_per_10msecs = 20;
1439
fe8ab488
A		 1440/* When true, reset sample data next chance we get. */
1441static boolean_t compressor_need_sample_reset = FALSE;
1442
39236c6e
A		 1443extern uint32_t vm_page_filecache_min;
1444
1445
1446void
1447compute_swapout_target_age(void)
1448{
1449 clock_sec_t cur_ts_sec;
1450 clock_nsec_t cur_ts_nsec;
1451 uint32_t min_operations_needed_in_this_sample;
1452 uint64_t elapsed_msecs_in_eval;
1453 uint64_t elapsed_msecs_in_sample;
39236c6e
A		 1454 boolean_t need_eval_reset = FALSE;
1455
1456 clock_get_system_nanotime(&cur_ts_sec, &cur_ts_nsec);
1457
fe8ab488 1458 elapsed_msecs_in_sample = vm_compressor_compute_elapsed_msecs(cur_ts_sec, cur_ts_nsec, start_of_sample_period_sec, start_of_sample_period_nsec);
39236c6e 1459
fe8ab488
A		 1460 if (compressor_need_sample_reset ||
1461 elapsed_msecs_in_sample >= compressor_sample_max_in_msecs) {
1462 compressor_need_sample_reset = TRUE;
39236c6e
A		 1463 need_eval_reset = TRUE;
1464 goto done;
1465 }
fe8ab488 1466 elapsed_msecs_in_eval = vm_compressor_compute_elapsed_msecs(cur_ts_sec, cur_ts_nsec, start_of_eval_period_sec, start_of_eval_period_nsec);
39236c6e
A		 1467
1468 if (elapsed_msecs_in_eval < compressor_eval_period_in_msecs)
1469 goto done;
1470 need_eval_reset = TRUE;
1471
1472 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_START, elapsed_msecs_in_eval, sample_period_compression_count, sample_period_decompression_count, 0, 0);
1473
1474 min_operations_needed_in_this_sample = (compressor_thrashing_min_per_10msecs * (uint32_t)elapsed_msecs_in_eval) / 10;
1475
1476 if ((sample_period_compression_count - last_eval_compression_count) < min_operations_needed_in_this_sample ||
1477 (sample_period_decompression_count - last_eval_decompression_count) < min_operations_needed_in_this_sample) {
1478
1479 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, sample_period_compression_count - last_eval_compression_count,
1480 sample_period_decompression_count - last_eval_decompression_count, 0, 1, 0);
1481
1482 swapout_target_age = 0;
1483
fe8ab488 1484 compressor_need_sample_reset = TRUE;
39236c6e
A		 1485 need_eval_reset = TRUE;
1486 goto done;
1487 }
1488 last_eval_compression_count = sample_period_compression_count;
1489 last_eval_decompression_count = sample_period_decompression_count;
1490
1491 if (elapsed_msecs_in_sample < compressor_sample_min_in_msecs) {
1492
1493 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, swapout_target_age, 0, 0, 5, 0);
1494 goto done;
1495 }
1496 if (sample_period_decompression_count > ((compressor_thrashing_threshold_per_10msecs * elapsed_msecs_in_sample) / 10)) {
1497
1498 uint64_t running_total;
1499 uint64_t working_target;
1500 uint64_t aging_target;
1501 uint32_t oldest_age_of_csegs_sampled = 0;
1502 uint64_t working_set_approximation = 0;
1503
1504 swapout_target_age = 0;
1505
1506 working_target = (sample_period_decompression_count / 100) * 95; /* 95 percent */
1507 aging_target = (sample_period_decompression_count / 100) * 1; /* 1 percent */
1508 running_total = 0;
1509
1510 for (oldest_age_of_csegs_sampled = 0; oldest_age_of_csegs_sampled < DECOMPRESSION_SAMPLE_MAX_AGE; oldest_age_of_csegs_sampled++) {
1511
1512 running_total += age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
1513
1514 working_set_approximation += oldest_age_of_csegs_sampled * age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
1515
1516 if (running_total >= working_target)
1517 break;
1518 }
1519 if (oldest_age_of_csegs_sampled < DECOMPRESSION_SAMPLE_MAX_AGE) {
1520
1521 working_set_approximation = (working_set_approximation * 1000) / elapsed_msecs_in_sample;
1522
1523 if (working_set_approximation < VM_PAGE_COMPRESSOR_COUNT) {
1524
1525 running_total = overage_decompressions_during_sample_period;
1526
1527 for (oldest_age_of_csegs_sampled = DECOMPRESSION_SAMPLE_MAX_AGE - 1; oldest_age_of_csegs_sampled; oldest_age_of_csegs_sampled--) {
1528 running_total += age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
1529
1530 if (running_total >= aging_target)
1531 break;
1532 }
1533 swapout_target_age = (uint32_t)cur_ts_sec - oldest_age_of_csegs_sampled;
1534
1535 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, swapout_target_age, working_set_approximation, VM_PAGE_COMPRESSOR_COUNT, 2, 0);
1536 } else {
1537 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, working_set_approximation, VM_PAGE_COMPRESSOR_COUNT, 0, 3, 0);
1538 }
1539 } else
1540 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, working_target, running_total, 0, 4, 0);
1541
fe8ab488 1542 compressor_need_sample_reset = TRUE;
39236c6e
A		 1543 need_eval_reset = TRUE;
1544 } else
1545 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, sample_period_decompression_count, (compressor_thrashing_threshold_per_10msecs * elapsed_msecs_in_sample) / 10, 0, 6, 0);
1546done:
fe8ab488 1547 if (compressor_need_sample_reset == TRUE) {
39236c6e
A		 1548 bzero(age_of_decompressions_during_sample_period, sizeof(age_of_decompressions_during_sample_period));
1549 overage_decompressions_during_sample_period = 0;
1550
1551 start_of_sample_period_sec = cur_ts_sec;
1552 start_of_sample_period_nsec = cur_ts_nsec;
1553 sample_period_decompression_count = 0;
1554 sample_period_compression_count = 0;
1555 last_eval_decompression_count = 0;
1556 last_eval_compression_count = 0;
fe8ab488 1557 compressor_need_sample_reset = FALSE;
39236c6e
A		 1558 }
1559 if (need_eval_reset == TRUE) {
1560 start_of_eval_period_sec = cur_ts_sec;
1561 start_of_eval_period_nsec = cur_ts_nsec;
1562 }
1563}
1564
1565
fe8ab488
A		 1566int compaction_swapper_inited = 0;
1567int compaction_swapper_init_now = 0;
39236c6e
A		 1568int compaction_swapper_running = 0;
1569int compaction_swapper_abort = 0;
1570
1571
1572#if CONFIG_JETSAM
1573boolean_t memorystatus_kill_on_VM_thrashing(boolean_t);
fe8ab488 1574boolean_t memorystatus_kill_on_FC_thrashing(boolean_t);
39236c6e 1575int compressor_thrashing_induced_jetsam = 0;
fe8ab488
A		 1576int filecache_thrashing_induced_jetsam = 0;
1577static boolean_t vm_compressor_thrashing_detected = FALSE;
39236c6e
A		 1578#endif /* CONFIG_JETSAM */
1579
1580static boolean_t
1581compressor_needs_to_swap(void)
1582{
1583 boolean_t should_swap = FALSE;
1584
3e170ce0
A		 1585 if (vm_swapout_ripe_segments == TRUE && c_overage_swapped_count < c_overage_swapped_limit) {
1586 c_segment_t c_seg;
1587 clock_sec_t now;
1588 clock_sec_t age;
1589 clock_nsec_t nsec;
1590
1591 clock_get_system_nanotime(&now, &nsec);
1592 age = 0;
1593
1594 lck_mtx_lock_spin_always(c_list_lock);
1595
1596 if ( !queue_empty(&c_age_list_head)) {
1597 c_seg = (c_segment_t) queue_first(&c_age_list_head);
1598
1599 age = now - c_seg->c_creation_ts;
1600 }
1601 lck_mtx_unlock_always(c_list_lock);
1602
1603 if (age >= vm_ripe_target_age)
1604 return (TRUE);
1605 }
1606 if ((vm_compressor_mode == VM_PAGER_COMPRESSOR_WITH_SWAP) && vm_swap_up == TRUE) {
39236c6e
A		 1607 if (COMPRESSOR_NEEDS_TO_SWAP()) {
1608 return (TRUE);
1609 }
1610 if (VM_PAGE_Q_THROTTLED(&vm_pageout_queue_external) && vm_page_anonymous_count < (vm_page_inactive_count / 20)) {
1611 return (TRUE);
1612 }
3e170ce0 1613 if (vm_page_free_count < (vm_page_free_reserved - (COMPRESSOR_FREE_RESERVED_LIMIT * 2)))
39236c6e
A		 1614 return (TRUE);
1615 }
1616 compute_swapout_target_age();
1617
1618 if (swapout_target_age) {
1619 c_segment_t c_seg;
1620
1621 lck_mtx_lock_spin_always(c_list_lock);
1622
1623 if (!queue_empty(&c_age_list_head)) {
1624
1625 c_seg = (c_segment_t) queue_first(&c_age_list_head);
1626
fe8ab488 1627 if (c_seg->c_creation_ts > swapout_target_age)
39236c6e
A		 1628 swapout_target_age = 0;
1629 }
1630 lck_mtx_unlock_always(c_list_lock);
1631 }
fe8ab488
A		 1632#if CONFIG_PHANTOM_CACHE
1633 if (vm_phantom_cache_check_pressure())
1634 should_swap = TRUE;
1635#endif
1636 if (swapout_target_age)
1637 should_swap = TRUE;
39236c6e 1638
fe8ab488 1639#if CONFIG_JETSAM
3e170ce0
A		 1640 if (should_swap || c_segment_pages_compressed > c_segment_pages_compressed_nearing_limit) {
1641
1642 if (vm_compressor_thrashing_detected == FALSE) {
1643 vm_compressor_thrashing_detected = TRUE;
fe8ab488 1644
3e170ce0
A		 1645 if (swapout_target_age || c_segment_pages_compressed > c_segment_pages_compressed_nearing_limit) {
1646 memorystatus_kill_on_VM_thrashing(TRUE /* async */);
1647 compressor_thrashing_induced_jetsam++;
1648 } else {
1649 memorystatus_kill_on_FC_thrashing(TRUE /* async */);
1650 filecache_thrashing_induced_jetsam++;
39236c6e 1651 }
3e170ce0
A		 1652 }
1653 /*
1654 * let the jetsam take precedence over
1655 * any major compactions we might have
1656 * been able to do... otherwise we run
1657 * the risk of doing major compactions
1658 * on segments we're about to free up
1659 * due to the jetsam activity.
1660 */
1661 should_swap = FALSE;
39236c6e 1662 }
fe8ab488 1663
3e170ce0
A		 1664#endif /* CONFIG_JETSAM */
1665
1666 if (should_swap == FALSE) {
1667 /*
1668 * COMPRESSOR_NEEDS_TO_MAJOR_COMPACT returns true only if we're
1669 * about to run out of available compressor segments... in this
1670 * case, we absolutely need to run a major compaction even if
1671 * we've just kicked off a jetsam or we don't otherwise need to
1672 * swap... terminating objects releases
1673 * pages back to the uncompressed cache, but does not guarantee
1674 * that we will free up even a single compression segment
1675 */
1676 should_swap = COMPRESSOR_NEEDS_TO_MAJOR_COMPACT();
1677 }
1678
1679 /*
1680 * returning TRUE when swap_supported == FALSE
39236c6e
A		 1681 * will cause the major compaction engine to
1682 * run, but will not trigger any swapping...
1683 * segments that have been major compacted
3e170ce0 1684 * will be moved to the majorcompact queue
39236c6e
A		 1685 */
1686 return (should_swap);
1687}
1688
fe8ab488
A		 1689#if CONFIG_JETSAM
1690/*
1691 * This function is called from the jetsam thread after killing something to
1692 * mitigate thrashing.
1693 *
1694 * We need to restart our thrashing detection heuristics since memory pressure
1695 * has potentially changed significantly, and we don't want to detect on old
1696 * data from before the jetsam.
1697 */
1698void
1699vm_thrashing_jetsam_done(void)
39236c6e 1700{
fe8ab488 1701 vm_compressor_thrashing_detected = FALSE;
39236c6e 1702
fe8ab488
A		 1703 /* Were we compressor-thrashing or filecache-thrashing? */
1704 if (swapout_target_age) {
1705 swapout_target_age = 0;
1706 compressor_need_sample_reset = TRUE;
39236c6e 1707 }
fe8ab488
A		 1708#if CONFIG_PHANTOM_CACHE
1709 else {
1710 vm_phantom_cache_restart_sample();
1711 }
1712#endif
39236c6e 1713}
fe8ab488 1714#endif /* CONFIG_JETSAM */
39236c6e
A		 1715
1716uint32_t vm_wake_compactor_swapper_calls = 0;
1717
1718void
1719vm_wake_compactor_swapper(void)
1720{
3e170ce0 1721 if (compaction_swapper_running || c_segment_count == 0)
39236c6e
A		 1722 return;
1723
3e170ce0
A		 1724 if (c_minor_count || COMPRESSOR_NEEDS_TO_MAJOR_COMPACT()) {
1725
1726 lck_mtx_lock_spin_always(c_list_lock);
1727
1728 fastwake_warmup = FALSE;
1729
1730 if (compaction_swapper_running == 0) {
1731
1732 vm_wake_compactor_swapper_calls++;
1733
1734 thread_wakeup((event_t)&c_compressor_swap_trigger);
1735
1736 compaction_swapper_running = 1;
1737 }
1738 lck_mtx_unlock_always(c_list_lock);
1739 }
1740}
1741
1742
1743void
1744vm_consider_swapping()
1745{
1746 c_segment_t c_seg, c_seg_next;
1747 clock_sec_t now;
1748 clock_nsec_t nsec;
1749
39236c6e
A		 1750
1751 lck_mtx_lock_spin_always(c_list_lock);
1752
3e170ce0 1753 compaction_swapper_abort = 1;
39236c6e 1754
3e170ce0
A		 1755 while (compaction_swapper_running) {
1756 assert_wait((event_t)&compaction_swapper_running, THREAD_UNINT);
39236c6e 1757
3e170ce0 1758 lck_mtx_unlock_always(c_list_lock);
39236c6e 1759
3e170ce0
A		 1760 thread_block(THREAD_CONTINUE_NULL);
1761
1762 lck_mtx_lock_spin_always(c_list_lock);
1763 }
1764 compaction_swapper_abort = 0;
1765 compaction_swapper_running = 1;
1766
1767 vm_swapout_ripe_segments = TRUE;
1768
1769 if (!queue_empty(&c_major_list_head)) {
1770
1771 clock_get_system_nanotime(&now, &nsec);
1772
1773 c_seg = (c_segment_t)queue_first(&c_major_list_head);
1774
1775 while (!queue_end(&c_major_list_head, (queue_entry_t)c_seg)) {
1776
1777 if (c_overage_swapped_count >= c_overage_swapped_limit)
1778 break;
1779
1780 c_seg_next = (c_segment_t) queue_next(&c_seg->c_age_list);
1781
1782 if ((now - c_seg->c_creation_ts) >= vm_ripe_target_age) {
1783
1784 lck_mtx_lock_spin_always(&c_seg->c_lock);
1785
1786 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
1787
1788 lck_mtx_unlock_always(&c_seg->c_lock);
1789 }
1790 c_seg = c_seg_next;
1791 }
39236c6e 1792 }
3e170ce0
A		 1793 vm_compressor_compact_and_swap(FALSE);
1794
1795 compaction_swapper_running = 0;
1796
1797 vm_swapout_ripe_segments = FALSE;
1798
39236c6e
A		 1799 lck_mtx_unlock_always(c_list_lock);
1800}
1801
fe8ab488 1802
39236c6e
A		 1803void
1804vm_consider_waking_compactor_swapper(void)
1805{
1806 boolean_t need_wakeup = FALSE;
1807
fe8ab488 1808 if (compaction_swapper_running)
39236c6e 1809 return;
fe8ab488 1810
3e170ce0
A		 1811 if (c_segment_count == 0)
1812 return;
1813
fe8ab488
A		 1814 if (!compaction_swapper_inited && !compaction_swapper_init_now) {
1815 compaction_swapper_init_now = 1;
1816 need_wakeup = TRUE;
1817 }
39236c6e
A		 1818
1819 if (c_minor_count && (COMPRESSOR_NEEDS_TO_MINOR_COMPACT())) {
1820
1821 need_wakeup = TRUE;
1822
1823 } else if (compressor_needs_to_swap()) {
1824
1825 need_wakeup = TRUE;
1826
1827 } else if (c_minor_count) {
1828 uint64_t total_bytes;
1829
1830 total_bytes = compressor_object->resident_page_count * PAGE_SIZE_64;
1831
1832 if ((total_bytes - compressor_bytes_used) > total_bytes / 10)
1833 need_wakeup = TRUE;
1834 }
1835 if (need_wakeup == TRUE) {
1836
1837 lck_mtx_lock_spin_always(c_list_lock);
1838
1839 fastwake_warmup = FALSE;
1840
1841 if (compaction_swapper_running == 0) {
1842 memoryshot(VM_WAKEUP_COMPACTOR_SWAPPER, DBG_FUNC_NONE);
1843
1844 thread_wakeup((event_t)&c_compressor_swap_trigger);
1845
1846 compaction_swapper_running = 1;
1847 }
1848 lck_mtx_unlock_always(c_list_lock);
1849 }
1850}
1851
1852
1853#define C_SWAPOUT_LIMIT 4
1854#define DELAYED_COMPACTIONS_PER_PASS 30
1855
1856void
1857vm_compressor_do_delayed_compactions(boolean_t flush_all)
1858{
1859 c_segment_t c_seg;
1860 int number_compacted = 0;
1861 boolean_t needs_to_swap = FALSE;
1862
1863
1864 lck_mtx_assert(c_list_lock, LCK_MTX_ASSERT_OWNED);
1865
1866 while (!queue_empty(&c_minor_list_head) && needs_to_swap == FALSE) {
1867
1868 c_seg = (c_segment_t)queue_first(&c_minor_list_head);
1869
1870 lck_mtx_lock_spin_always(&c_seg->c_lock);
8a3053a0 1871
fe8ab488 1872 if (c_seg->c_busy) {
8a3053a0 1873
fe8ab488
A		 1874 lck_mtx_unlock_always(c_list_lock);
1875 c_seg_wait_on_busy(c_seg);
1876 lck_mtx_lock_spin_always(c_list_lock);
8a3053a0 1877
fe8ab488
A		 1878 continue;
1879 }
1880 C_SEG_BUSY(c_seg);
39236c6e
A		 1881
1882 c_seg_do_minor_compaction_and_unlock(c_seg, TRUE, FALSE, TRUE);
1883
1884 if (vm_swap_up == TRUE && (number_compacted++ > DELAYED_COMPACTIONS_PER_PASS)) {
1885
1886 if ((flush_all == TRUE || compressor_needs_to_swap() == TRUE) && c_swapout_count < C_SWAPOUT_LIMIT)
1887 needs_to_swap = TRUE;
1888
1889 number_compacted = 0;
1890 }
1891 lck_mtx_lock_spin_always(c_list_lock);
1892 }
1893}
1894
1895
1896#define C_SEGMENT_SWAPPEDIN_AGE_LIMIT 10
1897
1898static void
1899vm_compressor_age_swapped_in_segments(boolean_t flush_all)
1900{
1901 c_segment_t c_seg;
1902 clock_sec_t now;
1903 clock_nsec_t nsec;
1904
1905 clock_get_system_nanotime(&now, &nsec);
1906
1907 while (!queue_empty(&c_swappedin_list_head)) {
1908
1909 c_seg = (c_segment_t)queue_first(&c_swappedin_list_head);
1910
1911 if (flush_all == FALSE && (now - c_seg->c_swappedin_ts) < C_SEGMENT_SWAPPEDIN_AGE_LIMIT)
1912 break;
1913
1914 lck_mtx_lock_spin_always(&c_seg->c_lock);
1915
3e170ce0 1916 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
39236c6e
A		 1917
1918 lck_mtx_unlock_always(&c_seg->c_lock);
1919 }
1920}
1921
1922
1923void
1924vm_compressor_flush(void)
1925{
1926 uint64_t vm_swap_put_failures_at_start;
1927 wait_result_t wait_result = 0;
1928 AbsoluteTime startTime, endTime;
1929 clock_sec_t now_sec;
1930 clock_nsec_t now_nsec;
1931 uint64_t nsec;
1932
1933 HIBLOG("vm_compressor_flush - starting\n");
1934
1935 clock_get_uptime(&startTime);
1936
1937 lck_mtx_lock_spin_always(c_list_lock);
1938
1939 fastwake_warmup = FALSE;
1940 compaction_swapper_abort = 1;
1941
1942 while (compaction_swapper_running) {
1943 assert_wait((event_t)&compaction_swapper_running, THREAD_UNINT);
1944
1945 lck_mtx_unlock_always(c_list_lock);
1946
1947 thread_block(THREAD_CONTINUE_NULL);
1948
1949 lck_mtx_lock_spin_always(c_list_lock);
1950 }
1951 compaction_swapper_abort = 0;
1952 compaction_swapper_running = 1;
1953
1954 hibernate_flushing = TRUE;
1955 hibernate_no_swapspace = FALSE;
1956 c_generation_id_flush_barrier = c_generation_id + 1000;
1957
1958 clock_get_system_nanotime(&now_sec, &now_nsec);
1959 hibernate_flushing_deadline = now_sec + HIBERNATE_FLUSHING_SECS_TO_COMPLETE;
1960
1961 vm_swap_put_failures_at_start = vm_swap_put_failures;
1962
1963 vm_compressor_compact_and_swap(TRUE);
1964
1965 while (!queue_empty(&c_swapout_list_head)) {
1966
1967 assert_wait_timeout((event_t) &compaction_swapper_running, THREAD_INTERRUPTIBLE, 5000, 1000*NSEC_PER_USEC);
1968
1969 lck_mtx_unlock_always(c_list_lock);
1970
1971 wait_result = thread_block(THREAD_CONTINUE_NULL);
1972
1973 lck_mtx_lock_spin_always(c_list_lock);
1974
1975 if (wait_result == THREAD_TIMED_OUT)
1976 break;
1977 }
1978 hibernate_flushing = FALSE;
1979 compaction_swapper_running = 0;
1980
1981 if (vm_swap_put_failures > vm_swap_put_failures_at_start)
1982 HIBLOG("vm_compressor_flush failed to clean %llu segments - vm_page_compressor_count(%d)\n",
1983 vm_swap_put_failures - vm_swap_put_failures_at_start, VM_PAGE_COMPRESSOR_COUNT);
1984
1985 lck_mtx_unlock_always(c_list_lock);
1986
1987 clock_get_uptime(&endTime);
1988 SUB_ABSOLUTETIME(&endTime, &startTime);
1989 absolutetime_to_nanoseconds(endTime, &nsec);
1990
1991 HIBLOG("vm_compressor_flush completed - took %qd msecs\n", nsec / 1000000ULL);
1992}
1993
1994
fe8ab488 1995extern void vm_swap_file_set_tuneables(void);
39236c6e
A		 1996int compaction_swap_trigger_thread_awakened = 0;
1997
fe8ab488 1998
39236c6e
A		 1999static void
2000vm_compressor_swap_trigger_thread(void)
2001{
3e170ce0
A		 2002 current_thread()->options |= TH_OPT_VMPRIV;
2003
fe8ab488
A		 2004 /*
2005 * compaction_swapper_init_now is set when the first call to
2006 * vm_consider_waking_compactor_swapper is made from
2007 * vm_pageout_scan... since this function is called upon
2008 * thread creation, we want to make sure to delay adjusting
2009 * the tuneables until we are awakened via vm_pageout_scan
2010 * so that we are at a point where the vm_swapfile_open will
2011 * be operating on the correct directory (in case the default
2012 * of /var/vm/ is overridden by the dymanic_pager
2013 */
2014 if (compaction_swapper_init_now && !compaction_swapper_inited) {
2015 if (vm_compressor_mode == VM_PAGER_COMPRESSOR_WITH_SWAP)
2016 vm_swap_file_set_tuneables();
39236c6e 2017
3e170ce0
A		 2018 if (vm_restricted_to_single_processor == TRUE)
2019 thread_vm_bind_group_add();
2020
fe8ab488
A		 2021 compaction_swapper_inited = 1;
2022 }
39236c6e
A		 2023 lck_mtx_lock_spin_always(c_list_lock);
2024
2025 compaction_swap_trigger_thread_awakened++;
2026
2027 vm_compressor_compact_and_swap(FALSE);
2028
2029 assert_wait((event_t)&c_compressor_swap_trigger, THREAD_UNINT);
2030
2031 compaction_swapper_running = 0;
2032 thread_wakeup((event_t)&compaction_swapper_running);
2033
2034 lck_mtx_unlock_always(c_list_lock);
2035
2036 thread_block((thread_continue_t)vm_compressor_swap_trigger_thread);
2037
2038 /* NOTREACHED */
2039}
2040
2041
2042void
2043vm_compressor_record_warmup_start(void)
2044{
2045 c_segment_t c_seg;
2046
2047 lck_mtx_lock_spin_always(c_list_lock);
2048
8a3053a0
A		 2049 if (first_c_segment_to_warm_generation_id == 0) {
2050 if (!queue_empty(&c_age_list_head)) {
39236c6e 2051
8a3053a0 2052 c_seg = (c_segment_t)queue_last(&c_age_list_head);
39236c6e 2053
8a3053a0
A		 2054 first_c_segment_to_warm_generation_id = c_seg->c_generation_id;
2055 } else
2056 first_c_segment_to_warm_generation_id = 0;
39236c6e 2057
8a3053a0
A		 2058 fastwake_recording_in_progress = TRUE;
2059 }
39236c6e
A		 2060 lck_mtx_unlock_always(c_list_lock);
2061}
2062
2063
2064void
2065vm_compressor_record_warmup_end(void)
2066{
2067 c_segment_t c_seg;
2068
2069 lck_mtx_lock_spin_always(c_list_lock);
2070
8a3053a0 2071 if (fastwake_recording_in_progress == TRUE) {
39236c6e 2072
8a3053a0 2073 if (!queue_empty(&c_age_list_head)) {
39236c6e 2074
8a3053a0
A		 2075 c_seg = (c_segment_t)queue_last(&c_age_list_head);
2076
2077 last_c_segment_to_warm_generation_id = c_seg->c_generation_id;
2078 } else
2079 last_c_segment_to_warm_generation_id = first_c_segment_to_warm_generation_id;
39236c6e 2080
8a3053a0 2081 fastwake_recording_in_progress = FALSE;
39236c6e 2082
8a3053a0
A		 2083 HIBLOG("vm_compressor_record_warmup (%qd - %qd)\n", first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id);
2084 }
39236c6e
A		 2085 lck_mtx_unlock_always(c_list_lock);
2086}
2087
2088
2089#define DELAY_TRIM_ON_WAKE_SECS 4
2090
2091void
8a3053a0 2092vm_compressor_delay_trim(void)
39236c6e 2093{
8a3053a0 2094 clock_sec_t sec;
39236c6e
A		 2095 clock_nsec_t nsec;
2096
2097 clock_get_system_nanotime(&sec, &nsec);
2098 dont_trim_until_ts = sec + DELAY_TRIM_ON_WAKE_SECS;
8a3053a0 2099}
39236c6e 2100
39236c6e 2101
8a3053a0
A		 2102void
2103vm_compressor_do_warmup(void)
2104{
39236c6e
A		 2105 lck_mtx_lock_spin_always(c_list_lock);
2106
8a3053a0
A		 2107 if (first_c_segment_to_warm_generation_id == last_c_segment_to_warm_generation_id) {
2108 first_c_segment_to_warm_generation_id = last_c_segment_to_warm_generation_id = 0;
2109
2110 lck_mtx_unlock_always(c_list_lock);
2111 return;
2112 }
2113
39236c6e
A		 2114 if (compaction_swapper_running == 0) {
2115
2116 fastwake_warmup = TRUE;
2117 compaction_swapper_running = 1;
2118 thread_wakeup((event_t)&c_compressor_swap_trigger);
2119 }
2120 lck_mtx_unlock_always(c_list_lock);
2121}
2122
2123
2124void
2125do_fastwake_warmup(void)
2126{
2127 uint64_t my_thread_id;
2128 c_segment_t c_seg = NULL;
8a3053a0
A		 2129 AbsoluteTime startTime, endTime;
2130 uint64_t nsec;
2131
2132
2133 HIBLOG("vm_compressor_fastwake_warmup (%qd - %qd) - starting\n", first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id);
2134
2135 clock_get_uptime(&startTime);
39236c6e
A		 2136
2137 lck_mtx_unlock_always(c_list_lock);
2138
2139 my_thread_id = current_thread()->thread_id;
2140 proc_set_task_policy_thread(kernel_task, my_thread_id,
2141 TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER2);
2142
2143 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2144
2145 lck_mtx_lock_spin_always(c_list_lock);
2146
2147 while (!queue_empty(&c_swappedout_list_head) && fastwake_warmup == TRUE) {
2148
2149 c_seg = (c_segment_t) queue_first(&c_swappedout_list_head);
2150
2151 if (c_seg->c_generation_id < first_c_segment_to_warm_generation_id ||
2152 c_seg->c_generation_id > last_c_segment_to_warm_generation_id)
2153 break;
2154
2155 lck_mtx_lock_spin_always(&c_seg->c_lock);
2156 lck_mtx_unlock_always(c_list_lock);
2157
8a3053a0
A		 2158 if (c_seg->c_busy) {
2159 PAGE_REPLACEMENT_DISALLOWED(FALSE);
39236c6e 2160 c_seg_wait_on_busy(c_seg);
8a3053a0
A		 2161 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2162 } else {
39236c6e
A		 2163 c_seg_swapin(c_seg, TRUE);
2164
2165 lck_mtx_unlock_always(&c_seg->c_lock);
39236c6e 2166 c_segment_warmup_count++;
8a3053a0
A		 2167
2168 PAGE_REPLACEMENT_DISALLOWED(FALSE);
39236c6e 2169 vm_pageout_io_throttle();
8a3053a0 2170 PAGE_REPLACEMENT_DISALLOWED(TRUE);
39236c6e
A		 2171 }
2172 lck_mtx_lock_spin_always(c_list_lock);
2173 }
2174 lck_mtx_unlock_always(c_list_lock);
2175
2176 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2177
2178 proc_set_task_policy_thread(kernel_task, my_thread_id,
2179 TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER0);
2180
8a3053a0
A		 2181 clock_get_uptime(&endTime);
2182 SUB_ABSOLUTETIME(&endTime, &startTime);
2183 absolutetime_to_nanoseconds(endTime, &nsec);
2184
2185 HIBLOG("vm_compressor_fastwake_warmup completed - took %qd msecs\n", nsec / 1000000ULL);
2186
39236c6e 2187 lck_mtx_lock_spin_always(c_list_lock);
8a3053a0
A		 2188
2189 first_c_segment_to_warm_generation_id = last_c_segment_to_warm_generation_id = 0;
39236c6e
A		 2190}
2191
2192
2193void
2194vm_compressor_compact_and_swap(boolean_t flush_all)
2195{
2196 c_segment_t c_seg, c_seg_next;
2197 boolean_t keep_compacting;
3e170ce0
A		 2198 clock_sec_t now;
2199 clock_nsec_t nsec;
39236c6e
A		 2200
2201
2202 if (fastwake_warmup == TRUE) {
2203 uint64_t starting_warmup_count;
2204
2205 starting_warmup_count = c_segment_warmup_count;
2206
2207 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 11) | DBG_FUNC_START, c_segment_warmup_count,
2208 first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id, 0, 0);
2209 do_fastwake_warmup();
2210 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 11) | DBG_FUNC_END, c_segment_warmup_count, c_segment_warmup_count - starting_warmup_count, 0, 0, 0);
2211
2212 fastwake_warmup = FALSE;
2213 }
2214
8a3053a0
A		 2215 /*
2216 * it's possible for the c_age_list_head to be empty if we
2217 * hit our limits for growing the compressor pool and we subsequently
2218 * hibernated... on the next hibernation we could see the queue as
2219 * empty and not proceeed even though we have a bunch of segments on
2220 * the swapped in queue that need to be dealt with.
2221 */
2222 vm_compressor_do_delayed_compactions(flush_all);
2223
2224 vm_compressor_age_swapped_in_segments(flush_all);
2225
3e170ce0
A		 2226 /*
2227 * we only need to grab the timestamp once per
2228 * invocation of this function since the
2229 * timescale we're interested in is measured
2230 * in days
2231 */
2232 clock_get_system_nanotime(&now, &nsec);
8a3053a0 2233
39236c6e
A		 2234 while (!queue_empty(&c_age_list_head) && compaction_swapper_abort == 0) {
2235
2236 if (hibernate_flushing == TRUE) {
2237 clock_sec_t sec;
39236c6e
A		 2238
2239 if (hibernate_should_abort()) {
2240 HIBLOG("vm_compressor_flush - hibernate_should_abort returned TRUE\n");
2241 break;
2242 }
2243 if (hibernate_no_swapspace == TRUE) {
2244 HIBLOG("vm_compressor_flush - out of swap space\n");
2245 break;
2246 }
2247 clock_get_system_nanotime(&sec, &nsec);
2248
2249 if (sec > hibernate_flushing_deadline) {
2250 HIBLOG("vm_compressor_flush - failed to finish before deadline\n");
2251 break;
2252 }
2253 }
2254 if (c_swapout_count >= C_SWAPOUT_LIMIT) {
2255
2256 assert_wait_timeout((event_t) &compaction_swapper_running, THREAD_INTERRUPTIBLE, 100, 1000*NSEC_PER_USEC);
2257
2258 lck_mtx_unlock_always(c_list_lock);
2259
2260 thread_block(THREAD_CONTINUE_NULL);
2261
2262 lck_mtx_lock_spin_always(c_list_lock);
2263 }
2264 /*
2265 * Minor compactions
2266 */
2267 vm_compressor_do_delayed_compactions(flush_all);
2268
2269 vm_compressor_age_swapped_in_segments(flush_all);
2270
2271 if (c_swapout_count >= C_SWAPOUT_LIMIT) {
2272 /*
2273 * we timed out on the above thread_block
2274 * let's loop around and try again
2275 * the timeout allows us to continue
2276 * to do minor compactions to make
2277 * more memory available
2278 */
2279 continue;
2280 }
2281
2282 /*
2283 * Swap out segments?
2284 */
2285 if (flush_all == FALSE) {
2286 boolean_t needs_to_swap;
2287
2288 lck_mtx_unlock_always(c_list_lock);
2289
2290 needs_to_swap = compressor_needs_to_swap();
2291
3e170ce0
A		 2292 if (needs_to_swap == TRUE && vm_swap_low_on_space())
2293 vm_compressor_take_paging_space_action();
2294
39236c6e
A		 2295 lck_mtx_lock_spin_always(c_list_lock);
2296
2297 if (needs_to_swap == FALSE)
2298 break;
2299 }
2300 if (queue_empty(&c_age_list_head))
2301 break;
2302 c_seg = (c_segment_t) queue_first(&c_age_list_head);
2303
3e170ce0 2304 assert(c_seg->c_state == C_ON_AGE_Q);
39236c6e 2305
3e170ce0 2306 if (flush_all == TRUE && c_seg->c_generation_id > c_generation_id_flush_barrier)
39236c6e 2307 break;
3e170ce0 2308
39236c6e
A		 2309 lck_mtx_lock_spin_always(&c_seg->c_lock);
2310
2311 if (c_seg->c_busy) {
2312
2313 lck_mtx_unlock_always(c_list_lock);
2314 c_seg_wait_on_busy(c_seg);
2315 lck_mtx_lock_spin_always(c_list_lock);
2316
2317 continue;
2318 }
fe8ab488 2319 C_SEG_BUSY(c_seg);
39236c6e
A		 2320
2321 if (c_seg_do_minor_compaction_and_unlock(c_seg, FALSE, TRUE, TRUE)) {
2322 /*
2323 * found an empty c_segment and freed it
2324 * so go grab the next guy in the queue
2325 */
3e170ce0 2326 c_seg_major_compact_stats.count_of_freed_segs++;
39236c6e
A		 2327 continue;
2328 }
2329 /*
2330 * Major compaction
2331 */
2332 keep_compacting = TRUE;
2333
2334 while (keep_compacting == TRUE) {
2335
2336 assert(c_seg->c_busy);
2337
2338 /* look for another segment to consolidate */
2339
2340 c_seg_next = (c_segment_t) queue_next(&c_seg->c_age_list);
2341
2342 if (queue_end(&c_age_list_head, (queue_entry_t)c_seg_next))
2343 break;
2344
3e170ce0
A		 2345 assert(c_seg_next->c_state == C_ON_AGE_Q);
2346
39236c6e
A		 2347 if (c_seg_major_compact_ok(c_seg, c_seg_next) == FALSE)
2348 break;
2349
2350 lck_mtx_lock_spin_always(&c_seg_next->c_lock);
2351
2352 if (c_seg_next->c_busy) {
2353
2354 lck_mtx_unlock_always(c_list_lock);
2355 c_seg_wait_on_busy(c_seg_next);
2356 lck_mtx_lock_spin_always(c_list_lock);
2357
2358 continue;
2359 }
2360 /* grab that segment */
fe8ab488 2361 C_SEG_BUSY(c_seg_next);
39236c6e
A		 2362
2363 if (c_seg_do_minor_compaction_and_unlock(c_seg_next, FALSE, TRUE, TRUE)) {
2364 /*
2365 * found an empty c_segment and freed it
2366 * so we can't continue to use c_seg_next
2367 */
3e170ce0 2368 c_seg_major_compact_stats.count_of_freed_segs++;
39236c6e
A		 2369 continue;
2370 }
2371
2372 /* unlock the list ... */
2373 lck_mtx_unlock_always(c_list_lock);
2374
2375 /* do the major compaction */
2376
2377 keep_compacting = c_seg_major_compact(c_seg, c_seg_next);
2378
2379 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2380
2381 lck_mtx_lock_spin_always(&c_seg_next->c_lock);
2382 /*
2383 * run a minor compaction on the donor segment
2384 * since we pulled at least some of it's
2385 * data into our target... if we've emptied
2386 * it, now is a good time to free it which
2387 * c_seg_minor_compaction_and_unlock also takes care of
2388 *
2389 * by passing TRUE, we ask for c_busy to be cleared
2390 * and c_wanted to be taken care of
2391 */
3e170ce0
A		 2392 if (c_seg_minor_compaction_and_unlock(c_seg_next, TRUE))
2393 c_seg_major_compact_stats.count_of_freed_segs++;
39236c6e
A		 2394
2395 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2396
2397 /* relock the list */
2398 lck_mtx_lock_spin_always(c_list_lock);
2399
2400 } /* major compaction */
2401
39236c6e
A		 2402 lck_mtx_lock_spin_always(&c_seg->c_lock);
2403
2404 assert(c_seg->c_busy);
39236c6e
A		 2405 assert(!c_seg->c_on_minorcompact_q);
2406
39236c6e 2407 if (vm_swap_up == TRUE) {
3e170ce0
A		 2408 /*
2409 * This mode of putting a generic c_seg on the swapout list is
2410 * only supported when we have general swap ON i.e.
2411 * we compress pages into c_segs as we process them off
2412 * the paging queues in vm_pageout_scan().
2413 */
2414 if (COMPRESSED_PAGER_IS_SWAPBACKED)
2415 c_seg_switch_state(c_seg, C_ON_SWAPOUT_Q, FALSE);
2416 else {
2417 if ((vm_swapout_ripe_segments == TRUE && c_overage_swapped_count < c_overage_swapped_limit)) {
2418 /*
2419 * we are running compressor sweeps with swap-behind
2420 * make sure the c_seg has aged enough before swapping it
2421 * out...
2422 */
2423 if ((now - c_seg->c_creation_ts) >= vm_ripe_target_age) {
2424 c_seg->c_overage_swap = TRUE;
2425 c_overage_swapped_count++;
2426 c_seg_switch_state(c_seg, C_ON_SWAPOUT_Q, FALSE);
2427 }
2428 }
2429 }
2430 }
2431 if (c_seg->c_state == C_ON_AGE_Q) {
2432 /*
2433 * this c_seg didn't get moved to the swapout queue
2434 * so we need to move it out of the way...
2435 * we just did a major compaction on it so put it
2436 * on that queue
2437 */
2438 c_seg_switch_state(c_seg, C_ON_MAJORCOMPACT_Q, FALSE);
39236c6e 2439 } else {
3e170ce0
A		 2440 c_seg_major_compact_stats.wasted_space_in_swapouts += C_SEG_BUFSIZE - c_seg->c_bytes_used;
2441 c_seg_major_compact_stats.count_of_swapouts++;
39236c6e
A		 2442 }
2443 C_SEG_WAKEUP_DONE(c_seg);
2444
2445 lck_mtx_unlock_always(&c_seg->c_lock);
2446
2447 if (c_swapout_count) {
2448 lck_mtx_unlock_always(c_list_lock);
2449
2450 thread_wakeup((event_t)&c_swapout_list_head);
2451
2452 lck_mtx_lock_spin_always(c_list_lock);
2453 }
2454 }
2455}
2456
2457
2458static c_segment_t
2459c_seg_allocate(c_segment_t *current_chead)
2460{
39236c6e 2461 c_segment_t c_seg;
3e170ce0
A		 2462 int min_needed;
2463 int size_to_populate;
2464
2465 if (vm_compressor_low_on_space())
2466 vm_compressor_take_paging_space_action();
39236c6e
A		 2467
2468 if ( (c_seg = *current_chead) == NULL ) {
2469 uint32_t c_segno;
2470
39236c6e
A		 2471 lck_mtx_lock_spin_always(c_list_lock);
2472
2473 while (c_segments_busy == TRUE) {
2474 assert_wait((event_t) (&c_segments_busy), THREAD_UNINT);
2475
2476 lck_mtx_unlock_always(c_list_lock);
2477
2478 thread_block(THREAD_CONTINUE_NULL);
2479
2480 lck_mtx_lock_spin_always(c_list_lock);
2481 }
2482 if (c_free_segno_head == (uint32_t)-1) {
3e170ce0 2483 uint32_t c_segments_available_new;
39236c6e
A		 2484
2485 if (c_segments_available >= c_segments_limit || c_segment_pages_compressed >= c_segment_pages_compressed_limit) {
2486 lck_mtx_unlock_always(c_list_lock);
2487
39236c6e
A		 2488 return (NULL);
2489 }
2490 c_segments_busy = TRUE;
2491 lck_mtx_unlock_always(c_list_lock);
2492
3e170ce0
A		 2493 kernel_memory_populate(kernel_map, (vm_offset_t)c_segments_next_page,
2494 PAGE_SIZE, KMA_KOBJECT, VM_KERN_MEMORY_COMPRESSOR);
39236c6e
A		 2495 c_segments_next_page += PAGE_SIZE;
2496
3e170ce0
A		 2497 c_segments_available_new = c_segments_available + C_SEGMENTS_PER_PAGE;
2498
2499 if (c_segments_available_new > c_segments_limit)
2500 c_segments_available_new = c_segments_limit;
2501
2502 for (c_segno = c_segments_available + 1; c_segno < c_segments_available_new; c_segno++)
39236c6e
A		 2503 c_segments[c_segno - 1].c_segno = c_segno;
2504
2505 lck_mtx_lock_spin_always(c_list_lock);
2506
2507 c_segments[c_segno - 1].c_segno = c_free_segno_head;
2508 c_free_segno_head = c_segments_available;
3e170ce0 2509 c_segments_available = c_segments_available_new;
39236c6e
A		 2510
2511 c_segments_busy = FALSE;
2512 thread_wakeup((event_t) (&c_segments_busy));
2513 }
2514 c_segno = c_free_segno_head;
3e170ce0
A		 2515 assert(c_segno >= 0 && c_segno < c_segments_limit);
2516
39236c6e
A		 2517 c_free_segno_head = c_segments[c_segno].c_segno;
2518
3e170ce0
A		 2519 /*
2520 * do the rest of the bookkeeping now while we're still behind
2521 * the list lock and grab our generation id now into a local
2522 * so that we can install it once we have the c_seg allocated
2523 */
2524 c_segment_count++;
2525 if (c_segment_count > c_segment_count_max)
2526 c_segment_count_max = c_segment_count;
2527
39236c6e
A		 2528 lck_mtx_unlock_always(c_list_lock);
2529
2530 c_seg = (c_segment_t)zalloc(compressor_segment_zone);
2531 bzero((char *)c_seg, sizeof(struct c_segment));
2532
3e170ce0 2533 c_seg->c_store.c_buffer = (int32_t *)C_SEG_BUFFER_ADDRESS(c_segno);
39236c6e
A		 2534
2535#if __i386__ || __x86_64__
2536 lck_mtx_init(&c_seg->c_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
2537#else /* __i386__ || __x86_64__ */
2538 lck_spin_init(&c_seg->c_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
2539#endif /* __i386__ || __x86_64__ */
2540
3e170ce0
A		 2541 c_seg->c_state = C_IS_EMPTY;
2542 c_seg->c_firstemptyslot = C_SLOT_MAX_INDEX;
39236c6e 2543 c_seg->c_mysegno = c_segno;
39236c6e
A		 2544
2545 lck_mtx_lock_spin_always(c_list_lock);
3e170ce0
A		 2546 c_empty_count++;
2547 c_seg_switch_state(c_seg, C_IS_FILLING, FALSE);
39236c6e 2548 c_segments[c_segno].c_seg = c_seg;
39236c6e
A		 2549 lck_mtx_unlock_always(c_list_lock);
2550
39236c6e 2551 *current_chead = c_seg;
39236c6e 2552 }
3e170ce0
A		 2553 c_seg_alloc_nextslot(c_seg);
2554
2555 size_to_populate = C_SEG_ALLOCSIZE - C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset);
2556
2557 if (size_to_populate) {
2558
2559 min_needed = PAGE_SIZE + (C_SEG_ALLOCSIZE - C_SEG_BUFSIZE);
39236c6e 2560
3e170ce0 2561 if (C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset - c_seg->c_nextoffset) < (unsigned) min_needed) {
39236c6e 2562
3e170ce0
A		 2563 if (size_to_populate > C_SEG_MAX_POPULATE_SIZE)
2564 size_to_populate = C_SEG_MAX_POPULATE_SIZE;
39236c6e 2565
3e170ce0
A		 2566 kernel_memory_populate(kernel_map,
2567 (vm_offset_t) &c_seg->c_store.c_buffer[c_seg->c_populated_offset],
2568 size_to_populate,
2569 KMA_COMPRESSOR,
2570 VM_KERN_MEMORY_COMPRESSOR);
2571 } else
2572 size_to_populate = 0;
39236c6e 2573 }
39236c6e
A		 2574 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2575
2576 lck_mtx_lock_spin_always(&c_seg->c_lock);
2577
3e170ce0
A		 2578 if (size_to_populate)
2579 c_seg->c_populated_offset += C_SEG_BYTES_TO_OFFSET(size_to_populate);
2580
39236c6e
A		 2581 return (c_seg);
2582}
2583
2584
39236c6e
A		 2585static void
2586c_current_seg_filled(c_segment_t c_seg, c_segment_t *current_chead)
2587{
2588 uint32_t unused_bytes;
2589 uint32_t offset_to_depopulate;
3e170ce0
A		 2590 int new_state = C_ON_AGE_Q;
2591 clock_sec_t sec;
2592 clock_nsec_t nsec;
39236c6e
A		 2593
2594 unused_bytes = trunc_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset - c_seg->c_nextoffset));
2595
2596 if (unused_bytes) {
2597
2598 offset_to_depopulate = C_SEG_BYTES_TO_OFFSET(round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_nextoffset)));
2599
2600 /*
2601 * release the extra physical page(s) at the end of the segment
2602 */
2603 lck_mtx_unlock_always(&c_seg->c_lock);
2604
2605 kernel_memory_depopulate(
2606 kernel_map,
2607 (vm_offset_t) &c_seg->c_store.c_buffer[offset_to_depopulate],
2608 unused_bytes,
2609 KMA_COMPRESSOR);
2610
2611 lck_mtx_lock_spin_always(&c_seg->c_lock);
2612
2613 c_seg->c_populated_offset = offset_to_depopulate;
2614 }
3e170ce0 2615 assert(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset) <= C_SEG_BUFSIZE);
39236c6e 2616
3e170ce0
A		 2617#if CONFIG_FREEZE
2618 if (current_chead == (c_segment_t*)&freezer_chead && DEFAULT_FREEZER_COMPRESSED_PAGER_IS_SWAPBACKED &&
2619 c_freezer_swapout_count < VM_MAX_FREEZER_CSEG_SWAP_COUNT) {
2620 new_state = C_ON_SWAPOUT_Q;
2621 }
2622#endif /* CONFIG_FREEZE */
39236c6e 2623
3e170ce0
A		 2624 clock_get_system_nanotime(&sec, &nsec);
2625 c_seg->c_creation_ts = (uint32_t)sec;
2626
2627 lck_mtx_lock_spin_always(c_list_lock);
2628
2629#if CONFIG_FREEZE
2630 if (c_seg->c_state == C_ON_SWAPOUT_Q)
2631 c_freezer_swapout_count++;
2632#endif /* CONFIG_FREEZE */
2633
2634 c_seg->c_generation_id = c_generation_id++;
2635 c_seg_switch_state(c_seg, new_state, FALSE);
2636
2637 lck_mtx_unlock_always(c_list_lock);
2638
2639#if CONFIG_FREEZE
2640 if (c_seg->c_state == C_ON_SWAPOUT_Q)
2641 thread_wakeup((event_t)&c_swapout_list_head);
2642#endif /* CONFIG_FREEZE */
2643
2644 if (c_seg->c_state == C_ON_AGE_Q && C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE)
2645 c_seg_need_delayed_compaction(c_seg);
39236c6e
A		 2646
2647 *current_chead = NULL;
2648}
2649
39236c6e
A		 2650/*
2651 * returns with c_seg locked
2652 */
2653void
3e170ce0 2654c_seg_swapin_requeue(c_segment_t c_seg, boolean_t has_data)
39236c6e
A		 2655{
2656 clock_sec_t sec;
2657 clock_nsec_t nsec;
2658
2659 clock_get_system_nanotime(&sec, &nsec);
2660
2661 lck_mtx_lock_spin_always(c_list_lock);
2662 lck_mtx_lock_spin_always(&c_seg->c_lock);
2663
3e170ce0
A		 2664 c_seg->c_busy_swapping = 0;
2665
2666 if (c_seg->c_overage_swap == TRUE) {
2667 c_overage_swapped_count--;
2668 c_seg->c_overage_swap = FALSE;
2669 }
2670 if (has_data == TRUE) {
2671 c_seg_switch_state(c_seg, C_ON_SWAPPEDIN_Q, FALSE);
39236c6e 2672 } else {
3e170ce0
A		 2673 c_seg->c_store.c_buffer = (int32_t*) NULL;
2674 c_seg->c_populated_offset = C_SEG_BYTES_TO_OFFSET(0);
39236c6e 2675
3e170ce0 2676 c_seg_switch_state(c_seg, C_ON_BAD_Q, FALSE);
39236c6e 2677 }
39236c6e 2678 c_seg->c_swappedin_ts = (uint32_t)sec;
39236c6e
A		 2679
2680 lck_mtx_unlock_always(c_list_lock);
2681}
2682
2683
2684
2685/*
2686 * c_seg has to be locked and is returned locked.
2687 * PAGE_REPLACMENT_DISALLOWED has to be TRUE on entry and is returned TRUE
2688 */
2689
2690void
2691c_seg_swapin(c_segment_t c_seg, boolean_t force_minor_compaction)
2692{
2693 vm_offset_t addr = 0;
2694 uint32_t io_size = 0;
2695 uint64_t f_offset;
2696
3e170ce0
A		 2697 assert(C_SEG_IS_ONDISK(c_seg));
2698
39236c6e 2699#if !CHECKSUM_THE_SWAP
3e170ce0 2700 c_seg_trim_tail(c_seg);
39236c6e
A		 2701#endif
2702 io_size = round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset));
2703 f_offset = c_seg->c_store.c_swap_handle;
fe8ab488
A		 2704
2705 C_SEG_BUSY(c_seg);
3e170ce0 2706 c_seg->c_busy_swapping = 1;
39236c6e 2707 lck_mtx_unlock_always(&c_seg->c_lock);
39236c6e 2708
3e170ce0 2709 PAGE_REPLACEMENT_DISALLOWED(FALSE);
39236c6e 2710
3e170ce0 2711 addr = (vm_offset_t)C_SEG_BUFFER_ADDRESS(c_seg->c_mysegno);
39236c6e 2712
3e170ce0 2713 kernel_memory_populate(kernel_map, addr, io_size, KMA_COMPRESSOR, VM_KERN_MEMORY_COMPRESSOR);
39236c6e 2714
3e170ce0
A		 2715 if (vm_swap_get(addr, f_offset, io_size) != KERN_SUCCESS) {
2716 PAGE_REPLACEMENT_DISALLOWED(TRUE);
39236c6e 2717
3e170ce0 2718 kernel_memory_depopulate(kernel_map, addr, io_size, KMA_COMPRESSOR);
39236c6e 2719
3e170ce0
A		 2720 c_seg_swapin_requeue(c_seg, FALSE);
2721 } else {
2722 c_seg->c_store.c_buffer = (int32_t*) addr;
fe8ab488 2723#if ENCRYPTED_SWAP
3e170ce0 2724 vm_swap_decrypt(c_seg);
fe8ab488 2725#endif /* ENCRYPTED_SWAP */
39236c6e
A		 2726
2727#if CHECKSUM_THE_SWAP
3e170ce0
A		 2728 if (c_seg->cseg_swap_size != io_size)
2729 panic("swapin size doesn't match swapout size");
39236c6e 2730
3e170ce0
A		 2731 if (c_seg->cseg_hash != hash_string((char*) c_seg->c_store.c_buffer, (int)io_size)) {
2732 panic("c_seg_swapin - Swap hash mismatch\n");
2733 }
39236c6e
A		 2734#endif /* CHECKSUM_THE_SWAP */
2735
3e170ce0 2736 PAGE_REPLACEMENT_DISALLOWED(TRUE);
39236c6e 2737
3e170ce0
A		 2738 if (force_minor_compaction == TRUE) {
2739 lck_mtx_lock_spin_always(&c_seg->c_lock);
39236c6e 2740
3e170ce0
A		 2741 c_seg_minor_compaction_and_unlock(c_seg, FALSE);
2742 }
2743 OSAddAtomic64(c_seg->c_bytes_used, &compressor_bytes_used);
2744
2745 c_seg_swapin_requeue(c_seg, TRUE);
2746 }
2747 C_SEG_WAKEUP_DONE(c_seg);
2748}
2749
2750
2751static void
2752c_segment_sv_hash_drop_ref(int hash_indx)
2753{
2754 struct c_sv_hash_entry o_sv_he, n_sv_he;
2755
2756 while (1) {
2757
2758 o_sv_he.he_record = c_segment_sv_hash_table[hash_indx].he_record;
2759
2760 n_sv_he.he_ref = o_sv_he.he_ref - 1;
2761 n_sv_he.he_data = o_sv_he.he_data;
2762
2763 if (OSCompareAndSwap64((UInt64)o_sv_he.he_record, (UInt64)n_sv_he.he_record, (UInt64 *) &c_segment_sv_hash_table[hash_indx].he_record) == TRUE) {
2764 if (n_sv_he.he_ref == 0)
2765 OSAddAtomic(-1, &c_segment_svp_in_hash);
2766 break;
2767 }
2768 }
2769}
2770
2771
2772static int
2773c_segment_sv_hash_insert(uint32_t data)
2774{
2775 int hash_sindx;
2776 int misses;
2777 struct c_sv_hash_entry o_sv_he, n_sv_he;
2778 boolean_t got_ref = FALSE;
2779
2780 if (data == 0)
2781 OSAddAtomic(1, &c_segment_svp_zero_compressions);
2782 else
2783 OSAddAtomic(1, &c_segment_svp_nonzero_compressions);
2784
2785 hash_sindx = data & C_SV_HASH_MASK;
2786
2787 for (misses = 0; misses < C_SV_HASH_MAX_MISS; misses++)
2788 {
2789 o_sv_he.he_record = c_segment_sv_hash_table[hash_sindx].he_record;
2790
2791 while (o_sv_he.he_data == data || o_sv_he.he_ref == 0) {
2792 n_sv_he.he_ref = o_sv_he.he_ref + 1;
2793 n_sv_he.he_data = data;
2794
2795 if (OSCompareAndSwap64((UInt64)o_sv_he.he_record, (UInt64)n_sv_he.he_record, (UInt64 *) &c_segment_sv_hash_table[hash_sindx].he_record) == TRUE) {
2796 if (n_sv_he.he_ref == 1)
2797 OSAddAtomic(1, &c_segment_svp_in_hash);
2798 got_ref = TRUE;
2799 break;
39236c6e 2800 }
3e170ce0 2801 o_sv_he.he_record = c_segment_sv_hash_table[hash_sindx].he_record;
39236c6e 2802 }
3e170ce0
A		 2803 if (got_ref == TRUE)
2804 break;
2805 hash_sindx++;
2806
2807 if (hash_sindx == C_SV_HASH_SIZE)
2808 hash_sindx = 0;
39236c6e 2809 }
3e170ce0
A		 2810 if (got_ref == FALSE)
2811 return(-1);
39236c6e 2812
3e170ce0
A		 2813 return (hash_sindx);
2814}
2815
2816
2817#if RECORD_THE_COMPRESSED_DATA
2818
2819static void
2820c_compressed_record_data(char *src, int c_size)
2821{
2822 if ((c_compressed_record_cptr + c_size + 4) >= c_compressed_record_ebuf)
2823 panic("c_compressed_record_cptr >= c_compressed_record_ebuf");
2824
2825 *(int *)((void *)c_compressed_record_cptr) = c_size;
2826
2827 c_compressed_record_cptr += 4;
2828
2829 memcpy(c_compressed_record_cptr, src, c_size);
2830 c_compressed_record_cptr += c_size;
39236c6e 2831}
3e170ce0 2832#endif
39236c6e
A		 2833
2834
2835static int
2836c_compress_page(char *src, c_slot_mapping_t slot_ptr, c_segment_t *current_chead, char *scratch_buf)
2837{
2838 int c_size;
04b8595b 2839 int c_rounded_size = 0;
39236c6e
A		 2840 int max_csize;
2841 c_slot_t cs;
2842 c_segment_t c_seg;
2843
2844 KERNEL_DEBUG(0xe0400000 | DBG_FUNC_START, *current_chead, 0, 0, 0, 0);
2845retry:
2846 if ((c_seg = c_seg_allocate(current_chead)) == NULL)
2847 return (1);
2848 /*
2849 * returns with c_seg lock held
3e170ce0
A		 2850 * and PAGE_REPLACEMENT_DISALLOWED(TRUE)...
2851 * c_nextslot has been allocated and
2852 * c_store.c_buffer populated
39236c6e 2853 */
3e170ce0
A		 2854 assert(c_seg->c_state == C_IS_FILLING);
2855
39236c6e
A		 2856 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_seg->c_nextslot);
2857
2858 cs->c_packed_ptr = C_SLOT_PACK_PTR(slot_ptr);
fe8ab488
A		 2859 assert(slot_ptr == (c_slot_mapping_t)C_SLOT_UNPACK_PTR(cs));
2860
39236c6e
A		 2861 cs->c_offset = c_seg->c_nextoffset;
2862
2863 max_csize = C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES((int32_t)cs->c_offset);
2864
2865 if (max_csize > PAGE_SIZE)
2866 max_csize = PAGE_SIZE;
2867
39236c6e
A		 2868#if CHECKSUM_THE_DATA
2869 cs->c_hash_data = hash_string(src, PAGE_SIZE);
2870#endif
39236c6e 2871
3e170ce0 2872 c_size = WKdm_compress_new((const WK_word *)(uintptr_t)src, (WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
fe8ab488 2873 (WK_word *)(uintptr_t)scratch_buf, max_csize - 4);
39236c6e
A		 2874 assert(c_size <= (max_csize - 4) && c_size >= -1);
2875
2876 if (c_size == -1) {
2877
2878 if (max_csize < PAGE_SIZE) {
2879 c_current_seg_filled(c_seg, current_chead);
3e170ce0 2880 assert(*current_chead == NULL);
39236c6e 2881
3e170ce0 2882 lck_mtx_unlock_always(&c_seg->c_lock);
39236c6e 2883
3e170ce0 2884 PAGE_REPLACEMENT_DISALLOWED(FALSE);
39236c6e
A		 2885 goto retry;
2886 }
2887 c_size = PAGE_SIZE;
2888
2889 memcpy(&c_seg->c_store.c_buffer[cs->c_offset], src, c_size);
3e170ce0
A		 2890
2891 OSAddAtomic(1, &c_segment_noncompressible_pages);
2892
2893 } else if (c_size == 0) {
2894 int hash_index;
2895
2896 /*
2897 * special case - this is a page completely full of a single 32 bit value
2898 */
2899 hash_index = c_segment_sv_hash_insert(*(uint32_t *)(uintptr_t)src);
2900
2901 if (hash_index != -1) {
2902 slot_ptr->s_cindx = hash_index;
2903 slot_ptr->s_cseg = C_SV_CSEG_ID;
2904
2905 OSAddAtomic(1, &c_segment_svp_hash_succeeded);
2906#if RECORD_THE_COMPRESSED_DATA
2907 c_compressed_record_data(src, 4);
2908#endif
2909 goto sv_compression;
2910 }
2911 c_size = 4;
2912
2913 memcpy(&c_seg->c_store.c_buffer[cs->c_offset], src, c_size);
2914
2915 OSAddAtomic(1, &c_segment_svp_hash_failed);
39236c6e 2916 }
3e170ce0
A		 2917
2918#if RECORD_THE_COMPRESSED_DATA
2919 c_compressed_record_data((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size);
2920#endif
2921
39236c6e
A		 2922#if CHECKSUM_THE_COMPRESSED_DATA
2923 cs->c_hash_compressed_data = hash_string((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size);
2924#endif
2925 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
2926
2927 PACK_C_SIZE(cs, c_size);
2928 c_seg->c_bytes_used += c_rounded_size;
2929 c_seg->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
2930
2931 slot_ptr->s_cindx = c_seg->c_nextslot++;
2932 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
2933 slot_ptr->s_cseg = c_seg->c_mysegno + 1;
2934
3e170ce0
A		 2935sv_compression:
2936 if (c_seg->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg->c_nextslot >= C_SLOT_MAX_INDEX) {
39236c6e 2937 c_current_seg_filled(c_seg, current_chead);
3e170ce0
A		 2938 assert(*current_chead == NULL);
2939 }
2940 lck_mtx_unlock_always(&c_seg->c_lock);
39236c6e
A		 2941
2942 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2943
3e170ce0
A		 2944#if RECORD_THE_COMPRESSED_DATA
2945 if ((c_compressed_record_cptr - c_compressed_record_sbuf) >= C_SEG_ALLOCSIZE) {
2946 c_compressed_record_write(c_compressed_record_sbuf, (int)(c_compressed_record_cptr - c_compressed_record_sbuf));
2947 c_compressed_record_cptr = c_compressed_record_sbuf;
2948 }
2949#endif
2950 if (c_size) {
2951 OSAddAtomic64(c_size, &c_segment_compressed_bytes);
2952 OSAddAtomic64(c_rounded_size, &compressor_bytes_used);
2953 }
39236c6e 2954 OSAddAtomic64(PAGE_SIZE, &c_segment_input_bytes);
39236c6e
A		 2955
2956 OSAddAtomic(1, &c_segment_pages_compressed);
2957 OSAddAtomic(1, &sample_period_compression_count);
2958
2959 KERNEL_DEBUG(0xe0400000 | DBG_FUNC_END, *current_chead, c_size, c_segment_input_bytes, c_segment_compressed_bytes, 0);
2960
39236c6e
A		 2961 return (0);
2962}
2963
2964
2965static int
2966c_decompress_page(char *dst, volatile c_slot_mapping_t slot_ptr, int flags, int *zeroslot)
2967{
2968 c_slot_t cs;
2969 c_segment_t c_seg;
2970 int c_indx;
2971 int c_rounded_size;
2972 uint32_t c_size;
2973 int retval = 0;
39236c6e
A		 2974 boolean_t need_unlock = TRUE;
2975 boolean_t consider_defragmenting = FALSE;
3e170ce0
A		 2976 boolean_t kdp_mode = FALSE;
2977
2978 if (flags & C_KDP) {
2979 if (not_in_kdp) {
2980 panic("C_KDP passed to decompress page from outside of debugger context");
2981 }
2982
2983 assert((flags & C_KEEP) == C_KEEP);
2984 assert((flags & C_DONT_BLOCK) == C_DONT_BLOCK);
2985
2986 if ((flags & (C_DONT_BLOCK | C_KEEP)) != (C_DONT_BLOCK | C_KEEP)) {
2987 return (-2);
2988 }
2989
2990 kdp_mode = TRUE;
2991 }
39236c6e
A		 2992
2993ReTry:
3e170ce0
A		 2994 if (!kdp_mode) {
2995 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2996 } else {
2997 if (kdp_lck_rw_lock_is_acquired_exclusive(&c_master_lock)) {
2998 return (-2);
2999 }
3000 }
fe8ab488 3001
39236c6e 3002#if HIBERNATION
fe8ab488
A		 3003 /*
3004 * if hibernation is enabled, it indicates (via a call
3005 * to 'vm_decompressor_lock' that no further
3006 * decompressions are allowed once it reaches
3007 * the point of flushing all of the currently dirty
3008 * anonymous memory through the compressor and out
3009 * to disk... in this state we allow freeing of compressed
3010 * pages and must honor the C_DONT_BLOCK case
3011 */
3012 if (dst && decompressions_blocked == TRUE) {
3013 if (flags & C_DONT_BLOCK) {
3014
3e170ce0
A		 3015 if (!kdp_mode) {
3016 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3017 }
fe8ab488
A		 3018
3019 *zeroslot = 0;
3020 return (-2);
39236c6e 3021 }
fe8ab488
A		 3022 /*
3023 * it's safe to atomically assert and block behind the
3024 * lock held in shared mode because "decompressions_blocked" is
3025 * only set and cleared and the thread_wakeup done when the lock
3026 * is held exclusively
3027 */
3028 assert_wait((event_t)&decompressions_blocked, THREAD_UNINT);
3029
3030 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3031
3032 thread_block(THREAD_CONTINUE_NULL);
3033
3034 goto ReTry;
39236c6e
A		 3035 }
3036#endif
39236c6e
A		 3037 /* s_cseg is actually "segno+1" */
3038 c_seg = c_segments[slot_ptr->s_cseg - 1].c_seg;
3039
3e170ce0
A		 3040 if (!kdp_mode) {
3041 lck_mtx_lock_spin_always(&c_seg->c_lock);
3042 } else {
3043 if (kdp_lck_mtx_lock_spin_is_acquired(&c_seg->c_lock)) {
3044 return (-2);
3045 }
3046 }
39236c6e 3047
3e170ce0 3048 assert(c_seg->c_state != C_IS_EMPTY && c_seg->c_state != C_IS_FREE);
39236c6e 3049
3e170ce0
A		 3050 if (flags & C_DONT_BLOCK) {
3051 if (c_seg->c_busy || (C_SEG_IS_ONDISK(c_seg) && dst)) {
39236c6e
A		 3052 *zeroslot = 0;
3053
3e170ce0 3054 retval = -2;
39236c6e
A		 3055 goto done;
3056 }
3057 }
3058 if (c_seg->c_busy) {
3059
3060 PAGE_REPLACEMENT_DISALLOWED(FALSE);
fe8ab488 3061
39236c6e
A		 3062 c_seg_wait_on_busy(c_seg);
3063
3064 goto ReTry;
3065 }
3066 c_indx = slot_ptr->s_cindx;
3067
3068 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
3069
3070 c_size = UNPACK_C_SIZE(cs);
3071
3072 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
3073
3074 if (dst) {
3075 uint32_t age_of_cseg;
3076 clock_sec_t cur_ts_sec;
3077 clock_nsec_t cur_ts_nsec;
3078
3e170ce0
A		 3079 if (C_SEG_IS_ONDISK(c_seg)) {
3080 assert(kdp_mode == FALSE);
39236c6e 3081 c_seg_swapin(c_seg, FALSE);
3e170ce0
A		 3082
3083 retval = 1;
39236c6e 3084 }
3e170ce0
A		 3085 if (c_seg->c_state == C_ON_BAD_Q) {
3086 assert(c_seg->c_store.c_buffer == NULL);
3087
3088 retval = -1;
39236c6e
A		 3089 goto c_seg_invalid_data;
3090 }
3091#if CHECKSUM_THE_COMPRESSED_DATA
3092 if (cs->c_hash_compressed_data != hash_string((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size))
3093 panic("compressed data doesn't match original");
3094#endif
3095 if (c_rounded_size == PAGE_SIZE) {
3096 /*
3097 * page wasn't compressible... just copy it out
3098 */
3099 memcpy(dst, &c_seg->c_store.c_buffer[cs->c_offset], PAGE_SIZE);
3e170ce0
A		 3100 } else if (c_size == 4) {
3101 int32_t data;
3102 int32_t *dptr;
3103
3104 /*
3105 * page was populated with a single value
3106 * that didn't fit into our fast hash
3107 * so we packed it in as a single non-compressed value
3108 * that we need to populate the page with
3109 */
3110 dptr = (int32_t *)(uintptr_t)dst;
3111 data = *(int32_t *)(&c_seg->c_store.c_buffer[cs->c_offset]);
3112#if __x86_64__
3113 memset_word(dptr, data, PAGE_SIZE / sizeof(int32_t));
3114#else
3115 {
3116 int i;
3117
3118 for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++)
3119 *dptr++ = data;
3120 }
3121#endif
39236c6e
A		 3122 } else {
3123 uint32_t my_cpu_no;
3124 char *scratch_buf;
3125
3e170ce0
A		 3126 if (!kdp_mode) {
3127 /*
3128 * we're behind the c_seg lock held in spin mode
3129 * which means pre-emption is disabled... therefore
3130 * the following sequence is atomic and safe
3131 */
3132 my_cpu_no = cpu_number();
39236c6e 3133
3e170ce0 3134 assert(my_cpu_no < compressor_cpus);
39236c6e 3135
3e170ce0
A		 3136 scratch_buf = &compressor_scratch_bufs[my_cpu_no * WKdm_SCRATCH_BUF_SIZE];
3137 } else {
3138 scratch_buf = kdp_compressor_scratch_buf;
3139 }
39236c6e
A		 3140 WKdm_decompress_new((WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
3141 (WK_word *)(uintptr_t)dst, (WK_word *)(uintptr_t)scratch_buf, c_size);
3142 }
3143
3144#if CHECKSUM_THE_DATA
3145 if (cs->c_hash_data != hash_string(dst, PAGE_SIZE))
3146 panic("decompressed data doesn't match original");
3147#endif
3e170ce0 3148 if (c_seg->c_swappedin_ts == 0 && !kdp_mode) {
39236c6e
A		 3149
3150 clock_get_system_nanotime(&cur_ts_sec, &cur_ts_nsec);
3151
3152 age_of_cseg = (uint32_t)cur_ts_sec - c_seg->c_creation_ts;
3153
3154 if (age_of_cseg < DECOMPRESSION_SAMPLE_MAX_AGE)
3155 OSAddAtomic(1, &age_of_decompressions_during_sample_period[age_of_cseg]);
3156 else
3157 OSAddAtomic(1, &overage_decompressions_during_sample_period);
3158
3159 OSAddAtomic(1, &sample_period_decompression_count);
3160 }
39236c6e
A		 3161 }
3162c_seg_invalid_data:
3163
39236c6e
A		 3164 if (flags & C_KEEP) {
3165 *zeroslot = 0;
3166 goto done;
3167 }
3e170ce0
A		 3168
3169 assert(kdp_mode == FALSE);
39236c6e
A		 3170 c_seg->c_bytes_unused += c_rounded_size;
3171 c_seg->c_bytes_used -= c_rounded_size;
3172 PACK_C_SIZE(cs, 0);
3173
3174 if (c_indx < c_seg->c_firstemptyslot)
3175 c_seg->c_firstemptyslot = c_indx;
3176
3177 OSAddAtomic(-1, &c_segment_pages_compressed);
3178
3e170ce0 3179 if (c_seg->c_state != C_ON_BAD_Q && !(C_SEG_IS_ONDISK(c_seg))) {
39236c6e 3180 /*
3e170ce0 3181 * C_SEG_IS_ONDISK == TRUE can occur when we're doing a
39236c6e
A		 3182 * free of a compressed page (i.e. dst == NULL)
3183 */
3184 OSAddAtomic64(-c_rounded_size, &compressor_bytes_used);
3185 }
3e170ce0 3186 if (c_seg->c_state != C_IS_FILLING) {
39236c6e 3187 if (c_seg->c_bytes_used == 0) {
3e170ce0 3188 if ( !(C_SEG_IS_ONDISK(c_seg))) {
8a3053a0
A		 3189 int pages_populated;
3190
3191 pages_populated = (round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE;
3192 c_seg->c_populated_offset = C_SEG_BYTES_TO_OFFSET(0);
3193
3194 if (pages_populated) {
3e170ce0
A		 3195
3196 assert(c_seg->c_state != C_ON_BAD_Q);
8a3053a0
A		 3197 assert(c_seg->c_store.c_buffer != NULL);
3198
fe8ab488 3199 C_SEG_BUSY(c_seg);
8a3053a0
A		 3200 lck_mtx_unlock_always(&c_seg->c_lock);
3201
3202 kernel_memory_depopulate(kernel_map, (vm_offset_t) c_seg->c_store.c_buffer, pages_populated * PAGE_SIZE, KMA_COMPRESSOR);
3203
3204 lck_mtx_lock_spin_always(&c_seg->c_lock);
3205 C_SEG_WAKEUP_DONE(c_seg);
3206 }
3e170ce0 3207 if (!c_seg->c_on_minorcompact_q)
8a3053a0
A		 3208 c_seg_need_delayed_compaction(c_seg);
3209 } else
3e170ce0 3210 assert(c_seg->c_state == C_ON_SWAPPEDOUTSPARSE_Q);
8a3053a0 3211
39236c6e
A		 3212 } else if (c_seg->c_on_minorcompact_q) {
3213
3e170ce0
A		 3214 assert(c_seg->c_state != C_ON_BAD_Q);
3215
3216 if (C_SEG_SHOULD_MINORCOMPACT(c_seg)) {
39236c6e
A		 3217 c_seg_try_minor_compaction_and_unlock(c_seg);
3218 need_unlock = FALSE;
3219 }
3e170ce0 3220 } else if ( !(C_SEG_IS_ONDISK(c_seg))) {
39236c6e 3221
3e170ce0 3222 if (c_seg->c_state != C_ON_BAD_Q && c_seg->c_state != C_ON_SWAPOUT_Q && C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE) {
39236c6e
A		 3223 c_seg_need_delayed_compaction(c_seg);
3224 }
3e170ce0 3225 } else if (c_seg->c_state != C_ON_SWAPPEDOUTSPARSE_Q && C_SEG_ONDISK_IS_SPARSE(c_seg)) {
39236c6e
A		 3226
3227 c_seg_move_to_sparse_list(c_seg);
3228 consider_defragmenting = TRUE;
3229 }
3230 }
3231done:
3e170ce0
A		 3232 if (kdp_mode) {
3233 return retval;
3234 }
3235
39236c6e
A		 3236 if (need_unlock == TRUE)
3237 lck_mtx_unlock_always(&c_seg->c_lock);
3238
3239 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3240
3241 if (consider_defragmenting == TRUE)
3242 vm_swap_consider_defragmenting();
fe8ab488
A		 3243
3244
39236c6e
A		 3245 return (retval);
3246}
3247
3248
3249int
3250vm_compressor_get(ppnum_t pn, int *slot, int flags)
3251{
3e170ce0 3252 c_slot_mapping_t slot_ptr;
39236c6e
A		 3253 char *dst;
3254 int zeroslot = 1;
3255 int retval;
3256
3257#if __x86_64__
3258 dst = PHYSMAP_PTOV((uint64_t)pn << (uint64_t)PAGE_SHIFT);
3259#else
3260#error "unsupported architecture"
3261#endif
3e170ce0
A		 3262 slot_ptr = (c_slot_mapping_t)slot;
3263
3264 if (slot_ptr->s_cseg == C_SV_CSEG_ID) {
3265 int32_t data;
3266 int32_t *dptr;
3267
3268 /*
3269 * page was populated with a single value
3270 * that found a home in our hash table
3271 * grab that value from the hash and populate the page
3272 * that we need to populate the page with
3273 */
3274 dptr = (int32_t *)(uintptr_t)dst;
3275 data = c_segment_sv_hash_table[slot_ptr->s_cindx].he_data;
3276#if __x86_64__
3277 memset_word(dptr, data, PAGE_SIZE / sizeof(int32_t));
3278#else
3279 {
3280 int i;
3281
3282 for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++)
3283 *dptr++ = data;
3284 }
3285#endif
3286 c_segment_sv_hash_drop_ref(slot_ptr->s_cindx);
39236c6e 3287
3e170ce0
A		 3288 if ( !(flags & C_KEEP)) {
3289 OSAddAtomic(-1, &c_segment_pages_compressed);
3290 *slot = 0;
3291 }
3292 if (data)
3293 OSAddAtomic(1, &c_segment_svp_nonzero_decompressions);
3294 else
3295 OSAddAtomic(1, &c_segment_svp_zero_decompressions);
3296
3297 return (0);
3298 }
3299
3300 retval = c_decompress_page(dst, slot_ptr, flags, &zeroslot);
39236c6e
A		 3301
3302 /*
3303 * zeroslot will be set to 0 by c_decompress_page if (flags & C_KEEP)
3e170ce0 3304 * or (flags & C_DONT_BLOCK) and we found 'c_busy' or 'C_SEG_IS_ONDISK' to be TRUE
39236c6e
A		 3305 */
3306 if (zeroslot) {
39236c6e
A		 3307 *slot = 0;
3308 }
3309 /*
3310 * returns 0 if we successfully decompressed a page from a segment already in memory
3311 * returns 1 if we had to first swap in the segment, before successfully decompressing the page
3312 * returns -1 if we encountered an error swapping in the segment - decompression failed
3e170ce0 3313 * returns -2 if (flags & C_DONT_BLOCK) and we found 'c_busy' or 'C_SEG_IS_ONDISK' to be true
39236c6e
A		 3314 */
3315 return (retval);
3316}
3317
3318
fe8ab488
A		 3319int
3320vm_compressor_free(int *slot, int flags)
39236c6e 3321{
3e170ce0 3322 c_slot_mapping_t slot_ptr;
39236c6e 3323 int zeroslot = 1;
fe8ab488 3324 int retval;
39236c6e 3325
fe8ab488 3326 assert(flags == 0 || flags == C_DONT_BLOCK);
39236c6e 3327
3e170ce0
A		 3328 slot_ptr = (c_slot_mapping_t)slot;
3329
3330 if (slot_ptr->s_cseg == C_SV_CSEG_ID) {
3331
3332 c_segment_sv_hash_drop_ref(slot_ptr->s_cindx);
3333 OSAddAtomic(-1, &c_segment_pages_compressed);
3334
3335 *slot = 0;
3336 return (0);
3337 }
3338 retval = c_decompress_page(NULL, slot_ptr, flags, &zeroslot);
fe8ab488
A		 3339 /*
3340 * returns 0 if we successfully freed the specified compressed page
3341 * returns -2 if (flags & C_DONT_BLOCK) and we found 'c_busy' set
3342 */
3343
3344 if (retval == 0)
3345 *slot = 0;
3e170ce0
A		 3346 else
3347 assert(retval == -2);
fe8ab488
A		 3348
3349 return (retval);
39236c6e
A		 3350}
3351
3352
3353int
3354vm_compressor_put(ppnum_t pn, int *slot, void **current_chead, char *scratch_buf)
3355{
3356 char *src;
3357 int retval;
3358
39236c6e
A		 3359#if __x86_64__
3360 src = PHYSMAP_PTOV((uint64_t)pn << (uint64_t)PAGE_SHIFT);
3361#else
3362#error "unsupported architecture"
3363#endif
3364 retval = c_compress_page(src, (c_slot_mapping_t)slot, (c_segment_t *)current_chead, scratch_buf);
3365
3366 return (retval);
3367}
fe8ab488
A		 3368
3369void
3370vm_compressor_transfer(
3371 int *dst_slot_p,
3372 int *src_slot_p)
3373{
3374 c_slot_mapping_t dst_slot, src_slot;
3375 c_segment_t c_seg;
3376 int c_indx;
3377 c_slot_t cs;
3378
fe8ab488
A		 3379 src_slot = (c_slot_mapping_t) src_slot_p;
3380
3e170ce0
A		 3381 if (src_slot->s_cseg == C_SV_CSEG_ID) {
3382 *dst_slot_p = *src_slot_p;
3383 *src_slot_p = 0;
3384 return;
3385 }
3386 dst_slot = (c_slot_mapping_t) dst_slot_p;
fe8ab488
A		 3387Retry:
3388 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3389 /* get segment for src_slot */
3390 c_seg = c_segments[src_slot->s_cseg -1].c_seg;
3391 /* lock segment */
3392 lck_mtx_lock_spin_always(&c_seg->c_lock);
3393 /* wait if it's busy */
3e170ce0 3394 if (c_seg->c_busy && !c_seg->c_busy_swapping) {
fe8ab488
A		 3395 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3396 c_seg_wait_on_busy(c_seg);
3397 goto Retry;
3398 }
3399 /* find the c_slot */
3400 c_indx = src_slot->s_cindx;
3401 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
3402 /* point the c_slot back to dst_slot instead of src_slot */
3403 cs->c_packed_ptr = C_SLOT_PACK_PTR(dst_slot);
3404 /* transfer */
3405 *dst_slot_p = *src_slot_p;
3406 *src_slot_p = 0;
3407 lck_mtx_unlock_always(&c_seg->c_lock);
3408 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3409}
3e170ce0
A		 3410
3411#if CONFIG_FREEZE
3412
3413int freezer_finished_filling = 0;
3414
3415void
3416vm_compressor_finished_filling(
3417 void **current_chead)
3418{
3419 c_segment_t c_seg;
3420
3421 if ((c_seg = *(c_segment_t *)current_chead) == NULL)
3422 return;
3423
3424 assert(c_seg->c_state == C_IS_FILLING);
3425
3426 lck_mtx_lock_spin_always(&c_seg->c_lock);
3427
3428 c_current_seg_filled(c_seg, (c_segment_t *)current_chead);
3429
3430 lck_mtx_unlock_always(&c_seg->c_lock);
3431
3432 freezer_finished_filling++;
3433}
3434
3435
3436/*
3437 * This routine is used to transfer the compressed chunks from
3438 * the c_seg/cindx pointed to by slot_p into a new c_seg headed
3439 * by the current_chead and a new cindx within that c_seg.
3440 *
3441 * Currently, this routine is only used by the "freezer backed by
3442 * compressor with swap" mode to create a series of c_segs that
3443 * only contain compressed data belonging to one task. So, we
3444 * move a task's previously compressed data into a set of new
3445 * c_segs which will also hold the task's yet to be compressed data.
3446 */
3447
3448kern_return_t
3449vm_compressor_relocate(
3450 void **current_chead,
3451 int *slot_p)
3452{
3453 c_slot_mapping_t slot_ptr;
3454 c_slot_mapping_t src_slot;
3455 uint32_t c_rounded_size;
3456 uint32_t c_size;
3457 uint16_t dst_slot;
3458 c_slot_t c_dst;
3459 c_slot_t c_src;
3460 int c_indx;
3461 c_segment_t c_seg_dst = NULL;
3462 c_segment_t c_seg_src = NULL;
3463 kern_return_t kr = KERN_SUCCESS;
3464
3465
3466 src_slot = (c_slot_mapping_t) slot_p;
3467
3468 if (src_slot->s_cseg == C_SV_CSEG_ID) {
3469 /*
3470 * no need to relocate... this is a page full of a single
3471 * value which is hashed to a single entry not contained
3472 * in a c_segment_t
3473 */
3474 return (kr);
3475 }
3476
3477Relookup_dst:
3478 c_seg_dst = c_seg_allocate((c_segment_t *)current_chead);
3479 /*
3480 * returns with c_seg lock held
3481 * and PAGE_REPLACEMENT_DISALLOWED(TRUE)...
3482 * c_nextslot has been allocated and
3483 * c_store.c_buffer populated
3484 */
3485 if (c_seg_dst == NULL) {
3486 /*
3487 * Out of compression segments?
3488 */
3489 kr = KERN_RESOURCE_SHORTAGE;
3490 goto out;
3491 }
3492
3493 assert(c_seg_dst->c_busy == 0);
3494
3495 C_SEG_BUSY(c_seg_dst);
3496
3497 dst_slot = c_seg_dst->c_nextslot;
3498
3499 lck_mtx_unlock_always(&c_seg_dst->c_lock);
3500
3501Relookup_src:
3502 c_seg_src = c_segments[src_slot->s_cseg - 1].c_seg;
3503
3504 assert(c_seg_dst != c_seg_src);
3505
3506 lck_mtx_lock_spin_always(&c_seg_src->c_lock);
3507
3508 if (C_SEG_IS_ONDISK(c_seg_src)) {
3509
3510 /*
3511 * A "thaw" can mark a process as eligible for
3512 * another freeze cycle without bringing any of
3513 * its swapped out c_segs back from disk (because
3514 * that is done on-demand).
3515 *
3516 * If the src c_seg we find for our pre-compressed
3517 * data is already on-disk, then we are dealing
3518 * with an app's data that is already packed and
3519 * swapped out. Don't do anything.
3520 */
3521
3522 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3523
3524 lck_mtx_unlock_always(&c_seg_src->c_lock);
3525
3526 c_seg_src = NULL;
3527
3528 goto out;
3529 }
3530
3531 if (c_seg_src->c_busy) {
3532
3533 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3534 c_seg_wait_on_busy(c_seg_src);
3535
3536 c_seg_src = NULL;
3537
3538 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3539
3540 goto Relookup_src;
3541 }
3542
3543 C_SEG_BUSY(c_seg_src);
3544
3545 lck_mtx_unlock_always(&c_seg_src->c_lock);
3546
3547 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3548
3549 /* find the c_slot */
3550 c_indx = src_slot->s_cindx;
3551
3552 c_src = C_SEG_SLOT_FROM_INDEX(c_seg_src, c_indx);
3553
3554 c_size = UNPACK_C_SIZE(c_src);
3555
3556 assert(c_size);
3557
3558 if (c_size > (uint32_t)(C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES((int32_t)c_seg_dst->c_nextoffset))) {
3559 /*
3560 * This segment is full. We need a new one.
3561 */
3562
3563 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3564
3565 lck_mtx_lock_spin_always(&c_seg_src->c_lock);
3566 C_SEG_WAKEUP_DONE(c_seg_src);
3567 lck_mtx_unlock_always(&c_seg_src->c_lock);
3568
3569 c_seg_src = NULL;
3570
3571 lck_mtx_lock_spin_always(&c_seg_dst->c_lock);
3572
3573 assert(c_seg_dst->c_busy);
3574 assert(c_seg_dst->c_state == C_IS_FILLING);
3575 assert(!c_seg_dst->c_on_minorcompact_q);
3576
3577 c_current_seg_filled(c_seg_dst, (c_segment_t *)current_chead);
3578 assert(*current_chead == NULL);
3579
3580 C_SEG_WAKEUP_DONE(c_seg_dst);
3581
3582 lck_mtx_unlock_always(&c_seg_dst->c_lock);
3583
3584 c_seg_dst = NULL;
3585
3586 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3587
3588 goto Relookup_dst;
3589 }
3590
3591 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, c_seg_dst->c_nextslot);
3592
3593 memcpy(&c_seg_dst->c_store.c_buffer[c_seg_dst->c_nextoffset], &c_seg_src->c_store.c_buffer[c_src->c_offset], c_size);
3594
3595 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
3596
3597#if CHECKSUM_THE_DATA
3598 c_dst->c_hash_data = c_src->c_hash_data;
3599#endif
3600#if CHECKSUM_THE_COMPRESSED_DATA
3601 c_dst->c_hash_compressed_data = c_src->c_hash_compressed_data;
3602#endif
3603
3604 c_dst->c_size = c_src->c_size;
3605 c_dst->c_packed_ptr = c_src->c_packed_ptr;
3606 c_dst->c_offset = c_seg_dst->c_nextoffset;
3607
3608 if (c_seg_dst->c_firstemptyslot == c_seg_dst->c_nextslot)
3609 c_seg_dst->c_firstemptyslot++;
3610
3611 c_seg_dst->c_nextslot++;
3612 c_seg_dst->c_bytes_used += c_rounded_size;
3613 c_seg_dst->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
3614
3615
3616 PACK_C_SIZE(c_src, 0);
3617
3618 c_seg_src->c_bytes_used -= c_rounded_size;
3619 c_seg_src->c_bytes_unused += c_rounded_size;
3620
3621 if (c_indx < c_seg_src->c_firstemptyslot) {
3622 c_seg_src->c_firstemptyslot = c_indx;
3623 }
3624
3625 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, dst_slot);
3626
3627 PAGE_REPLACEMENT_ALLOWED(TRUE);
3628 slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
3629 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
3630 slot_ptr->s_cseg = c_seg_dst->c_mysegno + 1;
3631 slot_ptr->s_cindx = dst_slot;
3632
3633 PAGE_REPLACEMENT_ALLOWED(FALSE);
3634
3635out:
3636 if (c_seg_src) {
3637
3638 lck_mtx_lock_spin_always(&c_seg_src->c_lock);
3639
3640 C_SEG_WAKEUP_DONE(c_seg_src);
3641
3642 if (c_seg_src->c_bytes_used == 0 && c_seg_src->c_state != C_IS_FILLING) {
3643 if (!c_seg_src->c_on_minorcompact_q)
3644 c_seg_need_delayed_compaction(c_seg_src);
3645 }
3646
3647 lck_mtx_unlock_always(&c_seg_src->c_lock);
3648 }
3649
3650 if (c_seg_dst) {
3651
3652 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3653
3654 lck_mtx_lock_spin_always(&c_seg_dst->c_lock);
3655
3656 if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX_INDEX) {
3657 /*
3658 * Nearing or exceeded maximum slot and offset capacity.
3659 */
3660 assert(c_seg_dst->c_busy);
3661 assert(c_seg_dst->c_state == C_IS_FILLING);
3662 assert(!c_seg_dst->c_on_minorcompact_q);
3663
3664 c_current_seg_filled(c_seg_dst, (c_segment_t *)current_chead);
3665 assert(*current_chead == NULL);
3666 }
3667
3668 C_SEG_WAKEUP_DONE(c_seg_dst);
3669
3670 lck_mtx_unlock_always(&c_seg_dst->c_lock);
3671
3672 c_seg_dst = NULL;
3673
3674 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3675 }
3676
3677 return kr;
3678}
3679#endif /* CONFIG_FREEZE */
mirror of XNU