]> git.saurik.com Git - apple/xnu.git/blob - osfmk/vm/vm_compressor.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-3789.1.32.tar.gz
[apple/xnu.git] / osfmk / vm / vm_compressor.c
1 /*
2 * Copyright (c) 2000-2013 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <vm/vm_compressor.h>
30
31 #if CONFIG_PHANTOM_CACHE
32 #include <vm/vm_phantom_cache.h>
33 #endif
34
35 #include <vm/vm_map.h>
36 #include <vm/vm_pageout.h>
37 #include <vm/memory_object.h>
38 #include <vm/vm_compressor_algorithms.h>
39 #include <vm/vm_fault.h>
40 #include <mach/mach_host.h> /* for host_info() */
41 #include <kern/ledger.h>
42 #include <kern/policy_internal.h>
43
44 #include <i386/misc_protos.h>
45
46 #include <IOKit/IOHibernatePrivate.h>
47
48 /*
49 * vm_compressor_mode has a heirarchy of control to set its value.
50 * boot-args are checked first, then device-tree, and finally
51 * the default value that is defined below. See vm_fault_init() for
52 * the boot-arg & device-tree code.
53 */
54
55 int vm_compressor_mode = VM_PAGER_COMPRESSOR_WITH_SWAP;
56 int vm_scale = 16;
57
58
59 int vm_compressor_is_active = 0;
60 int vm_compression_limit = 0;
61 int vm_compressor_available = 0;
62
63 extern void vm_pageout_io_throttle(void);
64
65 #if CHECKSUM_THE_DATA || CHECKSUM_THE_SWAP || CHECKSUM_THE_COMPRESSED_DATA
66 extern unsigned int hash_string(char *cp, int len);
67 #endif
68
69 #define UNPACK_C_SIZE(cs) ((cs->c_size == (PAGE_SIZE-1)) ? PAGE_SIZE : cs->c_size)
70 #define PACK_C_SIZE(cs, size) (cs->c_size = ((size == PAGE_SIZE) ? PAGE_SIZE - 1 : size))
71
72
73 struct c_sv_hash_entry {
74 union {
75 struct {
76 uint32_t c_sv_he_ref;
77 uint32_t c_sv_he_data;
78 } c_sv_he;
79 uint64_t c_sv_he_record;
80
81 } c_sv_he_un;
82 };
83
84 #define he_ref c_sv_he_un.c_sv_he.c_sv_he_ref
85 #define he_data c_sv_he_un.c_sv_he.c_sv_he_data
86 #define he_record c_sv_he_un.c_sv_he_record
87
88 #define C_SV_HASH_MAX_MISS 32
89 #define C_SV_HASH_SIZE ((1 << 10))
90 #define C_SV_HASH_MASK ((1 << 10) - 1)
91 #define C_SV_CSEG_ID ((1 << 22) - 1)
92
93
94 struct c_slot_mapping {
95 uint32_t s_cseg:22, /* segment number + 1 */
96 s_cindx:10; /* index in the segment */
97 };
98 #define C_SLOT_MAX_INDEX (1 << 10)
99
100 typedef struct c_slot_mapping *c_slot_mapping_t;
101
102
103 union c_segu {
104 c_segment_t c_seg;
105 uintptr_t c_segno;
106 };
107
108
109
110 #define C_SLOT_PACK_PTR(ptr) (((uintptr_t)ptr - (uintptr_t) KERNEL_PMAP_HEAP_RANGE_START) >> 2)
111 #define C_SLOT_UNPACK_PTR(cslot) ((uintptr_t)(cslot->c_packed_ptr << 2) + (uintptr_t) KERNEL_PMAP_HEAP_RANGE_START)
112
113
114 uint32_t c_segment_count = 0;
115 uint32_t c_segment_count_max = 0;
116
117 uint64_t c_generation_id = 0;
118 uint64_t c_generation_id_flush_barrier;
119
120
121 #define HIBERNATE_FLUSHING_SECS_TO_COMPLETE 120
122
123 boolean_t hibernate_no_swapspace = FALSE;
124 clock_sec_t hibernate_flushing_deadline = 0;
125
126
127 #if RECORD_THE_COMPRESSED_DATA
128 char *c_compressed_record_sbuf;
129 char *c_compressed_record_ebuf;
130 char *c_compressed_record_cptr;
131 #endif
132
133
134 queue_head_t c_age_list_head;
135 queue_head_t c_swapout_list_head;
136 queue_head_t c_swappedin_list_head;
137 queue_head_t c_swappedout_list_head;
138 queue_head_t c_swappedout_sparse_list_head;
139 queue_head_t c_major_list_head;
140 queue_head_t c_filling_list_head;
141 queue_head_t c_bad_list_head;
142
143 uint32_t c_age_count = 0;
144 uint32_t c_swapout_count = 0;
145 uint32_t c_swappedin_count = 0;
146 uint32_t c_swappedout_count = 0;
147 uint32_t c_swappedout_sparse_count = 0;
148 uint32_t c_major_count = 0;
149 uint32_t c_filling_count = 0;
150 uint32_t c_empty_count = 0;
151 uint32_t c_bad_count = 0;
152
153
154 queue_head_t c_minor_list_head;
155 uint32_t c_minor_count = 0;
156
157 int c_overage_swapped_count = 0;
158 int c_overage_swapped_limit = 0;
159
160 int c_seg_fixed_array_len;
161 union c_segu *c_segments;
162 vm_offset_t c_buffers;
163 vm_size_t c_buffers_size;
164 caddr_t c_segments_next_page;
165 boolean_t c_segments_busy;
166 uint32_t c_segments_available;
167 uint32_t c_segments_limit;
168 uint32_t c_segments_nearing_limit;
169
170 uint32_t c_segment_svp_in_hash;
171 uint32_t c_segment_svp_hash_succeeded;
172 uint32_t c_segment_svp_hash_failed;
173 uint32_t c_segment_svp_zero_compressions;
174 uint32_t c_segment_svp_nonzero_compressions;
175 uint32_t c_segment_svp_zero_decompressions;
176 uint32_t c_segment_svp_nonzero_decompressions;
177
178 uint32_t c_segment_noncompressible_pages;
179
180 uint32_t c_segment_pages_compressed;
181 uint32_t c_segment_pages_compressed_limit;
182 uint32_t c_segment_pages_compressed_nearing_limit;
183 uint32_t c_free_segno_head = (uint32_t)-1;
184
185 uint32_t vm_compressor_minorcompact_threshold_divisor = 10;
186 uint32_t vm_compressor_majorcompact_threshold_divisor = 10;
187 uint32_t vm_compressor_unthrottle_threshold_divisor = 10;
188 uint32_t vm_compressor_catchup_threshold_divisor = 10;
189
190 #define C_SEGMENTS_PER_PAGE (PAGE_SIZE / sizeof(union c_segu))
191
192
193 lck_grp_attr_t vm_compressor_lck_grp_attr;
194 lck_attr_t vm_compressor_lck_attr;
195 lck_grp_t vm_compressor_lck_grp;
196 lck_mtx_t *c_list_lock;
197 lck_rw_t c_master_lock;
198 boolean_t decompressions_blocked = FALSE;
199
200 zone_t compressor_segment_zone;
201 int c_compressor_swap_trigger = 0;
202
203 uint32_t compressor_cpus;
204 char *compressor_scratch_bufs;
205 char *kdp_compressor_scratch_buf;
206 char *kdp_compressor_decompressed_page;
207 addr64_t kdp_compressor_decompressed_page_paddr;
208 ppnum_t kdp_compressor_decompressed_page_ppnum;
209
210 clock_sec_t start_of_sample_period_sec = 0;
211 clock_nsec_t start_of_sample_period_nsec = 0;
212 clock_sec_t start_of_eval_period_sec = 0;
213 clock_nsec_t start_of_eval_period_nsec = 0;
214 uint32_t sample_period_decompression_count = 0;
215 uint32_t sample_period_compression_count = 0;
216 uint32_t last_eval_decompression_count = 0;
217 uint32_t last_eval_compression_count = 0;
218
219 #define DECOMPRESSION_SAMPLE_MAX_AGE (60 * 30)
220
221 boolean_t vm_swapout_ripe_segments = FALSE;
222 uint32_t vm_ripe_target_age = (60 * 60 * 48);
223
224 uint32_t swapout_target_age = 0;
225 uint32_t age_of_decompressions_during_sample_period[DECOMPRESSION_SAMPLE_MAX_AGE];
226 uint32_t overage_decompressions_during_sample_period = 0;
227
228 void do_fastwake_warmup(void);
229 boolean_t fastwake_warmup = FALSE;
230 boolean_t fastwake_recording_in_progress = FALSE;
231 clock_sec_t dont_trim_until_ts = 0;
232
233 uint64_t c_segment_warmup_count;
234 uint64_t first_c_segment_to_warm_generation_id = 0;
235 uint64_t last_c_segment_to_warm_generation_id = 0;
236 boolean_t hibernate_flushing = FALSE;
237
238 int64_t c_segment_input_bytes __attribute__((aligned(8))) = 0;
239 int64_t c_segment_compressed_bytes __attribute__((aligned(8))) = 0;
240 int64_t compressor_bytes_used __attribute__((aligned(8))) = 0;
241
242
243 struct c_sv_hash_entry c_segment_sv_hash_table[C_SV_HASH_SIZE] __attribute__ ((aligned (8)));
244
245
246 static boolean_t compressor_needs_to_swap(void);
247 static void vm_compressor_swap_trigger_thread(void);
248 static void vm_compressor_do_delayed_compactions(boolean_t);
249 static void vm_compressor_compact_and_swap(boolean_t);
250 static void vm_compressor_age_swapped_in_segments(boolean_t);
251
252 static void vm_compressor_take_paging_space_action(void);
253
254 boolean_t vm_compressor_low_on_space(void);
255
256 void compute_swapout_target_age(void);
257
258 boolean_t c_seg_major_compact(c_segment_t, c_segment_t);
259 boolean_t c_seg_major_compact_ok(c_segment_t, c_segment_t);
260
261 int c_seg_minor_compaction_and_unlock(c_segment_t, boolean_t);
262 int c_seg_do_minor_compaction_and_unlock(c_segment_t, boolean_t, boolean_t, boolean_t);
263 void c_seg_try_minor_compaction_and_unlock(c_segment_t c_seg);
264
265 void c_seg_move_to_sparse_list(c_segment_t);
266 void c_seg_insert_into_q(queue_head_t *, c_segment_t);
267
268 uint64_t vm_available_memory(void);
269 uint64_t vm_compressor_pages_compressed(void);
270
271 /*
272 * indicate the need to do a major compaction if
273 * the overall set of in-use compression segments
274 * becomes sparse... on systems that support pressure
275 * driven swapping, this will also cause swapouts to
276 * be initiated.
277 */
278 static inline boolean_t vm_compressor_needs_to_major_compact()
279 {
280 uint32_t incore_seg_count;
281
282 incore_seg_count = c_segment_count - c_swappedout_count - c_swappedout_sparse_count;
283
284 if ((c_segment_count >= (c_segments_nearing_limit / 8)) &&
285 ((incore_seg_count * C_SEG_MAX_PAGES) - VM_PAGE_COMPRESSOR_COUNT) >
286 ((incore_seg_count / 8) * C_SEG_MAX_PAGES))
287 return (1);
288 return (0);
289 }
290
291
292 uint64_t
293 vm_available_memory(void)
294 {
295 return (((uint64_t)AVAILABLE_NON_COMPRESSED_MEMORY) * PAGE_SIZE_64);
296 }
297
298
299 uint64_t
300 vm_compressor_pages_compressed(void)
301 {
302 return (c_segment_pages_compressed * PAGE_SIZE_64);
303 }
304
305
306 boolean_t
307 vm_compressor_low_on_space(void)
308 {
309 if ((c_segment_pages_compressed > c_segment_pages_compressed_nearing_limit) ||
310 (c_segment_count > c_segments_nearing_limit))
311 return (TRUE);
312
313 return (FALSE);
314 }
315
316
317 int
318 vm_wants_task_throttled(task_t task)
319 {
320 if (task == kernel_task)
321 return (0);
322
323 if (VM_CONFIG_SWAP_IS_ACTIVE) {
324 if ((vm_compressor_low_on_space() || HARD_THROTTLE_LIMIT_REACHED()) &&
325 (unsigned int)pmap_compressed(task->map->pmap) > (c_segment_pages_compressed / 4))
326 return (1);
327 }
328 return (0);
329 }
330
331
332
333 static uint32_t no_paging_space_action_in_progress = 0;
334 extern void memorystatus_send_low_swap_note(void);
335
336 static void
337 vm_compressor_take_paging_space_action(void)
338 {
339 if (no_paging_space_action_in_progress == 0) {
340
341 if (OSCompareAndSwap(0, 1, (UInt32 *)&no_paging_space_action_in_progress)) {
342
343 if (no_paging_space_action()) {
344 memorystatus_send_low_swap_note();
345 }
346
347 no_paging_space_action_in_progress = 0;
348 }
349 }
350 }
351
352
353 void
354 vm_compressor_init_locks(void)
355 {
356 lck_grp_attr_setdefault(&vm_compressor_lck_grp_attr);
357 lck_grp_init(&vm_compressor_lck_grp, "vm_compressor", &vm_compressor_lck_grp_attr);
358 lck_attr_setdefault(&vm_compressor_lck_attr);
359
360 lck_rw_init(&c_master_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
361 }
362
363
364 void
365 vm_decompressor_lock(void)
366 {
367 PAGE_REPLACEMENT_ALLOWED(TRUE);
368
369 decompressions_blocked = TRUE;
370
371 PAGE_REPLACEMENT_ALLOWED(FALSE);
372 }
373
374 void
375 vm_decompressor_unlock(void)
376 {
377 PAGE_REPLACEMENT_ALLOWED(TRUE);
378
379 decompressions_blocked = FALSE;
380
381 PAGE_REPLACEMENT_ALLOWED(FALSE);
382
383 thread_wakeup((event_t)&decompressions_blocked);
384 }
385
386 static inline void cslot_copy(c_slot_t cdst, c_slot_t csrc) {
387 #if CHECKSUM_THE_DATA
388 cdst->c_hash_data = csrc->c_hash_data;
389 #endif
390 #if CHECKSUM_THE_COMPRESSED_DATA
391 cdst->c_hash_compressed_data = csrc->c_hash_compressed_data;
392 #endif
393 cdst->c_size = csrc->c_size;
394 cdst->c_packed_ptr = csrc->c_packed_ptr;
395 }
396
397 vm_map_t compressor_map;
398
399 void
400 vm_compressor_init(void)
401 {
402 thread_t thread;
403 struct c_slot cs_dummy;
404 c_slot_t cs = &cs_dummy;
405 int c_segment_min_size;
406 int c_segment_padded_size;
407 kern_return_t retval = KERN_SUCCESS;
408 vm_offset_t start_addr = 0;
409 vm_size_t c_segments_arr_size = 0, compressor_submap_size = 0;
410 #if RECORD_THE_COMPRESSED_DATA
411 vm_size_t c_compressed_record_sbuf_size = 0;
412 #endif /* RECORD_THE_COMPRESSED_DATA */
413
414 /*
415 * ensure that any pointer that gets created from
416 * the vm_page zone can be packed properly
417 */
418 cs->c_packed_ptr = C_SLOT_PACK_PTR(zone_map_min_address);
419
420 if (C_SLOT_UNPACK_PTR(cs) != (uintptr_t)zone_map_min_address)
421 panic("C_SLOT_UNPACK_PTR failed on zone_map_min_address - %p", (void *)zone_map_min_address);
422
423 cs->c_packed_ptr = C_SLOT_PACK_PTR(zone_map_max_address);
424
425 if (C_SLOT_UNPACK_PTR(cs) != (uintptr_t)zone_map_max_address)
426 panic("C_SLOT_UNPACK_PTR failed on zone_map_max_address - %p", (void *)zone_map_max_address);
427
428
429 assert((C_SEGMENTS_PER_PAGE * sizeof(union c_segu)) == PAGE_SIZE);
430
431 PE_parse_boot_argn("vm_compression_limit", &vm_compression_limit, sizeof (vm_compression_limit));
432
433 if (max_mem <= (3ULL * 1024ULL * 1024ULL * 1024ULL)) {
434 vm_compressor_minorcompact_threshold_divisor = 11;
435 vm_compressor_majorcompact_threshold_divisor = 13;
436 vm_compressor_unthrottle_threshold_divisor = 20;
437 vm_compressor_catchup_threshold_divisor = 35;
438 } else {
439 vm_compressor_minorcompact_threshold_divisor = 20;
440 vm_compressor_majorcompact_threshold_divisor = 25;
441 vm_compressor_unthrottle_threshold_divisor = 35;
442 vm_compressor_catchup_threshold_divisor = 50;
443 }
444 /*
445 * vm_page_init_lck_grp is now responsible for calling vm_compressor_init_locks
446 * c_master_lock needs to be available early so that "vm_page_find_contiguous" can
447 * use PAGE_REPLACEMENT_ALLOWED to coordinate with the compressor.
448 */
449
450 c_list_lock = lck_mtx_alloc_init(&vm_compressor_lck_grp, &vm_compressor_lck_attr);
451
452 queue_init(&c_bad_list_head);
453 queue_init(&c_age_list_head);
454 queue_init(&c_minor_list_head);
455 queue_init(&c_major_list_head);
456 queue_init(&c_filling_list_head);
457 queue_init(&c_swapout_list_head);
458 queue_init(&c_swappedin_list_head);
459 queue_init(&c_swappedout_list_head);
460 queue_init(&c_swappedout_sparse_list_head);
461
462 c_segment_min_size = sizeof(struct c_segment) + (C_SEG_SLOT_VAR_ARRAY_MIN_LEN * sizeof(struct c_slot));
463
464 for (c_segment_padded_size = 128; c_segment_padded_size < c_segment_min_size; c_segment_padded_size = c_segment_padded_size << 1);
465
466 compressor_segment_zone = zinit(c_segment_padded_size, 128000 * c_segment_padded_size, PAGE_SIZE, "compressor_segment");
467 zone_change(compressor_segment_zone, Z_CALLERACCT, FALSE);
468 zone_change(compressor_segment_zone, Z_NOENCRYPT, TRUE);
469
470 c_seg_fixed_array_len = (c_segment_padded_size - sizeof(struct c_segment)) / sizeof(struct c_slot);
471
472 c_free_segno_head = -1;
473 c_segments_available = 0;
474
475 if (vm_compression_limit == 0) {
476 c_segment_pages_compressed_limit = (uint32_t)((max_mem / PAGE_SIZE)) * vm_scale;
477
478 #define OLD_SWAP_LIMIT (1024 * 1024 * 16)
479 #define MAX_SWAP_LIMIT (1024 * 1024 * 128)
480
481 if (c_segment_pages_compressed_limit > (OLD_SWAP_LIMIT))
482 c_segment_pages_compressed_limit = OLD_SWAP_LIMIT;
483
484 if (c_segment_pages_compressed_limit < (uint32_t)(max_mem / PAGE_SIZE_64))
485 c_segment_pages_compressed_limit = (uint32_t)(max_mem / PAGE_SIZE_64);
486 } else {
487 if (vm_compression_limit < MAX_SWAP_LIMIT)
488 c_segment_pages_compressed_limit = vm_compression_limit;
489 else
490 c_segment_pages_compressed_limit = MAX_SWAP_LIMIT;
491 }
492 if ((c_segments_limit = c_segment_pages_compressed_limit / (C_SEG_BUFSIZE / PAGE_SIZE)) > C_SEG_MAX_LIMIT)
493 c_segments_limit = C_SEG_MAX_LIMIT;
494
495 c_segment_pages_compressed_nearing_limit = (c_segment_pages_compressed_limit * 98) / 100;
496 c_segments_nearing_limit = (c_segments_limit * 98) / 100;
497
498 c_segments_busy = FALSE;
499
500 /*
501 * Submap needs space for:
502 * - c_segments
503 * - c_buffers
504 * - swap reclaimations -- C_SEG_BUFSIZE
505 */
506 c_segments_arr_size = vm_map_round_page((sizeof(union c_segu) * c_segments_limit),VM_MAP_PAGE_MASK(kernel_map));
507 c_buffers_size = vm_map_round_page(((vm_size_t)C_SEG_ALLOCSIZE * (vm_size_t)c_segments_limit), VM_MAP_PAGE_MASK(kernel_map));
508
509 compressor_submap_size = c_segments_arr_size + c_buffers_size + C_SEG_BUFSIZE;
510
511 #if RECORD_THE_COMPRESSED_DATA
512 c_compressed_record_sbuf_size = (vm_size_t)C_SEG_ALLOCSIZE + (PAGE_SIZE * 2);
513 compressor_submap_size += c_compressed_record_sbuf_size;
514 #endif /* RECORD_THE_COMPRESSED_DATA */
515
516 retval = kmem_suballoc(kernel_map, &start_addr, compressor_submap_size,
517 FALSE, VM_FLAGS_ANYWHERE | VM_FLAGS_PERMANENT | VM_MAKE_TAG(0),
518 &compressor_map);
519
520 if (retval != KERN_SUCCESS)
521 panic("vm_compressor_init: kmem_suballoc failed");
522
523 if (kernel_memory_allocate(compressor_map, (vm_offset_t *)(&c_segments), (sizeof(union c_segu) * c_segments_limit), 0, KMA_KOBJECT | KMA_VAONLY | KMA_PERMANENT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS)
524 panic("vm_compressor_init: kernel_memory_allocate failed - c_segments\n");
525 if (kernel_memory_allocate(compressor_map, &c_buffers, c_buffers_size, 0, KMA_COMPRESSOR | KMA_VAONLY | KMA_PERMANENT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS)
526 panic("vm_compressor_init: kernel_memory_allocate failed - c_buffers\n");
527
528 c_segments_next_page = (caddr_t)c_segments;
529 vm_compressor_algorithm_init();
530
531 {
532 host_basic_info_data_t hinfo;
533 mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
534
535 #define BSD_HOST 1
536 host_info((host_t)BSD_HOST, HOST_BASIC_INFO, (host_info_t)&hinfo, &count);
537
538 compressor_cpus = hinfo.max_cpus;
539 compressor_scratch_bufs = kalloc_tag(compressor_cpus * vm_compressor_get_decode_scratch_size(), VM_KERN_MEMORY_COMPRESSOR);
540
541 kdp_compressor_scratch_buf = kalloc_tag(vm_compressor_get_decode_scratch_size(), VM_KERN_MEMORY_COMPRESSOR);
542 kdp_compressor_decompressed_page = kalloc_tag(PAGE_SIZE, VM_KERN_MEMORY_COMPRESSOR);
543 kdp_compressor_decompressed_page_paddr = kvtophys((vm_offset_t)kdp_compressor_decompressed_page);
544 kdp_compressor_decompressed_page_ppnum = (ppnum_t) atop(kdp_compressor_decompressed_page_paddr);
545 }
546 #if CONFIG_FREEZE
547 freezer_compressor_scratch_buf = kalloc_tag(vm_compressor_get_encode_scratch_size(), VM_KERN_MEMORY_COMPRESSOR);
548 #endif
549
550 #if RECORD_THE_COMPRESSED_DATA
551 if (kernel_memory_allocate(compressor_map, (vm_offset_t *)&c_compressed_record_sbuf, c_compressed_record_sbuf_size, 0, KMA_KOBJECT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS)
552 panic("vm_compressor_init: kernel_memory_allocate failed - c_compressed_record_sbuf\n");
553
554 c_compressed_record_cptr = c_compressed_record_sbuf;
555 c_compressed_record_ebuf = c_compressed_record_sbuf + c_compressed_record_sbuf_size;
556 #endif
557
558 if (kernel_thread_start_priority((thread_continue_t)vm_compressor_swap_trigger_thread, NULL,
559 BASEPRI_PREEMPT - 1, &thread) != KERN_SUCCESS) {
560 panic("vm_compressor_swap_trigger_thread: create failed");
561 }
562 thread_deallocate(thread);
563
564 if (vm_pageout_internal_start() != KERN_SUCCESS) {
565 panic("vm_compressor_init: Failed to start the internal pageout thread.\n");
566 }
567 if (VM_CONFIG_SWAP_IS_PRESENT)
568 vm_compressor_swap_init();
569
570 if (VM_CONFIG_COMPRESSOR_IS_ACTIVE)
571 vm_compressor_is_active = 1;
572
573 #if CONFIG_FREEZE
574 memorystatus_freeze_enabled = TRUE;
575 #endif /* CONFIG_FREEZE */
576
577 vm_compressor_available = 1;
578
579 vm_page_reactivate_all_throttled();
580 }
581
582
583 #if VALIDATE_C_SEGMENTS
584
585 static void
586 c_seg_validate(c_segment_t c_seg, boolean_t must_be_compact)
587 {
588 int c_indx;
589 int32_t bytes_used;
590 int32_t bytes_unused;
591 uint32_t c_rounded_size;
592 uint32_t c_size;
593 c_slot_t cs;
594
595 if (c_seg->c_firstemptyslot < c_seg->c_nextslot) {
596 c_indx = c_seg->c_firstemptyslot;
597 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
598
599 if (cs == NULL)
600 panic("c_seg_validate: no slot backing c_firstemptyslot");
601
602 if (cs->c_size)
603 panic("c_seg_validate: c_firstemptyslot has non-zero size (%d)\n", cs->c_size);
604 }
605 bytes_used = 0;
606 bytes_unused = 0;
607
608 for (c_indx = 0; c_indx < c_seg->c_nextslot; c_indx++) {
609
610 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
611
612 c_size = UNPACK_C_SIZE(cs);
613
614 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
615
616 bytes_used += c_rounded_size;
617
618 #if CHECKSUM_THE_COMPRESSED_DATA
619 if (c_size && cs->c_hash_compressed_data != hash_string((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size))
620 panic("compressed data doesn't match original");
621 #endif
622 }
623
624 if (bytes_used != c_seg->c_bytes_used)
625 panic("c_seg_validate: bytes_used mismatch - found %d, segment has %d\n", bytes_used, c_seg->c_bytes_used);
626
627 if (c_seg->c_bytes_used > C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset))
628 panic("c_seg_validate: c_bytes_used > c_nextoffset - c_nextoffset = %d, c_bytes_used = %d\n",
629 (int32_t)C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset), c_seg->c_bytes_used);
630
631 if (must_be_compact) {
632 if (c_seg->c_bytes_used != C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset))
633 panic("c_seg_validate: c_bytes_used doesn't match c_nextoffset - c_nextoffset = %d, c_bytes_used = %d\n",
634 (int32_t)C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset), c_seg->c_bytes_used);
635 }
636 }
637
638 #endif
639
640
641 void
642 c_seg_need_delayed_compaction(c_segment_t c_seg, boolean_t c_list_lock_held)
643 {
644 boolean_t clear_busy = FALSE;
645
646 if (c_list_lock_held == FALSE) {
647 if ( !lck_mtx_try_lock_spin_always(c_list_lock)) {
648 C_SEG_BUSY(c_seg);
649
650 lck_mtx_unlock_always(&c_seg->c_lock);
651 lck_mtx_lock_spin_always(c_list_lock);
652 lck_mtx_lock_spin_always(&c_seg->c_lock);
653
654 clear_busy = TRUE;
655 }
656 }
657 assert(c_seg->c_state != C_IS_FILLING);
658
659 if (!c_seg->c_on_minorcompact_q && !(C_SEG_IS_ONDISK(c_seg))) {
660 queue_enter(&c_minor_list_head, c_seg, c_segment_t, c_list);
661 c_seg->c_on_minorcompact_q = 1;
662 c_minor_count++;
663 }
664 if (c_list_lock_held == FALSE)
665 lck_mtx_unlock_always(c_list_lock);
666
667 if (clear_busy == TRUE)
668 C_SEG_WAKEUP_DONE(c_seg);
669 }
670
671
672 unsigned int c_seg_moved_to_sparse_list = 0;
673
674 void
675 c_seg_move_to_sparse_list(c_segment_t c_seg)
676 {
677 boolean_t clear_busy = FALSE;
678
679 if ( !lck_mtx_try_lock_spin_always(c_list_lock)) {
680 C_SEG_BUSY(c_seg);
681
682 lck_mtx_unlock_always(&c_seg->c_lock);
683 lck_mtx_lock_spin_always(c_list_lock);
684 lck_mtx_lock_spin_always(&c_seg->c_lock);
685
686 clear_busy = TRUE;
687 }
688 c_seg_switch_state(c_seg, C_ON_SWAPPEDOUTSPARSE_Q, FALSE);
689
690 c_seg_moved_to_sparse_list++;
691
692 lck_mtx_unlock_always(c_list_lock);
693
694 if (clear_busy == TRUE)
695 C_SEG_WAKEUP_DONE(c_seg);
696 }
697
698
699 void
700 c_seg_insert_into_q(queue_head_t *qhead, c_segment_t c_seg)
701 {
702 c_segment_t c_seg_next;
703
704 if (queue_empty(qhead)) {
705 queue_enter(qhead, c_seg, c_segment_t, c_age_list);
706 } else {
707 c_seg_next = (c_segment_t)queue_first(qhead);
708
709 while (TRUE) {
710
711 if (c_seg->c_generation_id < c_seg_next->c_generation_id) {
712 queue_insert_before(qhead, c_seg, c_seg_next, c_segment_t, c_age_list);
713 break;
714 }
715 c_seg_next = (c_segment_t) queue_next(&c_seg_next->c_age_list);
716
717 if (queue_end(qhead, (queue_entry_t) c_seg_next)) {
718 queue_enter(qhead, c_seg, c_segment_t, c_age_list);
719 break;
720 }
721 }
722 }
723 }
724
725
726 int try_minor_compaction_failed = 0;
727 int try_minor_compaction_succeeded = 0;
728
729 void
730 c_seg_try_minor_compaction_and_unlock(c_segment_t c_seg)
731 {
732
733 assert(c_seg->c_on_minorcompact_q);
734 /*
735 * c_seg is currently on the delayed minor compaction
736 * queue and we have c_seg locked... if we can get the
737 * c_list_lock w/o blocking (if we blocked we could deadlock
738 * because the lock order is c_list_lock then c_seg's lock)
739 * we'll pull it from the delayed list and free it directly
740 */
741 if ( !lck_mtx_try_lock_spin_always(c_list_lock)) {
742 /*
743 * c_list_lock is held, we need to bail
744 */
745 try_minor_compaction_failed++;
746
747 lck_mtx_unlock_always(&c_seg->c_lock);
748 } else {
749 try_minor_compaction_succeeded++;
750
751 C_SEG_BUSY(c_seg);
752 c_seg_do_minor_compaction_and_unlock(c_seg, TRUE, FALSE, FALSE);
753 }
754 }
755
756
757 int
758 c_seg_do_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy, boolean_t need_list_lock, boolean_t disallow_page_replacement)
759 {
760 int c_seg_freed;
761
762 assert(c_seg->c_busy);
763
764 /*
765 * check for the case that can occur when we are not swapping
766 * and this segment has been major compacted in the past
767 * and moved to the majorcompact q to remove it from further
768 * consideration... if the occupancy falls too low we need
769 * to put it back on the age_q so that it will be considered
770 * in the next major compaction sweep... if we don't do this
771 * we will eventually run into the c_segments_limit
772 */
773 if (c_seg->c_state == C_ON_MAJORCOMPACT_Q && C_SEG_SHOULD_MAJORCOMPACT(c_seg)) {
774
775 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
776 }
777 if (!c_seg->c_on_minorcompact_q) {
778 if (clear_busy == TRUE)
779 C_SEG_WAKEUP_DONE(c_seg);
780
781 lck_mtx_unlock_always(&c_seg->c_lock);
782
783 return (0);
784 }
785 queue_remove(&c_minor_list_head, c_seg, c_segment_t, c_list);
786 c_seg->c_on_minorcompact_q = 0;
787 c_minor_count--;
788
789 lck_mtx_unlock_always(c_list_lock);
790
791 if (disallow_page_replacement == TRUE) {
792 lck_mtx_unlock_always(&c_seg->c_lock);
793
794 PAGE_REPLACEMENT_DISALLOWED(TRUE);
795
796 lck_mtx_lock_spin_always(&c_seg->c_lock);
797 }
798 c_seg_freed = c_seg_minor_compaction_and_unlock(c_seg, clear_busy);
799
800 if (disallow_page_replacement == TRUE)
801 PAGE_REPLACEMENT_DISALLOWED(FALSE);
802
803 if (need_list_lock == TRUE)
804 lck_mtx_lock_spin_always(c_list_lock);
805
806 return (c_seg_freed);
807 }
808
809
810 void
811 c_seg_wait_on_busy(c_segment_t c_seg)
812 {
813 c_seg->c_wanted = 1;
814 assert_wait((event_t) (c_seg), THREAD_UNINT);
815
816 lck_mtx_unlock_always(&c_seg->c_lock);
817 thread_block(THREAD_CONTINUE_NULL);
818 }
819
820
821 void
822 c_seg_switch_state(c_segment_t c_seg, int new_state, boolean_t insert_head)
823 {
824 int old_state = c_seg->c_state;
825
826 #if __i386__ || __x86_64__
827 if (new_state != C_IS_FILLING)
828 LCK_MTX_ASSERT(&c_seg->c_lock, LCK_MTX_ASSERT_OWNED);
829 LCK_MTX_ASSERT(c_list_lock, LCK_MTX_ASSERT_OWNED);
830 #endif
831 switch (old_state) {
832
833 case C_IS_EMPTY:
834 assert(new_state == C_IS_FILLING || new_state == C_IS_FREE);
835
836 c_empty_count--;
837 break;
838
839 case C_IS_FILLING:
840 assert(new_state == C_ON_AGE_Q || new_state == C_ON_SWAPOUT_Q);
841
842 queue_remove(&c_filling_list_head, c_seg, c_segment_t, c_age_list);
843 c_filling_count--;
844 break;
845
846 case C_ON_AGE_Q:
847 assert(new_state == C_ON_SWAPOUT_Q || new_state == C_ON_MAJORCOMPACT_Q ||
848 new_state == C_IS_FREE);
849
850 queue_remove(&c_age_list_head, c_seg, c_segment_t, c_age_list);
851 c_age_count--;
852 break;
853
854 case C_ON_SWAPPEDIN_Q:
855 assert(new_state == C_ON_AGE_Q || new_state == C_IS_FREE);
856
857 queue_remove(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
858 c_swappedin_count--;
859 break;
860
861 case C_ON_SWAPOUT_Q:
862 assert(new_state == C_ON_SWAPPEDOUT_Q || new_state == C_ON_SWAPPEDOUTSPARSE_Q ||
863 new_state == C_ON_AGE_Q || new_state == C_IS_FREE || new_state == C_IS_EMPTY);
864
865 queue_remove(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
866 thread_wakeup((event_t)&compaction_swapper_running);
867 c_swapout_count--;
868 break;
869
870 case C_ON_SWAPPEDOUT_Q:
871 assert(new_state == C_ON_SWAPPEDIN_Q || new_state == C_ON_AGE_Q ||
872 new_state == C_ON_SWAPPEDOUTSPARSE_Q ||
873 new_state == C_ON_BAD_Q || new_state == C_IS_EMPTY || new_state == C_IS_FREE);
874
875 queue_remove(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
876 c_swappedout_count--;
877 break;
878
879 case C_ON_SWAPPEDOUTSPARSE_Q:
880 assert(new_state == C_ON_SWAPPEDIN_Q || new_state == C_ON_AGE_Q ||
881 new_state == C_ON_BAD_Q || new_state == C_IS_EMPTY || new_state == C_IS_FREE);
882
883 queue_remove(&c_swappedout_sparse_list_head, c_seg, c_segment_t, c_age_list);
884 c_swappedout_sparse_count--;
885 break;
886
887 case C_ON_MAJORCOMPACT_Q:
888 assert(new_state == C_ON_AGE_Q || new_state == C_IS_FREE);
889
890 queue_remove(&c_major_list_head, c_seg, c_segment_t, c_age_list);
891 c_major_count--;
892 break;
893
894 case C_ON_BAD_Q:
895 assert(new_state == C_IS_FREE);
896
897 queue_remove(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
898 c_bad_count--;
899 break;
900
901 default:
902 panic("c_seg %p has bad c_state = %d\n", c_seg, old_state);
903 }
904
905 switch(new_state) {
906 case C_IS_FREE:
907 assert(old_state != C_IS_FILLING);
908
909 break;
910
911 case C_IS_EMPTY:
912 assert(old_state == C_ON_SWAPOUT_Q || old_state == C_ON_SWAPPEDOUT_Q || old_state == C_ON_SWAPPEDOUTSPARSE_Q);
913
914 c_empty_count++;
915 break;
916
917 case C_IS_FILLING:
918 assert(old_state == C_IS_EMPTY);
919
920 queue_enter(&c_filling_list_head, c_seg, c_segment_t, c_age_list);
921 c_filling_count++;
922 break;
923
924 case C_ON_AGE_Q:
925 assert(old_state == C_IS_FILLING || old_state == C_ON_SWAPPEDIN_Q || old_state == C_ON_SWAPOUT_Q ||
926 old_state == C_ON_MAJORCOMPACT_Q || old_state == C_ON_SWAPPEDOUT_Q || old_state == C_ON_SWAPPEDOUTSPARSE_Q);
927
928 if (old_state == C_IS_FILLING)
929 queue_enter(&c_age_list_head, c_seg, c_segment_t, c_age_list);
930 else {
931 if (!queue_empty(&c_age_list_head)) {
932 c_segment_t c_first;
933
934 c_first = (c_segment_t)queue_first(&c_age_list_head);
935 c_seg->c_creation_ts = c_first->c_creation_ts;
936 }
937 queue_enter_first(&c_age_list_head, c_seg, c_segment_t, c_age_list);
938 }
939 c_age_count++;
940 break;
941
942 case C_ON_SWAPPEDIN_Q:
943 assert(c_seg->c_state == C_ON_SWAPPEDOUT_Q || c_seg->c_state == C_ON_SWAPPEDOUTSPARSE_Q);
944
945 if (insert_head == TRUE)
946 queue_enter_first(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
947 else
948 queue_enter(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
949 c_swappedin_count++;
950 break;
951
952 case C_ON_SWAPOUT_Q:
953 assert(old_state == C_ON_AGE_Q || old_state == C_IS_FILLING);
954
955 if (insert_head == TRUE)
956 queue_enter_first(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
957 else
958 queue_enter(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
959 c_swapout_count++;
960 break;
961
962 case C_ON_SWAPPEDOUT_Q:
963 assert(c_seg->c_state == C_ON_SWAPOUT_Q);
964
965 if (insert_head == TRUE)
966 queue_enter_first(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
967 else
968 queue_enter(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
969 c_swappedout_count++;
970 break;
971
972 case C_ON_SWAPPEDOUTSPARSE_Q:
973 assert(c_seg->c_state == C_ON_SWAPOUT_Q || c_seg->c_state == C_ON_SWAPPEDOUT_Q);
974
975 if (insert_head == TRUE)
976 queue_enter_first(&c_swappedout_sparse_list_head, c_seg, c_segment_t, c_age_list);
977 else
978 queue_enter(&c_swappedout_sparse_list_head, c_seg, c_segment_t, c_age_list);
979
980 c_swappedout_sparse_count++;
981 break;
982
983 case C_ON_MAJORCOMPACT_Q:
984 assert(c_seg->c_state == C_ON_AGE_Q);
985
986 if (insert_head == TRUE)
987 queue_enter_first(&c_major_list_head, c_seg, c_segment_t, c_age_list);
988 else
989 queue_enter(&c_major_list_head, c_seg, c_segment_t, c_age_list);
990 c_major_count++;
991 break;
992
993 case C_ON_BAD_Q:
994 assert(c_seg->c_state == C_ON_SWAPPEDOUT_Q || c_seg->c_state == C_ON_SWAPPEDOUTSPARSE_Q);
995
996 if (insert_head == TRUE)
997 queue_enter_first(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
998 else
999 queue_enter(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
1000 c_bad_count++;
1001 break;
1002
1003 default:
1004 panic("c_seg %p requesting bad c_state = %d\n", c_seg, new_state);
1005 }
1006 c_seg->c_state = new_state;
1007 }
1008
1009
1010
1011 void
1012 c_seg_free(c_segment_t c_seg)
1013 {
1014 assert(c_seg->c_busy);
1015
1016 lck_mtx_unlock_always(&c_seg->c_lock);
1017 lck_mtx_lock_spin_always(c_list_lock);
1018 lck_mtx_lock_spin_always(&c_seg->c_lock);
1019
1020 c_seg_free_locked(c_seg);
1021 }
1022
1023
1024 void
1025 c_seg_free_locked(c_segment_t c_seg)
1026 {
1027 int segno;
1028 int pages_populated = 0;
1029 int32_t *c_buffer = NULL;
1030 uint64_t c_swap_handle = 0;
1031
1032 assert(c_seg->c_busy);
1033 assert(!c_seg->c_on_minorcompact_q);
1034 assert(!c_seg->c_busy_swapping);
1035
1036 if (c_seg->c_overage_swap == TRUE) {
1037 c_overage_swapped_count--;
1038 c_seg->c_overage_swap = FALSE;
1039 }
1040 if ( !(C_SEG_IS_ONDISK(c_seg)))
1041 c_buffer = c_seg->c_store.c_buffer;
1042 else
1043 c_swap_handle = c_seg->c_store.c_swap_handle;
1044
1045 c_seg_switch_state(c_seg, C_IS_FREE, FALSE);
1046
1047 lck_mtx_unlock_always(c_list_lock);
1048
1049 if (c_buffer) {
1050 pages_populated = (round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE;
1051 c_seg->c_store.c_buffer = NULL;
1052 } else
1053 c_seg->c_store.c_swap_handle = (uint64_t)-1;
1054
1055 lck_mtx_unlock_always(&c_seg->c_lock);
1056
1057 if (c_buffer) {
1058 if (pages_populated)
1059 kernel_memory_depopulate(compressor_map, (vm_offset_t) c_buffer, pages_populated * PAGE_SIZE, KMA_COMPRESSOR);
1060
1061 } else if (c_swap_handle) {
1062 /*
1063 * Free swap space on disk.
1064 */
1065 vm_swap_free(c_swap_handle);
1066 }
1067 lck_mtx_lock_spin_always(&c_seg->c_lock);
1068
1069 C_SEG_WAKEUP_DONE(c_seg);
1070 lck_mtx_unlock_always(&c_seg->c_lock);
1071
1072 segno = c_seg->c_mysegno;
1073
1074 lck_mtx_lock_spin_always(c_list_lock);
1075 /*
1076 * because the c_buffer is now associated with the segno,
1077 * we can't put the segno back on the free list until
1078 * after we have depopulated the c_buffer range, or
1079 * we run the risk of depopulating a range that is
1080 * now being used in one of the compressor heads
1081 */
1082 c_segments[segno].c_segno = c_free_segno_head;
1083 c_free_segno_head = segno;
1084 c_segment_count--;
1085
1086 lck_mtx_unlock_always(c_list_lock);
1087
1088 lck_mtx_destroy(&c_seg->c_lock, &vm_compressor_lck_grp);
1089
1090 if (c_seg->c_slot_var_array_len)
1091 kfree(c_seg->c_slot_var_array, sizeof(struct c_slot) * c_seg->c_slot_var_array_len);
1092
1093 zfree(compressor_segment_zone, c_seg);
1094 }
1095
1096
1097 int c_seg_trim_page_count = 0;
1098
1099 void
1100 c_seg_trim_tail(c_segment_t c_seg)
1101 {
1102 c_slot_t cs;
1103 uint32_t c_size;
1104 uint32_t c_offset;
1105 uint32_t c_rounded_size;
1106 uint16_t current_nextslot;
1107 uint32_t current_populated_offset;
1108
1109 if (c_seg->c_bytes_used == 0)
1110 return;
1111 current_nextslot = c_seg->c_nextslot;
1112 current_populated_offset = c_seg->c_populated_offset;
1113
1114 while (c_seg->c_nextslot) {
1115
1116 cs = C_SEG_SLOT_FROM_INDEX(c_seg, (c_seg->c_nextslot - 1));
1117
1118 c_size = UNPACK_C_SIZE(cs);
1119
1120 if (c_size) {
1121 if (current_nextslot != c_seg->c_nextslot) {
1122 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
1123 c_offset = cs->c_offset + C_SEG_BYTES_TO_OFFSET(c_rounded_size);
1124
1125 c_seg->c_nextoffset = c_offset;
1126 c_seg->c_populated_offset = (c_offset + (C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1)) & ~(C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1);
1127
1128 if (c_seg->c_firstemptyslot > c_seg->c_nextslot)
1129 c_seg->c_firstemptyslot = c_seg->c_nextslot;
1130
1131 c_seg_trim_page_count += ((round_page_32(C_SEG_OFFSET_TO_BYTES(current_populated_offset)) -
1132 round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE);
1133 }
1134 break;
1135 }
1136 c_seg->c_nextslot--;
1137 }
1138 assert(c_seg->c_nextslot);
1139 }
1140
1141
1142 int
1143 c_seg_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy)
1144 {
1145 c_slot_mapping_t slot_ptr;
1146 uint32_t c_offset = 0;
1147 uint32_t old_populated_offset;
1148 uint32_t c_rounded_size;
1149 uint32_t c_size;
1150 int c_indx = 0;
1151 int i;
1152 c_slot_t c_dst;
1153 c_slot_t c_src;
1154
1155 assert(c_seg->c_busy);
1156
1157 #if VALIDATE_C_SEGMENTS
1158 c_seg_validate(c_seg, FALSE);
1159 #endif
1160 if (c_seg->c_bytes_used == 0) {
1161 c_seg_free(c_seg);
1162 return (1);
1163 }
1164 lck_mtx_unlock_always(&c_seg->c_lock);
1165
1166 if (c_seg->c_firstemptyslot >= c_seg->c_nextslot || C_SEG_UNUSED_BYTES(c_seg) < PAGE_SIZE)
1167 goto done;
1168
1169 #if DEVELOPMENT || DEBUG
1170 C_SEG_MAKE_WRITEABLE(c_seg);
1171 #endif
1172
1173 #if VALIDATE_C_SEGMENTS
1174 c_seg->c_was_minor_compacted++;
1175 #endif
1176 c_indx = c_seg->c_firstemptyslot;
1177 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
1178
1179 old_populated_offset = c_seg->c_populated_offset;
1180 c_offset = c_dst->c_offset;
1181
1182 for (i = c_indx + 1; i < c_seg->c_nextslot && c_offset < c_seg->c_nextoffset; i++) {
1183
1184 c_src = C_SEG_SLOT_FROM_INDEX(c_seg, i);
1185
1186 c_size = UNPACK_C_SIZE(c_src);
1187
1188 if (c_size == 0)
1189 continue;
1190
1191 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
1192
1193 memcpy(&c_seg->c_store.c_buffer[c_offset], &c_seg->c_store.c_buffer[c_src->c_offset], c_rounded_size);
1194
1195 cslot_copy(c_dst, c_src);
1196 c_dst->c_offset = c_offset;
1197
1198 slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
1199 slot_ptr->s_cindx = c_indx;
1200
1201 c_offset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
1202 PACK_C_SIZE(c_src, 0);
1203 c_indx++;
1204
1205 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
1206 }
1207 c_seg->c_firstemptyslot = c_indx;
1208 c_seg->c_nextslot = c_indx;
1209 c_seg->c_nextoffset = c_offset;
1210 c_seg->c_populated_offset = (c_offset + (C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1)) & ~(C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1);
1211 c_seg->c_bytes_unused = 0;
1212
1213 #if VALIDATE_C_SEGMENTS
1214 c_seg_validate(c_seg, TRUE);
1215 #endif
1216 if (old_populated_offset > c_seg->c_populated_offset) {
1217 uint32_t gc_size;
1218 int32_t *gc_ptr;
1219
1220 gc_size = C_SEG_OFFSET_TO_BYTES(old_populated_offset - c_seg->c_populated_offset);
1221 gc_ptr = &c_seg->c_store.c_buffer[c_seg->c_populated_offset];
1222
1223 kernel_memory_depopulate(compressor_map, (vm_offset_t)gc_ptr, gc_size, KMA_COMPRESSOR);
1224 }
1225
1226 #if DEVELOPMENT || DEBUG
1227 C_SEG_WRITE_PROTECT(c_seg);
1228 #endif
1229
1230 done:
1231 if (clear_busy == TRUE) {
1232 lck_mtx_lock_spin_always(&c_seg->c_lock);
1233 C_SEG_WAKEUP_DONE(c_seg);
1234 lck_mtx_unlock_always(&c_seg->c_lock);
1235 }
1236 return (0);
1237 }
1238
1239
1240 static void
1241 c_seg_alloc_nextslot(c_segment_t c_seg)
1242 {
1243 struct c_slot *old_slot_array = NULL;
1244 struct c_slot *new_slot_array = NULL;
1245 int newlen;
1246 int oldlen;
1247
1248 if (c_seg->c_nextslot < c_seg_fixed_array_len)
1249 return;
1250
1251 if ((c_seg->c_nextslot - c_seg_fixed_array_len) >= c_seg->c_slot_var_array_len) {
1252
1253 oldlen = c_seg->c_slot_var_array_len;
1254 old_slot_array = c_seg->c_slot_var_array;
1255
1256 if (oldlen == 0)
1257 newlen = C_SEG_SLOT_VAR_ARRAY_MIN_LEN;
1258 else
1259 newlen = oldlen * 2;
1260
1261 new_slot_array = (struct c_slot *)kalloc(sizeof(struct c_slot) * newlen);
1262
1263 lck_mtx_lock_spin_always(&c_seg->c_lock);
1264
1265 if (old_slot_array)
1266 memcpy((char *)new_slot_array, (char *)old_slot_array, sizeof(struct c_slot) * oldlen);
1267
1268 c_seg->c_slot_var_array_len = newlen;
1269 c_seg->c_slot_var_array = new_slot_array;
1270
1271 lck_mtx_unlock_always(&c_seg->c_lock);
1272
1273 if (old_slot_array)
1274 kfree(old_slot_array, sizeof(struct c_slot) * oldlen);
1275 }
1276 }
1277
1278
1279
1280 struct {
1281 uint64_t asked_permission;
1282 uint64_t compactions;
1283 uint64_t moved_slots;
1284 uint64_t moved_bytes;
1285 uint64_t wasted_space_in_swapouts;
1286 uint64_t count_of_swapouts;
1287 uint64_t count_of_freed_segs;
1288 } c_seg_major_compact_stats;
1289
1290
1291 #define C_MAJOR_COMPACTION_SIZE_APPROPRIATE ((C_SEG_BUFSIZE * 90) / 100)
1292
1293
1294 boolean_t
1295 c_seg_major_compact_ok(
1296 c_segment_t c_seg_dst,
1297 c_segment_t c_seg_src)
1298 {
1299
1300 c_seg_major_compact_stats.asked_permission++;
1301
1302 if (c_seg_src->c_bytes_used >= C_MAJOR_COMPACTION_SIZE_APPROPRIATE &&
1303 c_seg_dst->c_bytes_used >= C_MAJOR_COMPACTION_SIZE_APPROPRIATE)
1304 return (FALSE);
1305
1306 if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX_INDEX) {
1307 /*
1308 * destination segment is full... can't compact
1309 */
1310 return (FALSE);
1311 }
1312
1313 return (TRUE);
1314 }
1315
1316
1317 boolean_t
1318 c_seg_major_compact(
1319 c_segment_t c_seg_dst,
1320 c_segment_t c_seg_src)
1321 {
1322 c_slot_mapping_t slot_ptr;
1323 uint32_t c_rounded_size;
1324 uint32_t c_size;
1325 uint16_t dst_slot;
1326 int i;
1327 c_slot_t c_dst;
1328 c_slot_t c_src;
1329 boolean_t keep_compacting = TRUE;
1330
1331 /*
1332 * segments are not locked but they are both marked c_busy
1333 * which keeps c_decompress from working on them...
1334 * we can safely allocate new pages, move compressed data
1335 * from c_seg_src to c_seg_dst and update both c_segment's
1336 * state w/o holding the master lock
1337 */
1338 #if DEVELOPMENT || DEBUG
1339 C_SEG_MAKE_WRITEABLE(c_seg_dst);
1340 #endif
1341
1342 #if VALIDATE_C_SEGMENTS
1343 c_seg_dst->c_was_major_compacted++;
1344 c_seg_src->c_was_major_donor++;
1345 #endif
1346 c_seg_major_compact_stats.compactions++;
1347
1348 dst_slot = c_seg_dst->c_nextslot;
1349
1350 for (i = 0; i < c_seg_src->c_nextslot; i++) {
1351
1352 c_src = C_SEG_SLOT_FROM_INDEX(c_seg_src, i);
1353
1354 c_size = UNPACK_C_SIZE(c_src);
1355
1356 if (c_size == 0) {
1357 /* BATCH: move what we have so far; */
1358 continue;
1359 }
1360
1361 if (C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset - c_seg_dst->c_nextoffset) < (unsigned) c_size) {
1362 int size_to_populate;
1363
1364 /* doesn't fit */
1365 size_to_populate = C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset);
1366
1367 if (size_to_populate == 0) {
1368 /* can't fit */
1369 keep_compacting = FALSE;
1370 break;
1371 }
1372 if (size_to_populate > C_SEG_MAX_POPULATE_SIZE)
1373 size_to_populate = C_SEG_MAX_POPULATE_SIZE;
1374
1375 kernel_memory_populate(compressor_map,
1376 (vm_offset_t) &c_seg_dst->c_store.c_buffer[c_seg_dst->c_populated_offset],
1377 size_to_populate,
1378 KMA_COMPRESSOR,
1379 VM_KERN_MEMORY_COMPRESSOR);
1380
1381 c_seg_dst->c_populated_offset += C_SEG_BYTES_TO_OFFSET(size_to_populate);
1382 assert(C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset) <= C_SEG_BUFSIZE);
1383 }
1384 c_seg_alloc_nextslot(c_seg_dst);
1385
1386 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, c_seg_dst->c_nextslot);
1387
1388 memcpy(&c_seg_dst->c_store.c_buffer[c_seg_dst->c_nextoffset], &c_seg_src->c_store.c_buffer[c_src->c_offset], c_size);
1389
1390 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
1391
1392 c_seg_major_compact_stats.moved_slots++;
1393 c_seg_major_compact_stats.moved_bytes += c_size;
1394
1395 cslot_copy(c_dst, c_src);
1396 c_dst->c_offset = c_seg_dst->c_nextoffset;
1397
1398 if (c_seg_dst->c_firstemptyslot == c_seg_dst->c_nextslot)
1399 c_seg_dst->c_firstemptyslot++;
1400 c_seg_dst->c_nextslot++;
1401 c_seg_dst->c_bytes_used += c_rounded_size;
1402 c_seg_dst->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
1403
1404 PACK_C_SIZE(c_src, 0);
1405
1406 c_seg_src->c_bytes_used -= c_rounded_size;
1407 c_seg_src->c_bytes_unused += c_rounded_size;
1408 c_seg_src->c_firstemptyslot = 0;
1409
1410 if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX_INDEX) {
1411 /* dest segment is now full */
1412 keep_compacting = FALSE;
1413 break;
1414 }
1415 }
1416 #if DEVELOPMENT || DEBUG
1417 C_SEG_WRITE_PROTECT(c_seg_dst);
1418 #endif
1419 if (dst_slot < c_seg_dst->c_nextslot) {
1420
1421 PAGE_REPLACEMENT_ALLOWED(TRUE);
1422 /*
1423 * we've now locked out c_decompress from
1424 * converting the slot passed into it into
1425 * a c_segment_t which allows us to use
1426 * the backptr to change which c_segment and
1427 * index the slot points to
1428 */
1429 while (dst_slot < c_seg_dst->c_nextslot) {
1430
1431 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, dst_slot);
1432
1433 slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
1434 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
1435 slot_ptr->s_cseg = c_seg_dst->c_mysegno + 1;
1436 slot_ptr->s_cindx = dst_slot++;
1437 }
1438 PAGE_REPLACEMENT_ALLOWED(FALSE);
1439 }
1440 return (keep_compacting);
1441 }
1442
1443
1444 uint64_t
1445 vm_compressor_compute_elapsed_msecs(clock_sec_t end_sec, clock_nsec_t end_nsec, clock_sec_t start_sec, clock_nsec_t start_nsec)
1446 {
1447 uint64_t end_msecs;
1448 uint64_t start_msecs;
1449
1450 end_msecs = (end_sec * 1000) + end_nsec / 1000000;
1451 start_msecs = (start_sec * 1000) + start_nsec / 1000000;
1452
1453 return (end_msecs - start_msecs);
1454 }
1455
1456
1457
1458 uint32_t compressor_eval_period_in_msecs = 250;
1459 uint32_t compressor_sample_min_in_msecs = 500;
1460 uint32_t compressor_sample_max_in_msecs = 10000;
1461 uint32_t compressor_thrashing_threshold_per_10msecs = 50;
1462 uint32_t compressor_thrashing_min_per_10msecs = 20;
1463
1464 /* When true, reset sample data next chance we get. */
1465 static boolean_t compressor_need_sample_reset = FALSE;
1466
1467 extern uint32_t vm_page_filecache_min;
1468
1469
1470 void
1471 compute_swapout_target_age(void)
1472 {
1473 clock_sec_t cur_ts_sec;
1474 clock_nsec_t cur_ts_nsec;
1475 uint32_t min_operations_needed_in_this_sample;
1476 uint64_t elapsed_msecs_in_eval;
1477 uint64_t elapsed_msecs_in_sample;
1478 boolean_t need_eval_reset = FALSE;
1479
1480 clock_get_system_nanotime(&cur_ts_sec, &cur_ts_nsec);
1481
1482 elapsed_msecs_in_sample = vm_compressor_compute_elapsed_msecs(cur_ts_sec, cur_ts_nsec, start_of_sample_period_sec, start_of_sample_period_nsec);
1483
1484 if (compressor_need_sample_reset ||
1485 elapsed_msecs_in_sample >= compressor_sample_max_in_msecs) {
1486 compressor_need_sample_reset = TRUE;
1487 need_eval_reset = TRUE;
1488 goto done;
1489 }
1490 elapsed_msecs_in_eval = vm_compressor_compute_elapsed_msecs(cur_ts_sec, cur_ts_nsec, start_of_eval_period_sec, start_of_eval_period_nsec);
1491
1492 if (elapsed_msecs_in_eval < compressor_eval_period_in_msecs)
1493 goto done;
1494 need_eval_reset = TRUE;
1495
1496 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_START, elapsed_msecs_in_eval, sample_period_compression_count, sample_period_decompression_count, 0, 0);
1497
1498 min_operations_needed_in_this_sample = (compressor_thrashing_min_per_10msecs * (uint32_t)elapsed_msecs_in_eval) / 10;
1499
1500 if ((sample_period_compression_count - last_eval_compression_count) < min_operations_needed_in_this_sample ||
1501 (sample_period_decompression_count - last_eval_decompression_count) < min_operations_needed_in_this_sample) {
1502
1503 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, sample_period_compression_count - last_eval_compression_count,
1504 sample_period_decompression_count - last_eval_decompression_count, 0, 1, 0);
1505
1506 swapout_target_age = 0;
1507
1508 compressor_need_sample_reset = TRUE;
1509 need_eval_reset = TRUE;
1510 goto done;
1511 }
1512 last_eval_compression_count = sample_period_compression_count;
1513 last_eval_decompression_count = sample_period_decompression_count;
1514
1515 if (elapsed_msecs_in_sample < compressor_sample_min_in_msecs) {
1516
1517 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, swapout_target_age, 0, 0, 5, 0);
1518 goto done;
1519 }
1520 if (sample_period_decompression_count > ((compressor_thrashing_threshold_per_10msecs * elapsed_msecs_in_sample) / 10)) {
1521
1522 uint64_t running_total;
1523 uint64_t working_target;
1524 uint64_t aging_target;
1525 uint32_t oldest_age_of_csegs_sampled = 0;
1526 uint64_t working_set_approximation = 0;
1527
1528 swapout_target_age = 0;
1529
1530 working_target = (sample_period_decompression_count / 100) * 95; /* 95 percent */
1531 aging_target = (sample_period_decompression_count / 100) * 1; /* 1 percent */
1532 running_total = 0;
1533
1534 for (oldest_age_of_csegs_sampled = 0; oldest_age_of_csegs_sampled < DECOMPRESSION_SAMPLE_MAX_AGE; oldest_age_of_csegs_sampled++) {
1535
1536 running_total += age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
1537
1538 working_set_approximation += oldest_age_of_csegs_sampled * age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
1539
1540 if (running_total >= working_target)
1541 break;
1542 }
1543 if (oldest_age_of_csegs_sampled < DECOMPRESSION_SAMPLE_MAX_AGE) {
1544
1545 working_set_approximation = (working_set_approximation * 1000) / elapsed_msecs_in_sample;
1546
1547 if (working_set_approximation < VM_PAGE_COMPRESSOR_COUNT) {
1548
1549 running_total = overage_decompressions_during_sample_period;
1550
1551 for (oldest_age_of_csegs_sampled = DECOMPRESSION_SAMPLE_MAX_AGE - 1; oldest_age_of_csegs_sampled; oldest_age_of_csegs_sampled--) {
1552 running_total += age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
1553
1554 if (running_total >= aging_target)
1555 break;
1556 }
1557 swapout_target_age = (uint32_t)cur_ts_sec - oldest_age_of_csegs_sampled;
1558
1559 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, swapout_target_age, working_set_approximation, VM_PAGE_COMPRESSOR_COUNT, 2, 0);
1560 } else {
1561 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, working_set_approximation, VM_PAGE_COMPRESSOR_COUNT, 0, 3, 0);
1562 }
1563 } else
1564 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, working_target, running_total, 0, 4, 0);
1565
1566 compressor_need_sample_reset = TRUE;
1567 need_eval_reset = TRUE;
1568 } else
1569 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, sample_period_decompression_count, (compressor_thrashing_threshold_per_10msecs * elapsed_msecs_in_sample) / 10, 0, 6, 0);
1570 done:
1571 if (compressor_need_sample_reset == TRUE) {
1572 bzero(age_of_decompressions_during_sample_period, sizeof(age_of_decompressions_during_sample_period));
1573 overage_decompressions_during_sample_period = 0;
1574
1575 start_of_sample_period_sec = cur_ts_sec;
1576 start_of_sample_period_nsec = cur_ts_nsec;
1577 sample_period_decompression_count = 0;
1578 sample_period_compression_count = 0;
1579 last_eval_decompression_count = 0;
1580 last_eval_compression_count = 0;
1581 compressor_need_sample_reset = FALSE;
1582 }
1583 if (need_eval_reset == TRUE) {
1584 start_of_eval_period_sec = cur_ts_sec;
1585 start_of_eval_period_nsec = cur_ts_nsec;
1586 }
1587 }
1588
1589
1590 int compaction_swapper_init_now = 0;
1591 int compaction_swapper_running = 0;
1592 int compaction_swapper_awakened = 0;
1593 int compaction_swapper_abort = 0;
1594
1595
1596 #if CONFIG_JETSAM
1597 boolean_t memorystatus_kill_on_VM_thrashing(boolean_t);
1598 boolean_t memorystatus_kill_on_FC_thrashing(boolean_t);
1599 int compressor_thrashing_induced_jetsam = 0;
1600 int filecache_thrashing_induced_jetsam = 0;
1601 static boolean_t vm_compressor_thrashing_detected = FALSE;
1602 #endif /* CONFIG_JETSAM */
1603
1604 static boolean_t
1605 compressor_needs_to_swap(void)
1606 {
1607 boolean_t should_swap = FALSE;
1608
1609 if (vm_swapout_ripe_segments == TRUE && c_overage_swapped_count < c_overage_swapped_limit) {
1610 c_segment_t c_seg;
1611 clock_sec_t now;
1612 clock_sec_t age;
1613 clock_nsec_t nsec;
1614
1615 clock_get_system_nanotime(&now, &nsec);
1616 age = 0;
1617
1618 lck_mtx_lock_spin_always(c_list_lock);
1619
1620 if ( !queue_empty(&c_age_list_head)) {
1621 c_seg = (c_segment_t) queue_first(&c_age_list_head);
1622
1623 age = now - c_seg->c_creation_ts;
1624 }
1625 lck_mtx_unlock_always(c_list_lock);
1626
1627 if (age >= vm_ripe_target_age)
1628 return (TRUE);
1629 }
1630 if (VM_CONFIG_SWAP_IS_ACTIVE) {
1631 if (COMPRESSOR_NEEDS_TO_SWAP()) {
1632 return (TRUE);
1633 }
1634 if (VM_PAGE_Q_THROTTLED(&vm_pageout_queue_external) && vm_page_anonymous_count < (vm_page_inactive_count / 20)) {
1635 return (TRUE);
1636 }
1637 if (vm_page_free_count < (vm_page_free_reserved - (COMPRESSOR_FREE_RESERVED_LIMIT * 2)))
1638 return (TRUE);
1639 }
1640 compute_swapout_target_age();
1641
1642 if (swapout_target_age) {
1643 c_segment_t c_seg;
1644
1645 lck_mtx_lock_spin_always(c_list_lock);
1646
1647 if (!queue_empty(&c_age_list_head)) {
1648
1649 c_seg = (c_segment_t) queue_first(&c_age_list_head);
1650
1651 if (c_seg->c_creation_ts > swapout_target_age)
1652 swapout_target_age = 0;
1653 }
1654 lck_mtx_unlock_always(c_list_lock);
1655 }
1656 #if CONFIG_PHANTOM_CACHE
1657 if (vm_phantom_cache_check_pressure())
1658 should_swap = TRUE;
1659 #endif
1660 if (swapout_target_age)
1661 should_swap = TRUE;
1662
1663 #if CONFIG_JETSAM
1664 if (should_swap || c_segment_pages_compressed > c_segment_pages_compressed_nearing_limit) {
1665
1666 if (vm_compressor_thrashing_detected == FALSE) {
1667 vm_compressor_thrashing_detected = TRUE;
1668
1669 if (swapout_target_age || c_segment_pages_compressed > c_segment_pages_compressed_nearing_limit) {
1670 memorystatus_kill_on_VM_thrashing(TRUE /* async */);
1671 compressor_thrashing_induced_jetsam++;
1672 } else {
1673 memorystatus_kill_on_FC_thrashing(TRUE /* async */);
1674 filecache_thrashing_induced_jetsam++;
1675 }
1676 }
1677 /*
1678 * let the jetsam take precedence over
1679 * any major compactions we might have
1680 * been able to do... otherwise we run
1681 * the risk of doing major compactions
1682 * on segments we're about to free up
1683 * due to the jetsam activity.
1684 */
1685 should_swap = FALSE;
1686 }
1687
1688 #endif /* CONFIG_JETSAM */
1689
1690 if (should_swap == FALSE) {
1691 /*
1692 * vm_compressor_needs_to_major_compact returns true only if we're
1693 * about to run out of available compressor segments... in this
1694 * case, we absolutely need to run a major compaction even if
1695 * we've just kicked off a jetsam or we don't otherwise need to
1696 * swap... terminating objects releases
1697 * pages back to the uncompressed cache, but does not guarantee
1698 * that we will free up even a single compression segment
1699 */
1700 should_swap = vm_compressor_needs_to_major_compact();
1701 }
1702
1703 /*
1704 * returning TRUE when swap_supported == FALSE
1705 * will cause the major compaction engine to
1706 * run, but will not trigger any swapping...
1707 * segments that have been major compacted
1708 * will be moved to the majorcompact queue
1709 */
1710 return (should_swap);
1711 }
1712
1713 #if CONFIG_JETSAM
1714 /*
1715 * This function is called from the jetsam thread after killing something to
1716 * mitigate thrashing.
1717 *
1718 * We need to restart our thrashing detection heuristics since memory pressure
1719 * has potentially changed significantly, and we don't want to detect on old
1720 * data from before the jetsam.
1721 */
1722 void
1723 vm_thrashing_jetsam_done(void)
1724 {
1725 vm_compressor_thrashing_detected = FALSE;
1726
1727 /* Were we compressor-thrashing or filecache-thrashing? */
1728 if (swapout_target_age) {
1729 swapout_target_age = 0;
1730 compressor_need_sample_reset = TRUE;
1731 }
1732 #if CONFIG_PHANTOM_CACHE
1733 else {
1734 vm_phantom_cache_restart_sample();
1735 }
1736 #endif
1737 }
1738 #endif /* CONFIG_JETSAM */
1739
1740 uint32_t vm_wake_compactor_swapper_calls = 0;
1741 uint32_t vm_run_compactor_already_running = 0;
1742 uint32_t vm_run_compactor_empty_minor_q = 0;
1743 uint32_t vm_run_compactor_did_compact = 0;
1744 uint32_t vm_run_compactor_waited = 0;
1745
1746 void
1747 vm_run_compactor(void)
1748 {
1749 if (c_segment_count == 0)
1750 return;
1751
1752 lck_mtx_lock_spin_always(c_list_lock);
1753
1754 if (c_minor_count == 0) {
1755 vm_run_compactor_empty_minor_q++;
1756
1757 lck_mtx_unlock_always(c_list_lock);
1758 return;
1759 }
1760 if (compaction_swapper_running) {
1761
1762 if (vm_restricted_to_single_processor == FALSE) {
1763 vm_run_compactor_already_running++;
1764
1765 lck_mtx_unlock_always(c_list_lock);
1766 return;
1767 }
1768 vm_run_compactor_waited++;
1769
1770 assert_wait((event_t)&compaction_swapper_running, THREAD_UNINT);
1771
1772 lck_mtx_unlock_always(c_list_lock);
1773
1774 thread_block(THREAD_CONTINUE_NULL);
1775
1776 return;
1777 }
1778 vm_run_compactor_did_compact++;
1779
1780 fastwake_warmup = FALSE;
1781 compaction_swapper_running = 1;
1782
1783 vm_compressor_do_delayed_compactions(FALSE);
1784
1785 compaction_swapper_running = 0;
1786
1787 lck_mtx_unlock_always(c_list_lock);
1788
1789 thread_wakeup((event_t)&compaction_swapper_running);
1790 }
1791
1792
1793 void
1794 vm_wake_compactor_swapper(void)
1795 {
1796 if (compaction_swapper_running || compaction_swapper_awakened || c_segment_count == 0)
1797 return;
1798
1799 if (c_minor_count || vm_compressor_needs_to_major_compact()) {
1800
1801 lck_mtx_lock_spin_always(c_list_lock);
1802
1803 fastwake_warmup = FALSE;
1804
1805 if (compaction_swapper_running == 0 && compaction_swapper_awakened == 0) {
1806
1807 vm_wake_compactor_swapper_calls++;
1808
1809 compaction_swapper_awakened = 1;
1810 thread_wakeup((event_t)&c_compressor_swap_trigger);
1811 }
1812 lck_mtx_unlock_always(c_list_lock);
1813 }
1814 }
1815
1816
1817 void
1818 vm_consider_swapping()
1819 {
1820 c_segment_t c_seg, c_seg_next;
1821 clock_sec_t now;
1822 clock_nsec_t nsec;
1823
1824 assert(VM_CONFIG_SWAP_IS_PRESENT);
1825
1826 lck_mtx_lock_spin_always(c_list_lock);
1827
1828 compaction_swapper_abort = 1;
1829
1830 while (compaction_swapper_running) {
1831 assert_wait((event_t)&compaction_swapper_running, THREAD_UNINT);
1832
1833 lck_mtx_unlock_always(c_list_lock);
1834
1835 thread_block(THREAD_CONTINUE_NULL);
1836
1837 lck_mtx_lock_spin_always(c_list_lock);
1838 }
1839 compaction_swapper_abort = 0;
1840 compaction_swapper_running = 1;
1841
1842 vm_swapout_ripe_segments = TRUE;
1843
1844 if (!queue_empty(&c_major_list_head)) {
1845
1846 clock_get_system_nanotime(&now, &nsec);
1847
1848 c_seg = (c_segment_t)queue_first(&c_major_list_head);
1849
1850 while (!queue_end(&c_major_list_head, (queue_entry_t)c_seg)) {
1851
1852 if (c_overage_swapped_count >= c_overage_swapped_limit)
1853 break;
1854
1855 c_seg_next = (c_segment_t) queue_next(&c_seg->c_age_list);
1856
1857 if ((now - c_seg->c_creation_ts) >= vm_ripe_target_age) {
1858
1859 lck_mtx_lock_spin_always(&c_seg->c_lock);
1860
1861 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
1862
1863 lck_mtx_unlock_always(&c_seg->c_lock);
1864 }
1865 c_seg = c_seg_next;
1866 }
1867 }
1868 vm_compressor_compact_and_swap(FALSE);
1869
1870 compaction_swapper_running = 0;
1871
1872 vm_swapout_ripe_segments = FALSE;
1873
1874 lck_mtx_unlock_always(c_list_lock);
1875
1876 thread_wakeup((event_t)&compaction_swapper_running);
1877 }
1878
1879
1880 void
1881 vm_consider_waking_compactor_swapper(void)
1882 {
1883 boolean_t need_wakeup = FALSE;
1884
1885 if (c_segment_count == 0)
1886 return;
1887
1888 if (compaction_swapper_running || compaction_swapper_awakened)
1889 return;
1890
1891 if (!compaction_swapper_inited && !compaction_swapper_init_now) {
1892 compaction_swapper_init_now = 1;
1893 need_wakeup = TRUE;
1894 }
1895
1896 if (c_minor_count && (COMPRESSOR_NEEDS_TO_MINOR_COMPACT())) {
1897
1898 need_wakeup = TRUE;
1899
1900 } else if (compressor_needs_to_swap()) {
1901
1902 need_wakeup = TRUE;
1903
1904 } else if (c_minor_count) {
1905 uint64_t total_bytes;
1906
1907 total_bytes = compressor_object->resident_page_count * PAGE_SIZE_64;
1908
1909 if ((total_bytes - compressor_bytes_used) > total_bytes / 10)
1910 need_wakeup = TRUE;
1911 }
1912 if (need_wakeup == TRUE) {
1913
1914 lck_mtx_lock_spin_always(c_list_lock);
1915
1916 fastwake_warmup = FALSE;
1917
1918 if (compaction_swapper_running == 0 && compaction_swapper_awakened == 0) {
1919 memoryshot(VM_WAKEUP_COMPACTOR_SWAPPER, DBG_FUNC_NONE);
1920
1921 compaction_swapper_awakened = 1;
1922 thread_wakeup((event_t)&c_compressor_swap_trigger);
1923 }
1924 lck_mtx_unlock_always(c_list_lock);
1925 }
1926 }
1927
1928
1929 #define C_SWAPOUT_LIMIT 4
1930 #define DELAYED_COMPACTIONS_PER_PASS 30
1931
1932 void
1933 vm_compressor_do_delayed_compactions(boolean_t flush_all)
1934 {
1935 c_segment_t c_seg;
1936 int number_compacted = 0;
1937 boolean_t needs_to_swap = FALSE;
1938
1939
1940 LCK_MTX_ASSERT(c_list_lock, LCK_MTX_ASSERT_OWNED);
1941
1942 while (!queue_empty(&c_minor_list_head) && needs_to_swap == FALSE) {
1943
1944 c_seg = (c_segment_t)queue_first(&c_minor_list_head);
1945
1946 lck_mtx_lock_spin_always(&c_seg->c_lock);
1947
1948 if (c_seg->c_busy) {
1949
1950 lck_mtx_unlock_always(c_list_lock);
1951 c_seg_wait_on_busy(c_seg);
1952 lck_mtx_lock_spin_always(c_list_lock);
1953
1954 continue;
1955 }
1956 C_SEG_BUSY(c_seg);
1957
1958 c_seg_do_minor_compaction_and_unlock(c_seg, TRUE, FALSE, TRUE);
1959
1960 if (VM_CONFIG_SWAP_IS_ACTIVE && (number_compacted++ > DELAYED_COMPACTIONS_PER_PASS)) {
1961
1962 if ((flush_all == TRUE || compressor_needs_to_swap() == TRUE) && c_swapout_count < C_SWAPOUT_LIMIT)
1963 needs_to_swap = TRUE;
1964
1965 number_compacted = 0;
1966 }
1967 lck_mtx_lock_spin_always(c_list_lock);
1968 }
1969 }
1970
1971
1972 #define C_SEGMENT_SWAPPEDIN_AGE_LIMIT 10
1973
1974 static void
1975 vm_compressor_age_swapped_in_segments(boolean_t flush_all)
1976 {
1977 c_segment_t c_seg;
1978 clock_sec_t now;
1979 clock_nsec_t nsec;
1980
1981 clock_get_system_nanotime(&now, &nsec);
1982
1983 while (!queue_empty(&c_swappedin_list_head)) {
1984
1985 c_seg = (c_segment_t)queue_first(&c_swappedin_list_head);
1986
1987 if (flush_all == FALSE && (now - c_seg->c_swappedin_ts) < C_SEGMENT_SWAPPEDIN_AGE_LIMIT)
1988 break;
1989
1990 lck_mtx_lock_spin_always(&c_seg->c_lock);
1991
1992 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
1993
1994 lck_mtx_unlock_always(&c_seg->c_lock);
1995 }
1996 }
1997
1998
1999 extern int vm_num_swap_files;
2000 extern int vm_num_pinned_swap_files;
2001 extern int vm_swappin_enabled;
2002
2003 extern unsigned int vm_swapfile_total_segs_used;
2004 extern unsigned int vm_swapfile_total_segs_alloced;
2005
2006
2007 void
2008 vm_compressor_flush(void)
2009 {
2010 uint64_t vm_swap_put_failures_at_start;
2011 wait_result_t wait_result = 0;
2012 AbsoluteTime startTime, endTime;
2013 clock_sec_t now_sec;
2014 clock_nsec_t now_nsec;
2015 uint64_t nsec;
2016
2017 HIBLOG("vm_compressor_flush - starting\n");
2018
2019 clock_get_uptime(&startTime);
2020
2021 lck_mtx_lock_spin_always(c_list_lock);
2022
2023 fastwake_warmup = FALSE;
2024 compaction_swapper_abort = 1;
2025
2026 while (compaction_swapper_running) {
2027 assert_wait((event_t)&compaction_swapper_running, THREAD_UNINT);
2028
2029 lck_mtx_unlock_always(c_list_lock);
2030
2031 thread_block(THREAD_CONTINUE_NULL);
2032
2033 lck_mtx_lock_spin_always(c_list_lock);
2034 }
2035 compaction_swapper_abort = 0;
2036 compaction_swapper_running = 1;
2037
2038 hibernate_flushing = TRUE;
2039 hibernate_no_swapspace = FALSE;
2040 c_generation_id_flush_barrier = c_generation_id + 1000;
2041
2042 clock_get_system_nanotime(&now_sec, &now_nsec);
2043 hibernate_flushing_deadline = now_sec + HIBERNATE_FLUSHING_SECS_TO_COMPLETE;
2044
2045 vm_swap_put_failures_at_start = vm_swap_put_failures;
2046
2047 vm_compressor_compact_and_swap(TRUE);
2048
2049 while (!queue_empty(&c_swapout_list_head)) {
2050
2051 assert_wait_timeout((event_t) &compaction_swapper_running, THREAD_INTERRUPTIBLE, 5000, 1000*NSEC_PER_USEC);
2052
2053 lck_mtx_unlock_always(c_list_lock);
2054
2055 wait_result = thread_block(THREAD_CONTINUE_NULL);
2056
2057 lck_mtx_lock_spin_always(c_list_lock);
2058
2059 if (wait_result == THREAD_TIMED_OUT)
2060 break;
2061 }
2062 hibernate_flushing = FALSE;
2063 compaction_swapper_running = 0;
2064
2065 if (vm_swap_put_failures > vm_swap_put_failures_at_start)
2066 HIBLOG("vm_compressor_flush failed to clean %llu segments - vm_page_compressor_count(%d)\n",
2067 vm_swap_put_failures - vm_swap_put_failures_at_start, VM_PAGE_COMPRESSOR_COUNT);
2068
2069 lck_mtx_unlock_always(c_list_lock);
2070
2071 thread_wakeup((event_t)&compaction_swapper_running);
2072
2073 clock_get_uptime(&endTime);
2074 SUB_ABSOLUTETIME(&endTime, &startTime);
2075 absolutetime_to_nanoseconds(endTime, &nsec);
2076
2077 HIBLOG("vm_compressor_flush completed - took %qd msecs - vm_num_swap_files = %d, vm_num_pinned_swap_files = %d, vm_swappin_enabled = %d\n",
2078 nsec / 1000000ULL, vm_num_swap_files, vm_num_pinned_swap_files, vm_swappin_enabled);
2079 }
2080
2081
2082 int compaction_swap_trigger_thread_awakened = 0;
2083
2084 static void
2085 vm_compressor_swap_trigger_thread(void)
2086 {
2087 current_thread()->options |= TH_OPT_VMPRIV;
2088
2089 /*
2090 * compaction_swapper_init_now is set when the first call to
2091 * vm_consider_waking_compactor_swapper is made from
2092 * vm_pageout_scan... since this function is called upon
2093 * thread creation, we want to make sure to delay adjusting
2094 * the tuneables until we are awakened via vm_pageout_scan
2095 * so that we are at a point where the vm_swapfile_open will
2096 * be operating on the correct directory (in case the default
2097 * of /var/vm/ is overridden by the dymanic_pager
2098 */
2099 if (compaction_swapper_init_now) {
2100 vm_compaction_swapper_do_init();
2101
2102 if (vm_restricted_to_single_processor == TRUE)
2103 thread_vm_bind_group_add();
2104
2105 compaction_swapper_init_now = 0;
2106 }
2107 lck_mtx_lock_spin_always(c_list_lock);
2108
2109 compaction_swap_trigger_thread_awakened++;
2110 compaction_swapper_awakened = 0;
2111
2112 if (compaction_swapper_running == 0) {
2113
2114 compaction_swapper_running = 1;
2115
2116 vm_compressor_compact_and_swap(FALSE);
2117
2118 compaction_swapper_running = 0;
2119 }
2120 assert_wait((event_t)&c_compressor_swap_trigger, THREAD_UNINT);
2121
2122 if (compaction_swapper_running == 0)
2123 thread_wakeup((event_t)&compaction_swapper_running);
2124
2125 lck_mtx_unlock_always(c_list_lock);
2126
2127 thread_block((thread_continue_t)vm_compressor_swap_trigger_thread);
2128
2129 /* NOTREACHED */
2130 }
2131
2132
2133 void
2134 vm_compressor_record_warmup_start(void)
2135 {
2136 c_segment_t c_seg;
2137
2138 lck_mtx_lock_spin_always(c_list_lock);
2139
2140 if (first_c_segment_to_warm_generation_id == 0) {
2141 if (!queue_empty(&c_age_list_head)) {
2142
2143 c_seg = (c_segment_t)queue_last(&c_age_list_head);
2144
2145 first_c_segment_to_warm_generation_id = c_seg->c_generation_id;
2146 } else
2147 first_c_segment_to_warm_generation_id = 0;
2148
2149 fastwake_recording_in_progress = TRUE;
2150 }
2151 lck_mtx_unlock_always(c_list_lock);
2152 }
2153
2154
2155 void
2156 vm_compressor_record_warmup_end(void)
2157 {
2158 c_segment_t c_seg;
2159
2160 lck_mtx_lock_spin_always(c_list_lock);
2161
2162 if (fastwake_recording_in_progress == TRUE) {
2163
2164 if (!queue_empty(&c_age_list_head)) {
2165
2166 c_seg = (c_segment_t)queue_last(&c_age_list_head);
2167
2168 last_c_segment_to_warm_generation_id = c_seg->c_generation_id;
2169 } else
2170 last_c_segment_to_warm_generation_id = first_c_segment_to_warm_generation_id;
2171
2172 fastwake_recording_in_progress = FALSE;
2173
2174 HIBLOG("vm_compressor_record_warmup (%qd - %qd)\n", first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id);
2175 }
2176 lck_mtx_unlock_always(c_list_lock);
2177 }
2178
2179
2180 #define DELAY_TRIM_ON_WAKE_SECS 25
2181
2182 void
2183 vm_compressor_delay_trim(void)
2184 {
2185 clock_sec_t sec;
2186 clock_nsec_t nsec;
2187
2188 clock_get_system_nanotime(&sec, &nsec);
2189 dont_trim_until_ts = sec + DELAY_TRIM_ON_WAKE_SECS;
2190 }
2191
2192
2193 void
2194 vm_compressor_do_warmup(void)
2195 {
2196 lck_mtx_lock_spin_always(c_list_lock);
2197
2198 if (first_c_segment_to_warm_generation_id == last_c_segment_to_warm_generation_id) {
2199 first_c_segment_to_warm_generation_id = last_c_segment_to_warm_generation_id = 0;
2200
2201 lck_mtx_unlock_always(c_list_lock);
2202 return;
2203 }
2204
2205 if (compaction_swapper_running == 0 && compaction_swapper_awakened == 0) {
2206
2207 fastwake_warmup = TRUE;
2208
2209 compaction_swapper_awakened = 1;
2210 thread_wakeup((event_t)&c_compressor_swap_trigger);
2211 }
2212 lck_mtx_unlock_always(c_list_lock);
2213 }
2214
2215
2216 void
2217 do_fastwake_warmup(void)
2218 {
2219 c_segment_t c_seg = NULL;
2220 AbsoluteTime startTime, endTime;
2221 uint64_t nsec;
2222
2223
2224 HIBLOG("vm_compressor_fastwake_warmup (%qd - %qd) - starting\n", first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id);
2225
2226 clock_get_uptime(&startTime);
2227
2228 lck_mtx_unlock_always(c_list_lock);
2229
2230 proc_set_thread_policy(current_thread(),
2231 TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER2);
2232
2233 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2234
2235 lck_mtx_lock_spin_always(c_list_lock);
2236
2237 while (!queue_empty(&c_swappedout_list_head) && fastwake_warmup == TRUE) {
2238
2239 c_seg = (c_segment_t) queue_first(&c_swappedout_list_head);
2240
2241 if (c_seg->c_generation_id < first_c_segment_to_warm_generation_id ||
2242 c_seg->c_generation_id > last_c_segment_to_warm_generation_id)
2243 break;
2244
2245 if (vm_page_free_count < (AVAILABLE_MEMORY / 4))
2246 break;
2247
2248 lck_mtx_lock_spin_always(&c_seg->c_lock);
2249 lck_mtx_unlock_always(c_list_lock);
2250
2251 if (c_seg->c_busy) {
2252 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2253 c_seg_wait_on_busy(c_seg);
2254 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2255 } else {
2256 if (c_seg_swapin(c_seg, TRUE, FALSE) == 0)
2257 lck_mtx_unlock_always(&c_seg->c_lock);
2258 c_segment_warmup_count++;
2259
2260 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2261 vm_pageout_io_throttle();
2262 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2263 }
2264 lck_mtx_lock_spin_always(c_list_lock);
2265 }
2266 lck_mtx_unlock_always(c_list_lock);
2267
2268 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2269
2270 proc_set_thread_policy(current_thread(),
2271 TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER0);
2272
2273 clock_get_uptime(&endTime);
2274 SUB_ABSOLUTETIME(&endTime, &startTime);
2275 absolutetime_to_nanoseconds(endTime, &nsec);
2276
2277 HIBLOG("vm_compressor_fastwake_warmup completed - took %qd msecs\n", nsec / 1000000ULL);
2278
2279 lck_mtx_lock_spin_always(c_list_lock);
2280
2281 first_c_segment_to_warm_generation_id = last_c_segment_to_warm_generation_id = 0;
2282 }
2283
2284
2285 void
2286 vm_compressor_compact_and_swap(boolean_t flush_all)
2287 {
2288 c_segment_t c_seg, c_seg_next;
2289 boolean_t keep_compacting;
2290 clock_sec_t now;
2291 clock_nsec_t nsec;
2292
2293
2294 if (fastwake_warmup == TRUE) {
2295 uint64_t starting_warmup_count;
2296
2297 starting_warmup_count = c_segment_warmup_count;
2298
2299 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 11) | DBG_FUNC_START, c_segment_warmup_count,
2300 first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id, 0, 0);
2301 do_fastwake_warmup();
2302 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 11) | DBG_FUNC_END, c_segment_warmup_count, c_segment_warmup_count - starting_warmup_count, 0, 0, 0);
2303
2304 fastwake_warmup = FALSE;
2305 }
2306
2307 /*
2308 * it's possible for the c_age_list_head to be empty if we
2309 * hit our limits for growing the compressor pool and we subsequently
2310 * hibernated... on the next hibernation we could see the queue as
2311 * empty and not proceeed even though we have a bunch of segments on
2312 * the swapped in queue that need to be dealt with.
2313 */
2314 vm_compressor_do_delayed_compactions(flush_all);
2315
2316 vm_compressor_age_swapped_in_segments(flush_all);
2317
2318 /*
2319 * we only need to grab the timestamp once per
2320 * invocation of this function since the
2321 * timescale we're interested in is measured
2322 * in days
2323 */
2324 clock_get_system_nanotime(&now, &nsec);
2325
2326 while (!queue_empty(&c_age_list_head) && compaction_swapper_abort == 0) {
2327
2328 if (hibernate_flushing == TRUE) {
2329 clock_sec_t sec;
2330
2331 if (hibernate_should_abort()) {
2332 HIBLOG("vm_compressor_flush - hibernate_should_abort returned TRUE\n");
2333 break;
2334 }
2335 if (hibernate_no_swapspace == TRUE) {
2336 HIBLOG("vm_compressor_flush - out of swap space\n");
2337 break;
2338 }
2339 if (vm_swap_files_pinned() == FALSE) {
2340 HIBLOG("vm_compressor_flush - unpinned swap files\n");
2341 break;
2342 }
2343 if (hibernate_in_progress_with_pinned_swap == TRUE &&
2344 (vm_swapfile_total_segs_alloced == vm_swapfile_total_segs_used)) {
2345 HIBLOG("vm_compressor_flush - out of pinned swap space\n");
2346 break;
2347 }
2348 clock_get_system_nanotime(&sec, &nsec);
2349
2350 if (sec > hibernate_flushing_deadline) {
2351 HIBLOG("vm_compressor_flush - failed to finish before deadline\n");
2352 break;
2353 }
2354 }
2355 if (c_swapout_count >= C_SWAPOUT_LIMIT) {
2356
2357 assert_wait_timeout((event_t) &compaction_swapper_running, THREAD_INTERRUPTIBLE, 100, 1000*NSEC_PER_USEC);
2358
2359 lck_mtx_unlock_always(c_list_lock);
2360
2361 thread_block(THREAD_CONTINUE_NULL);
2362
2363 lck_mtx_lock_spin_always(c_list_lock);
2364 }
2365 /*
2366 * Minor compactions
2367 */
2368 vm_compressor_do_delayed_compactions(flush_all);
2369
2370 vm_compressor_age_swapped_in_segments(flush_all);
2371
2372 if (c_swapout_count >= C_SWAPOUT_LIMIT) {
2373 /*
2374 * we timed out on the above thread_block
2375 * let's loop around and try again
2376 * the timeout allows us to continue
2377 * to do minor compactions to make
2378 * more memory available
2379 */
2380 continue;
2381 }
2382
2383 /*
2384 * Swap out segments?
2385 */
2386 if (flush_all == FALSE) {
2387 boolean_t needs_to_swap;
2388
2389 lck_mtx_unlock_always(c_list_lock);
2390
2391 needs_to_swap = compressor_needs_to_swap();
2392
2393 if (needs_to_swap == TRUE && vm_swap_low_on_space())
2394 vm_compressor_take_paging_space_action();
2395
2396 lck_mtx_lock_spin_always(c_list_lock);
2397
2398 if (needs_to_swap == FALSE)
2399 break;
2400 }
2401 if (queue_empty(&c_age_list_head))
2402 break;
2403 c_seg = (c_segment_t) queue_first(&c_age_list_head);
2404
2405 assert(c_seg->c_state == C_ON_AGE_Q);
2406
2407 if (flush_all == TRUE && c_seg->c_generation_id > c_generation_id_flush_barrier)
2408 break;
2409
2410 lck_mtx_lock_spin_always(&c_seg->c_lock);
2411
2412 if (c_seg->c_busy) {
2413
2414 lck_mtx_unlock_always(c_list_lock);
2415 c_seg_wait_on_busy(c_seg);
2416 lck_mtx_lock_spin_always(c_list_lock);
2417
2418 continue;
2419 }
2420 C_SEG_BUSY(c_seg);
2421
2422 if (c_seg_do_minor_compaction_and_unlock(c_seg, FALSE, TRUE, TRUE)) {
2423 /*
2424 * found an empty c_segment and freed it
2425 * so go grab the next guy in the queue
2426 */
2427 c_seg_major_compact_stats.count_of_freed_segs++;
2428 continue;
2429 }
2430 /*
2431 * Major compaction
2432 */
2433 keep_compacting = TRUE;
2434
2435 while (keep_compacting == TRUE) {
2436
2437 assert(c_seg->c_busy);
2438
2439 /* look for another segment to consolidate */
2440
2441 c_seg_next = (c_segment_t) queue_next(&c_seg->c_age_list);
2442
2443 if (queue_end(&c_age_list_head, (queue_entry_t)c_seg_next))
2444 break;
2445
2446 assert(c_seg_next->c_state == C_ON_AGE_Q);
2447
2448 if (c_seg_major_compact_ok(c_seg, c_seg_next) == FALSE)
2449 break;
2450
2451 lck_mtx_lock_spin_always(&c_seg_next->c_lock);
2452
2453 if (c_seg_next->c_busy) {
2454
2455 lck_mtx_unlock_always(c_list_lock);
2456 c_seg_wait_on_busy(c_seg_next);
2457 lck_mtx_lock_spin_always(c_list_lock);
2458
2459 continue;
2460 }
2461 /* grab that segment */
2462 C_SEG_BUSY(c_seg_next);
2463
2464 if (c_seg_do_minor_compaction_and_unlock(c_seg_next, FALSE, TRUE, TRUE)) {
2465 /*
2466 * found an empty c_segment and freed it
2467 * so we can't continue to use c_seg_next
2468 */
2469 c_seg_major_compact_stats.count_of_freed_segs++;
2470 continue;
2471 }
2472
2473 /* unlock the list ... */
2474 lck_mtx_unlock_always(c_list_lock);
2475
2476 /* do the major compaction */
2477
2478 keep_compacting = c_seg_major_compact(c_seg, c_seg_next);
2479
2480 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2481
2482 lck_mtx_lock_spin_always(&c_seg_next->c_lock);
2483 /*
2484 * run a minor compaction on the donor segment
2485 * since we pulled at least some of it's
2486 * data into our target... if we've emptied
2487 * it, now is a good time to free it which
2488 * c_seg_minor_compaction_and_unlock also takes care of
2489 *
2490 * by passing TRUE, we ask for c_busy to be cleared
2491 * and c_wanted to be taken care of
2492 */
2493 if (c_seg_minor_compaction_and_unlock(c_seg_next, TRUE))
2494 c_seg_major_compact_stats.count_of_freed_segs++;
2495
2496 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2497
2498 /* relock the list */
2499 lck_mtx_lock_spin_always(c_list_lock);
2500
2501 } /* major compaction */
2502
2503 lck_mtx_lock_spin_always(&c_seg->c_lock);
2504
2505 assert(c_seg->c_busy);
2506 assert(!c_seg->c_on_minorcompact_q);
2507
2508 if (VM_CONFIG_SWAP_IS_ACTIVE) {
2509 /*
2510 * This mode of putting a generic c_seg on the swapout list is
2511 * only supported when we have general swapping enabled
2512 */
2513 c_seg_switch_state(c_seg, C_ON_SWAPOUT_Q, FALSE);
2514 } else {
2515 if ((vm_swapout_ripe_segments == TRUE && c_overage_swapped_count < c_overage_swapped_limit)) {
2516
2517 assert(VM_CONFIG_SWAP_IS_PRESENT);
2518 /*
2519 * we are running compressor sweeps with swap-behind
2520 * make sure the c_seg has aged enough before swapping it
2521 * out...
2522 */
2523 if ((now - c_seg->c_creation_ts) >= vm_ripe_target_age) {
2524 c_seg->c_overage_swap = TRUE;
2525 c_overage_swapped_count++;
2526 c_seg_switch_state(c_seg, C_ON_SWAPOUT_Q, FALSE);
2527 }
2528 }
2529 }
2530 if (c_seg->c_state == C_ON_AGE_Q) {
2531 /*
2532 * this c_seg didn't get moved to the swapout queue
2533 * so we need to move it out of the way...
2534 * we just did a major compaction on it so put it
2535 * on that queue
2536 */
2537 c_seg_switch_state(c_seg, C_ON_MAJORCOMPACT_Q, FALSE);
2538 } else {
2539 c_seg_major_compact_stats.wasted_space_in_swapouts += C_SEG_BUFSIZE - c_seg->c_bytes_used;
2540 c_seg_major_compact_stats.count_of_swapouts++;
2541 }
2542 C_SEG_WAKEUP_DONE(c_seg);
2543
2544 lck_mtx_unlock_always(&c_seg->c_lock);
2545
2546 if (c_swapout_count) {
2547 lck_mtx_unlock_always(c_list_lock);
2548
2549 thread_wakeup((event_t)&c_swapout_list_head);
2550
2551 lck_mtx_lock_spin_always(c_list_lock);
2552 }
2553 }
2554 }
2555
2556
2557 static c_segment_t
2558 c_seg_allocate(c_segment_t *current_chead)
2559 {
2560 c_segment_t c_seg;
2561 int min_needed;
2562 int size_to_populate;
2563
2564 if (vm_compressor_low_on_space())
2565 vm_compressor_take_paging_space_action();
2566
2567 if ( (c_seg = *current_chead) == NULL ) {
2568 uint32_t c_segno;
2569
2570 lck_mtx_lock_spin_always(c_list_lock);
2571
2572 while (c_segments_busy == TRUE) {
2573 assert_wait((event_t) (&c_segments_busy), THREAD_UNINT);
2574
2575 lck_mtx_unlock_always(c_list_lock);
2576
2577 thread_block(THREAD_CONTINUE_NULL);
2578
2579 lck_mtx_lock_spin_always(c_list_lock);
2580 }
2581 if (c_free_segno_head == (uint32_t)-1) {
2582 uint32_t c_segments_available_new;
2583
2584 if (c_segments_available >= c_segments_limit || c_segment_pages_compressed >= c_segment_pages_compressed_limit) {
2585 lck_mtx_unlock_always(c_list_lock);
2586
2587 return (NULL);
2588 }
2589 c_segments_busy = TRUE;
2590 lck_mtx_unlock_always(c_list_lock);
2591
2592 kernel_memory_populate(compressor_map, (vm_offset_t)c_segments_next_page,
2593 PAGE_SIZE, KMA_KOBJECT, VM_KERN_MEMORY_COMPRESSOR);
2594 c_segments_next_page += PAGE_SIZE;
2595
2596 c_segments_available_new = c_segments_available + C_SEGMENTS_PER_PAGE;
2597
2598 if (c_segments_available_new > c_segments_limit)
2599 c_segments_available_new = c_segments_limit;
2600
2601 for (c_segno = c_segments_available + 1; c_segno < c_segments_available_new; c_segno++)
2602 c_segments[c_segno - 1].c_segno = c_segno;
2603
2604 lck_mtx_lock_spin_always(c_list_lock);
2605
2606 c_segments[c_segno - 1].c_segno = c_free_segno_head;
2607 c_free_segno_head = c_segments_available;
2608 c_segments_available = c_segments_available_new;
2609
2610 c_segments_busy = FALSE;
2611 thread_wakeup((event_t) (&c_segments_busy));
2612 }
2613 c_segno = c_free_segno_head;
2614 assert(c_segno >= 0 && c_segno < c_segments_limit);
2615
2616 c_free_segno_head = (uint32_t)c_segments[c_segno].c_segno;
2617
2618 /*
2619 * do the rest of the bookkeeping now while we're still behind
2620 * the list lock and grab our generation id now into a local
2621 * so that we can install it once we have the c_seg allocated
2622 */
2623 c_segment_count++;
2624 if (c_segment_count > c_segment_count_max)
2625 c_segment_count_max = c_segment_count;
2626
2627 lck_mtx_unlock_always(c_list_lock);
2628
2629 c_seg = (c_segment_t)zalloc(compressor_segment_zone);
2630 bzero((char *)c_seg, sizeof(struct c_segment));
2631
2632 c_seg->c_store.c_buffer = (int32_t *)C_SEG_BUFFER_ADDRESS(c_segno);
2633
2634 lck_mtx_init(&c_seg->c_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
2635
2636 c_seg->c_state = C_IS_EMPTY;
2637 c_seg->c_firstemptyslot = C_SLOT_MAX_INDEX;
2638 c_seg->c_mysegno = c_segno;
2639
2640 lck_mtx_lock_spin_always(c_list_lock);
2641 c_empty_count++;
2642 c_seg_switch_state(c_seg, C_IS_FILLING, FALSE);
2643 c_segments[c_segno].c_seg = c_seg;
2644 assert(c_segments[c_segno].c_segno > c_segments_available);
2645 lck_mtx_unlock_always(c_list_lock);
2646
2647 *current_chead = c_seg;
2648
2649 #if DEVELOPMENT || DEBUG
2650 C_SEG_MAKE_WRITEABLE(c_seg);
2651 #endif
2652
2653 }
2654 c_seg_alloc_nextslot(c_seg);
2655
2656 size_to_populate = C_SEG_ALLOCSIZE - C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset);
2657
2658 if (size_to_populate) {
2659
2660 min_needed = PAGE_SIZE + (C_SEG_ALLOCSIZE - C_SEG_BUFSIZE);
2661
2662 if (C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset - c_seg->c_nextoffset) < (unsigned) min_needed) {
2663
2664 if (size_to_populate > C_SEG_MAX_POPULATE_SIZE)
2665 size_to_populate = C_SEG_MAX_POPULATE_SIZE;
2666
2667 kernel_memory_populate(compressor_map,
2668 (vm_offset_t) &c_seg->c_store.c_buffer[c_seg->c_populated_offset],
2669 size_to_populate,
2670 KMA_COMPRESSOR,
2671 VM_KERN_MEMORY_COMPRESSOR);
2672 } else
2673 size_to_populate = 0;
2674 }
2675 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2676
2677 lck_mtx_lock_spin_always(&c_seg->c_lock);
2678
2679 if (size_to_populate)
2680 c_seg->c_populated_offset += C_SEG_BYTES_TO_OFFSET(size_to_populate);
2681
2682 return (c_seg);
2683 }
2684
2685
2686 static void
2687 c_current_seg_filled(c_segment_t c_seg, c_segment_t *current_chead)
2688 {
2689 uint32_t unused_bytes;
2690 uint32_t offset_to_depopulate;
2691 int new_state = C_ON_AGE_Q;
2692 clock_sec_t sec;
2693 clock_nsec_t nsec;
2694
2695 unused_bytes = trunc_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset - c_seg->c_nextoffset));
2696
2697 if (unused_bytes) {
2698
2699 offset_to_depopulate = C_SEG_BYTES_TO_OFFSET(round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_nextoffset)));
2700
2701 /*
2702 * release the extra physical page(s) at the end of the segment
2703 */
2704 lck_mtx_unlock_always(&c_seg->c_lock);
2705
2706 kernel_memory_depopulate(
2707 compressor_map,
2708 (vm_offset_t) &c_seg->c_store.c_buffer[offset_to_depopulate],
2709 unused_bytes,
2710 KMA_COMPRESSOR);
2711
2712 lck_mtx_lock_spin_always(&c_seg->c_lock);
2713
2714 c_seg->c_populated_offset = offset_to_depopulate;
2715 }
2716 assert(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset) <= C_SEG_BUFSIZE);
2717
2718 #if DEVELOPMENT || DEBUG
2719 {
2720 boolean_t c_seg_was_busy = FALSE;
2721
2722 if ( !c_seg->c_busy)
2723 C_SEG_BUSY(c_seg);
2724 else
2725 c_seg_was_busy = TRUE;
2726
2727 lck_mtx_unlock_always(&c_seg->c_lock);
2728
2729 C_SEG_WRITE_PROTECT(c_seg);
2730
2731 lck_mtx_lock_spin_always(&c_seg->c_lock);
2732
2733 if (c_seg_was_busy == FALSE)
2734 C_SEG_WAKEUP_DONE(c_seg);
2735 }
2736 #endif
2737
2738 #if CONFIG_FREEZE
2739 if (current_chead == (c_segment_t*)&freezer_chead &&
2740 VM_CONFIG_SWAP_IS_PRESENT &&
2741 VM_CONFIG_FREEZER_SWAP_IS_ACTIVE &&
2742 c_freezer_swapout_count < VM_MAX_FREEZER_CSEG_SWAP_COUNT) {
2743 new_state = C_ON_SWAPOUT_Q;
2744 }
2745 #endif /* CONFIG_FREEZE */
2746
2747 clock_get_system_nanotime(&sec, &nsec);
2748 c_seg->c_creation_ts = (uint32_t)sec;
2749
2750 lck_mtx_lock_spin_always(c_list_lock);
2751
2752 #if CONFIG_FREEZE
2753 if (c_seg->c_state == C_ON_SWAPOUT_Q)
2754 c_freezer_swapout_count++;
2755 #endif /* CONFIG_FREEZE */
2756
2757 c_seg->c_generation_id = c_generation_id++;
2758 c_seg_switch_state(c_seg, new_state, FALSE);
2759
2760 if (c_seg->c_state == C_ON_AGE_Q && C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE)
2761 c_seg_need_delayed_compaction(c_seg, TRUE);
2762
2763 lck_mtx_unlock_always(c_list_lock);
2764
2765 #if CONFIG_FREEZE
2766 if (c_seg->c_state == C_ON_SWAPOUT_Q)
2767 thread_wakeup((event_t)&c_swapout_list_head);
2768 #endif /* CONFIG_FREEZE */
2769
2770 *current_chead = NULL;
2771 }
2772
2773
2774 /*
2775 * returns with c_seg locked
2776 */
2777 void
2778 c_seg_swapin_requeue(c_segment_t c_seg, boolean_t has_data, boolean_t minor_compact_ok, boolean_t age_on_swapin_q)
2779 {
2780 clock_sec_t sec;
2781 clock_nsec_t nsec;
2782
2783 clock_get_system_nanotime(&sec, &nsec);
2784
2785 lck_mtx_lock_spin_always(c_list_lock);
2786 lck_mtx_lock_spin_always(&c_seg->c_lock);
2787
2788 assert(c_seg->c_busy_swapping);
2789 assert(c_seg->c_busy);
2790
2791 c_seg->c_busy_swapping = 0;
2792
2793 if (c_seg->c_overage_swap == TRUE) {
2794 c_overage_swapped_count--;
2795 c_seg->c_overage_swap = FALSE;
2796 }
2797 if (has_data == TRUE) {
2798 if (age_on_swapin_q == TRUE)
2799 c_seg_switch_state(c_seg, C_ON_SWAPPEDIN_Q, FALSE);
2800 else
2801 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
2802
2803 if (minor_compact_ok == TRUE && !c_seg->c_on_minorcompact_q && C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE)
2804 c_seg_need_delayed_compaction(c_seg, TRUE);
2805 } else {
2806 c_seg->c_store.c_buffer = (int32_t*) NULL;
2807 c_seg->c_populated_offset = C_SEG_BYTES_TO_OFFSET(0);
2808
2809 c_seg_switch_state(c_seg, C_ON_BAD_Q, FALSE);
2810 }
2811 c_seg->c_swappedin_ts = (uint32_t)sec;
2812
2813 lck_mtx_unlock_always(c_list_lock);
2814 }
2815
2816
2817
2818 /*
2819 * c_seg has to be locked and is returned locked if the c_seg isn't freed
2820 * PAGE_REPLACMENT_DISALLOWED has to be TRUE on entry and is returned TRUE
2821 * c_seg_swapin returns 1 if the c_seg was freed, 0 otherwise
2822 */
2823
2824 int
2825 c_seg_swapin(c_segment_t c_seg, boolean_t force_minor_compaction, boolean_t age_on_swapin_q)
2826 {
2827 vm_offset_t addr = 0;
2828 uint32_t io_size = 0;
2829 uint64_t f_offset;
2830
2831 assert(C_SEG_IS_ONDISK(c_seg));
2832
2833 #if !CHECKSUM_THE_SWAP
2834 c_seg_trim_tail(c_seg);
2835 #endif
2836 io_size = round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset));
2837 f_offset = c_seg->c_store.c_swap_handle;
2838
2839 C_SEG_BUSY(c_seg);
2840 c_seg->c_busy_swapping = 1;
2841
2842 /*
2843 * This thread is likely going to block for I/O.
2844 * Make sure it is ready to run when the I/O completes because
2845 * it needs to clear the busy bit on the c_seg so that other
2846 * waiting threads can make progress too. To do that, boost
2847 * the rwlock_count so that the priority is boosted.
2848 */
2849 set_thread_rwlock_boost();
2850 lck_mtx_unlock_always(&c_seg->c_lock);
2851
2852 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2853
2854 addr = (vm_offset_t)C_SEG_BUFFER_ADDRESS(c_seg->c_mysegno);
2855 c_seg->c_store.c_buffer = (int32_t*) addr;
2856
2857 kernel_memory_populate(compressor_map, addr, io_size, KMA_COMPRESSOR, VM_KERN_MEMORY_COMPRESSOR);
2858
2859 if (vm_swap_get(c_seg, f_offset, io_size) != KERN_SUCCESS) {
2860 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2861
2862 c_seg->c_store.c_swap_handle = f_offset;
2863
2864 kernel_memory_depopulate(compressor_map, addr, io_size, KMA_COMPRESSOR);
2865
2866 c_seg_swapin_requeue(c_seg, FALSE, TRUE, age_on_swapin_q);
2867 } else {
2868 c_seg->c_store.c_buffer = (int32_t*) addr;
2869 #if ENCRYPTED_SWAP
2870 vm_swap_decrypt(c_seg);
2871 #endif /* ENCRYPTED_SWAP */
2872
2873 #if CHECKSUM_THE_SWAP
2874 if (c_seg->cseg_swap_size != io_size)
2875 panic("swapin size doesn't match swapout size");
2876
2877 if (c_seg->cseg_hash != hash_string((char*) c_seg->c_store.c_buffer, (int)io_size)) {
2878 panic("c_seg_swapin - Swap hash mismatch\n");
2879 }
2880 #endif /* CHECKSUM_THE_SWAP */
2881
2882 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2883
2884 c_seg_swapin_requeue(c_seg, TRUE, force_minor_compaction == TRUE ? FALSE : TRUE, age_on_swapin_q);
2885
2886 OSAddAtomic64(c_seg->c_bytes_used, &compressor_bytes_used);
2887
2888 if (force_minor_compaction == TRUE) {
2889 if (c_seg_minor_compaction_and_unlock(c_seg, FALSE)) {
2890 /*
2891 * Drop the rwlock_count so that the thread priority
2892 * is returned back to where it is supposed to be.
2893 */
2894 clear_thread_rwlock_boost();
2895 return (1);
2896 }
2897
2898 lck_mtx_lock_spin_always(&c_seg->c_lock);
2899 }
2900 }
2901 C_SEG_WAKEUP_DONE(c_seg);
2902
2903 /*
2904 * Drop the rwlock_count so that the thread priority
2905 * is returned back to where it is supposed to be.
2906 */
2907 clear_thread_rwlock_boost();
2908
2909 return (0);
2910 }
2911
2912
2913 static void
2914 c_segment_sv_hash_drop_ref(int hash_indx)
2915 {
2916 struct c_sv_hash_entry o_sv_he, n_sv_he;
2917
2918 while (1) {
2919
2920 o_sv_he.he_record = c_segment_sv_hash_table[hash_indx].he_record;
2921
2922 n_sv_he.he_ref = o_sv_he.he_ref - 1;
2923 n_sv_he.he_data = o_sv_he.he_data;
2924
2925 if (OSCompareAndSwap64((UInt64)o_sv_he.he_record, (UInt64)n_sv_he.he_record, (UInt64 *) &c_segment_sv_hash_table[hash_indx].he_record) == TRUE) {
2926 if (n_sv_he.he_ref == 0)
2927 OSAddAtomic(-1, &c_segment_svp_in_hash);
2928 break;
2929 }
2930 }
2931 }
2932
2933
2934 static int
2935 c_segment_sv_hash_insert(uint32_t data)
2936 {
2937 int hash_sindx;
2938 int misses;
2939 struct c_sv_hash_entry o_sv_he, n_sv_he;
2940 boolean_t got_ref = FALSE;
2941
2942 if (data == 0)
2943 OSAddAtomic(1, &c_segment_svp_zero_compressions);
2944 else
2945 OSAddAtomic(1, &c_segment_svp_nonzero_compressions);
2946
2947 hash_sindx = data & C_SV_HASH_MASK;
2948
2949 for (misses = 0; misses < C_SV_HASH_MAX_MISS; misses++)
2950 {
2951 o_sv_he.he_record = c_segment_sv_hash_table[hash_sindx].he_record;
2952
2953 while (o_sv_he.he_data == data || o_sv_he.he_ref == 0) {
2954 n_sv_he.he_ref = o_sv_he.he_ref + 1;
2955 n_sv_he.he_data = data;
2956
2957 if (OSCompareAndSwap64((UInt64)o_sv_he.he_record, (UInt64)n_sv_he.he_record, (UInt64 *) &c_segment_sv_hash_table[hash_sindx].he_record) == TRUE) {
2958 if (n_sv_he.he_ref == 1)
2959 OSAddAtomic(1, &c_segment_svp_in_hash);
2960 got_ref = TRUE;
2961 break;
2962 }
2963 o_sv_he.he_record = c_segment_sv_hash_table[hash_sindx].he_record;
2964 }
2965 if (got_ref == TRUE)
2966 break;
2967 hash_sindx++;
2968
2969 if (hash_sindx == C_SV_HASH_SIZE)
2970 hash_sindx = 0;
2971 }
2972 if (got_ref == FALSE)
2973 return(-1);
2974
2975 return (hash_sindx);
2976 }
2977
2978
2979 #if RECORD_THE_COMPRESSED_DATA
2980
2981 static void
2982 c_compressed_record_data(char *src, int c_size)
2983 {
2984 if ((c_compressed_record_cptr + c_size + 4) >= c_compressed_record_ebuf)
2985 panic("c_compressed_record_cptr >= c_compressed_record_ebuf");
2986
2987 *(int *)((void *)c_compressed_record_cptr) = c_size;
2988
2989 c_compressed_record_cptr += 4;
2990
2991 memcpy(c_compressed_record_cptr, src, c_size);
2992 c_compressed_record_cptr += c_size;
2993 }
2994 #endif
2995
2996
2997 static int
2998 c_compress_page(char *src, c_slot_mapping_t slot_ptr, c_segment_t *current_chead, char *scratch_buf)
2999 {
3000 int c_size;
3001 int c_rounded_size = 0;
3002 int max_csize;
3003 c_slot_t cs;
3004 c_segment_t c_seg;
3005
3006 KERNEL_DEBUG(0xe0400000 | DBG_FUNC_START, *current_chead, 0, 0, 0, 0);
3007 retry:
3008 if ((c_seg = c_seg_allocate(current_chead)) == NULL)
3009 return (1);
3010 /*
3011 * returns with c_seg lock held
3012 * and PAGE_REPLACEMENT_DISALLOWED(TRUE)...
3013 * c_nextslot has been allocated and
3014 * c_store.c_buffer populated
3015 */
3016 assert(c_seg->c_state == C_IS_FILLING);
3017
3018 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_seg->c_nextslot);
3019
3020 cs->c_packed_ptr = C_SLOT_PACK_PTR(slot_ptr);
3021 assert(slot_ptr == (c_slot_mapping_t)C_SLOT_UNPACK_PTR(cs));
3022
3023 cs->c_offset = c_seg->c_nextoffset;
3024
3025 max_csize = C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES((int32_t)cs->c_offset);
3026
3027 if (max_csize > PAGE_SIZE)
3028 max_csize = PAGE_SIZE;
3029
3030 #if CHECKSUM_THE_DATA
3031 cs->c_hash_data = hash_string(src, PAGE_SIZE);
3032 #endif
3033
3034 if (vm_compressor_algorithm() != VM_COMPRESSOR_DEFAULT_CODEC) {
3035 } else {
3036 c_size = WKdm_compress_new((const WK_word *)(uintptr_t)src, (WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
3037 (WK_word *)(uintptr_t)scratch_buf, max_csize - 4);
3038 }
3039 assert(c_size <= (max_csize - 4) && c_size >= -1);
3040
3041 if (c_size == -1) {
3042
3043 if (max_csize < PAGE_SIZE) {
3044 c_current_seg_filled(c_seg, current_chead);
3045 assert(*current_chead == NULL);
3046
3047 lck_mtx_unlock_always(&c_seg->c_lock);
3048
3049 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3050 goto retry;
3051 }
3052 c_size = PAGE_SIZE;
3053
3054 memcpy(&c_seg->c_store.c_buffer[cs->c_offset], src, c_size);
3055
3056 OSAddAtomic(1, &c_segment_noncompressible_pages);
3057
3058 } else if (c_size == 0) {
3059 int hash_index;
3060
3061 /*
3062 * special case - this is a page completely full of a single 32 bit value
3063 */
3064 hash_index = c_segment_sv_hash_insert(*(uint32_t *)(uintptr_t)src);
3065
3066 if (hash_index != -1) {
3067 slot_ptr->s_cindx = hash_index;
3068 slot_ptr->s_cseg = C_SV_CSEG_ID;
3069
3070 OSAddAtomic(1, &c_segment_svp_hash_succeeded);
3071 #if RECORD_THE_COMPRESSED_DATA
3072 c_compressed_record_data(src, 4);
3073 #endif
3074 goto sv_compression;
3075 }
3076 c_size = 4;
3077
3078 memcpy(&c_seg->c_store.c_buffer[cs->c_offset], src, c_size);
3079
3080 OSAddAtomic(1, &c_segment_svp_hash_failed);
3081 }
3082
3083 #if RECORD_THE_COMPRESSED_DATA
3084 c_compressed_record_data((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size);
3085 #endif
3086
3087 #if CHECKSUM_THE_COMPRESSED_DATA
3088 cs->c_hash_compressed_data = hash_string((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size);
3089 #endif
3090 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
3091
3092 PACK_C_SIZE(cs, c_size);
3093 c_seg->c_bytes_used += c_rounded_size;
3094 c_seg->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
3095
3096 slot_ptr->s_cindx = c_seg->c_nextslot++;
3097 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
3098 slot_ptr->s_cseg = c_seg->c_mysegno + 1;
3099
3100 sv_compression:
3101 if (c_seg->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg->c_nextslot >= C_SLOT_MAX_INDEX) {
3102 c_current_seg_filled(c_seg, current_chead);
3103 assert(*current_chead == NULL);
3104 }
3105 lck_mtx_unlock_always(&c_seg->c_lock);
3106
3107 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3108
3109 #if RECORD_THE_COMPRESSED_DATA
3110 if ((c_compressed_record_cptr - c_compressed_record_sbuf) >= C_SEG_ALLOCSIZE) {
3111 c_compressed_record_write(c_compressed_record_sbuf, (int)(c_compressed_record_cptr - c_compressed_record_sbuf));
3112 c_compressed_record_cptr = c_compressed_record_sbuf;
3113 }
3114 #endif
3115 if (c_size) {
3116 OSAddAtomic64(c_size, &c_segment_compressed_bytes);
3117 OSAddAtomic64(c_rounded_size, &compressor_bytes_used);
3118 }
3119 OSAddAtomic64(PAGE_SIZE, &c_segment_input_bytes);
3120
3121 OSAddAtomic(1, &c_segment_pages_compressed);
3122 OSAddAtomic(1, &sample_period_compression_count);
3123
3124 KERNEL_DEBUG(0xe0400000 | DBG_FUNC_END, *current_chead, c_size, c_segment_input_bytes, c_segment_compressed_bytes, 0);
3125
3126 return (0);
3127 }
3128
3129 static inline void sv_decompress(int32_t *ddst, int32_t pattern) {
3130 #if __x86_64__
3131 memset_word(ddst, pattern, PAGE_SIZE / sizeof(int32_t));
3132 #else
3133 size_t i;
3134
3135 /* Unroll the pattern fill loop 4x to encourage the
3136 * compiler to emit NEON stores, cf.
3137 * <rdar://problem/25839866> Loop autovectorization
3138 * anomalies.
3139 * We use separate loops for each PAGE_SIZE
3140 * to allow the autovectorizer to engage, as PAGE_SIZE
3141 * is currently not a constant.
3142 */
3143
3144 if (PAGE_SIZE == 4096) {
3145 for (i = 0; i < (4096U / sizeof(int32_t)); i += 4) {
3146 *ddst++ = pattern;
3147 *ddst++ = pattern;
3148 *ddst++ = pattern;
3149 *ddst++ = pattern;
3150 }
3151 } else {
3152 assert(PAGE_SIZE == 16384);
3153 for (i = 0; i < (int)(16384U / sizeof(int32_t)); i += 4) {
3154 *ddst++ = pattern;
3155 *ddst++ = pattern;
3156 *ddst++ = pattern;
3157 *ddst++ = pattern;
3158 }
3159 }
3160 #endif
3161 }
3162
3163 static int
3164 c_decompress_page(char *dst, volatile c_slot_mapping_t slot_ptr, int flags, int *zeroslot)
3165 {
3166 c_slot_t cs;
3167 c_segment_t c_seg;
3168 uint32_t c_segno;
3169 int c_indx;
3170 int c_rounded_size;
3171 uint32_t c_size;
3172 int retval = 0;
3173 boolean_t need_unlock = TRUE;
3174 boolean_t consider_defragmenting = FALSE;
3175 boolean_t kdp_mode = FALSE;
3176
3177 if (__improbable(flags & C_KDP)) {
3178 if (not_in_kdp) {
3179 panic("C_KDP passed to decompress page from outside of debugger context");
3180 }
3181
3182 assert((flags & C_KEEP) == C_KEEP);
3183 assert((flags & C_DONT_BLOCK) == C_DONT_BLOCK);
3184
3185 if ((flags & (C_DONT_BLOCK | C_KEEP)) != (C_DONT_BLOCK | C_KEEP)) {
3186 return (-2);
3187 }
3188
3189 kdp_mode = TRUE;
3190 *zeroslot = 0;
3191 }
3192
3193 ReTry:
3194 if (__probable(!kdp_mode)) {
3195 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3196 } else {
3197 if (kdp_lck_rw_lock_is_acquired_exclusive(&c_master_lock)) {
3198 return (-2);
3199 }
3200 }
3201
3202 #if HIBERNATION
3203 /*
3204 * if hibernation is enabled, it indicates (via a call
3205 * to 'vm_decompressor_lock' that no further
3206 * decompressions are allowed once it reaches
3207 * the point of flushing all of the currently dirty
3208 * anonymous memory through the compressor and out
3209 * to disk... in this state we allow freeing of compressed
3210 * pages and must honor the C_DONT_BLOCK case
3211 */
3212 if (dst && decompressions_blocked == TRUE) {
3213 if (flags & C_DONT_BLOCK) {
3214
3215 if (__probable(!kdp_mode)) {
3216 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3217 }
3218
3219 *zeroslot = 0;
3220 return (-2);
3221 }
3222 /*
3223 * it's safe to atomically assert and block behind the
3224 * lock held in shared mode because "decompressions_blocked" is
3225 * only set and cleared and the thread_wakeup done when the lock
3226 * is held exclusively
3227 */
3228 assert_wait((event_t)&decompressions_blocked, THREAD_UNINT);
3229
3230 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3231
3232 thread_block(THREAD_CONTINUE_NULL);
3233
3234 goto ReTry;
3235 }
3236 #endif
3237 /* s_cseg is actually "segno+1" */
3238 c_segno = slot_ptr->s_cseg - 1;
3239
3240 if (__improbable(c_segno >= c_segments_available))
3241 panic("c_decompress_page: c_segno %d >= c_segments_available %d, slot_ptr(%p), slot_data(%x)",
3242 c_segno, c_segments_available, slot_ptr, *(int *)((void *)slot_ptr));
3243
3244 if (__improbable(c_segments[c_segno].c_segno < c_segments_available))
3245 panic("c_decompress_page: c_segno %d is free, slot_ptr(%p), slot_data(%x)",
3246 c_segno, slot_ptr, *(int *)((void *)slot_ptr));
3247
3248 c_seg = c_segments[c_segno].c_seg;
3249
3250 if (__probable(!kdp_mode)) {
3251 lck_mtx_lock_spin_always(&c_seg->c_lock);
3252 } else {
3253 if (kdp_lck_mtx_lock_spin_is_acquired(&c_seg->c_lock)) {
3254 return (-2);
3255 }
3256 }
3257
3258 assert(c_seg->c_state != C_IS_EMPTY && c_seg->c_state != C_IS_FREE);
3259
3260 if (dst == NULL && c_seg->c_busy_swapping) {
3261 assert(c_seg->c_busy);
3262
3263 goto bypass_busy_check;
3264 }
3265 if (flags & C_DONT_BLOCK) {
3266 if (c_seg->c_busy || (C_SEG_IS_ONDISK(c_seg) && dst)) {
3267 *zeroslot = 0;
3268
3269 retval = -2;
3270 goto done;
3271 }
3272 }
3273 if (c_seg->c_busy) {
3274
3275 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3276
3277 c_seg_wait_on_busy(c_seg);
3278
3279 goto ReTry;
3280 }
3281 bypass_busy_check:
3282
3283 c_indx = slot_ptr->s_cindx;
3284
3285 if (__improbable(c_indx >= c_seg->c_nextslot))
3286 panic("c_decompress_page: c_indx %d >= c_nextslot %d, c_seg(%p), slot_ptr(%p), slot_data(%x)",
3287 c_indx, c_seg->c_nextslot, c_seg, slot_ptr, *(int *)((void *)slot_ptr));
3288
3289 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
3290
3291 c_size = UNPACK_C_SIZE(cs);
3292
3293 if (__improbable(c_size == 0))
3294 panic("c_decompress_page: c_size == 0, c_seg(%p), slot_ptr(%p), slot_data(%x)",
3295 c_seg, slot_ptr, *(int *)((void *)slot_ptr));
3296
3297 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
3298
3299 if (dst) {
3300 uint32_t age_of_cseg;
3301 clock_sec_t cur_ts_sec;
3302 clock_nsec_t cur_ts_nsec;
3303
3304 if (C_SEG_IS_ONDISK(c_seg)) {
3305 assert(kdp_mode == FALSE);
3306 retval = c_seg_swapin(c_seg, FALSE, TRUE);
3307 assert(retval == 0);
3308
3309 retval = 1;
3310 }
3311 if (c_seg->c_state == C_ON_BAD_Q) {
3312 assert(c_seg->c_store.c_buffer == NULL);
3313
3314 retval = -1;
3315 goto c_seg_invalid_data;
3316 }
3317 #if CHECKSUM_THE_COMPRESSED_DATA
3318 if (cs->c_hash_compressed_data != hash_string((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size))
3319 panic("compressed data doesn't match original hash: 0x%x, seg: %p, offset: %d, c_size: %d", cs->c_hash_compressed_data, c_seg, cs->c_offset, c_size);
3320 #endif
3321 if (c_rounded_size == PAGE_SIZE) {
3322 /*
3323 * page wasn't compressible... just copy it out
3324 */
3325 memcpy(dst, &c_seg->c_store.c_buffer[cs->c_offset], PAGE_SIZE);
3326 } else if (c_size == 4) {
3327 int32_t data;
3328 int32_t *dptr;
3329
3330 /*
3331 * page was populated with a single value
3332 * that didn't fit into our fast hash
3333 * so we packed it in as a single non-compressed value
3334 * that we need to populate the page with
3335 */
3336 dptr = (int32_t *)(uintptr_t)dst;
3337 data = *(int32_t *)(&c_seg->c_store.c_buffer[cs->c_offset]);
3338 sv_decompress(dptr, data);
3339 } else {
3340 uint32_t my_cpu_no;
3341 char *scratch_buf;
3342
3343 if (__probable(!kdp_mode)) {
3344 /*
3345 * we're behind the c_seg lock held in spin mode
3346 * which means pre-emption is disabled... therefore
3347 * the following sequence is atomic and safe
3348 */
3349 my_cpu_no = cpu_number();
3350
3351 assert(my_cpu_no < compressor_cpus);
3352
3353 scratch_buf = &compressor_scratch_bufs[my_cpu_no * vm_compressor_get_decode_scratch_size()];
3354 } else {
3355 scratch_buf = kdp_compressor_scratch_buf;
3356 }
3357
3358 if (vm_compressor_algorithm() != VM_COMPRESSOR_DEFAULT_CODEC) {
3359 } else {
3360 WKdm_decompress_new((WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
3361 (WK_word *)(uintptr_t)dst, (WK_word *)(uintptr_t)scratch_buf, c_size);
3362 }
3363 }
3364
3365 #if CHECKSUM_THE_DATA
3366 if (cs->c_hash_data != hash_string(dst, PAGE_SIZE))
3367 panic("decompressed data doesn't match original cs: %p, hash: %d, offset: %d, c_size: %d", cs, cs->c_hash_data, cs->c_offset, c_size);
3368
3369 #endif
3370 if (c_seg->c_swappedin_ts == 0 && !kdp_mode) {
3371
3372 clock_get_system_nanotime(&cur_ts_sec, &cur_ts_nsec);
3373
3374 age_of_cseg = (uint32_t)cur_ts_sec - c_seg->c_creation_ts;
3375 if (age_of_cseg < DECOMPRESSION_SAMPLE_MAX_AGE)
3376 OSAddAtomic(1, &age_of_decompressions_during_sample_period[age_of_cseg]);
3377 else
3378 OSAddAtomic(1, &overage_decompressions_during_sample_period);
3379
3380 OSAddAtomic(1, &sample_period_decompression_count);
3381 }
3382 }
3383 c_seg_invalid_data:
3384
3385 if (flags & C_KEEP) {
3386 *zeroslot = 0;
3387 goto done;
3388 }
3389 assert(kdp_mode == FALSE);
3390
3391 c_seg->c_bytes_unused += c_rounded_size;
3392 c_seg->c_bytes_used -= c_rounded_size;
3393 PACK_C_SIZE(cs, 0);
3394
3395 if (c_indx < c_seg->c_firstemptyslot)
3396 c_seg->c_firstemptyslot = c_indx;
3397
3398 OSAddAtomic(-1, &c_segment_pages_compressed);
3399
3400 if (c_seg->c_state != C_ON_BAD_Q && !(C_SEG_IS_ONDISK(c_seg))) {
3401 /*
3402 * C_SEG_IS_ONDISK == TRUE can occur when we're doing a
3403 * free of a compressed page (i.e. dst == NULL)
3404 */
3405 OSAddAtomic64(-c_rounded_size, &compressor_bytes_used);
3406 }
3407 if (c_seg->c_busy_swapping) {
3408 /*
3409 * bypass case for c_busy_swapping...
3410 * let the swapin/swapout paths deal with putting
3411 * the c_seg on the minor compaction queue if needed
3412 */
3413 assert(c_seg->c_busy);
3414 goto done;
3415 }
3416 assert(!c_seg->c_busy);
3417
3418 if (c_seg->c_state != C_IS_FILLING) {
3419 if (c_seg->c_bytes_used == 0) {
3420 if ( !(C_SEG_IS_ONDISK(c_seg))) {
3421 int pages_populated;
3422
3423 pages_populated = (round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE;
3424 c_seg->c_populated_offset = C_SEG_BYTES_TO_OFFSET(0);
3425
3426 if (pages_populated) {
3427
3428 assert(c_seg->c_state != C_ON_BAD_Q);
3429 assert(c_seg->c_store.c_buffer != NULL);
3430
3431 C_SEG_BUSY(c_seg);
3432 lck_mtx_unlock_always(&c_seg->c_lock);
3433
3434 kernel_memory_depopulate(compressor_map, (vm_offset_t) c_seg->c_store.c_buffer, pages_populated * PAGE_SIZE, KMA_COMPRESSOR);
3435
3436 lck_mtx_lock_spin_always(&c_seg->c_lock);
3437 C_SEG_WAKEUP_DONE(c_seg);
3438 }
3439 if (!c_seg->c_on_minorcompact_q && c_seg->c_state != C_ON_SWAPOUT_Q)
3440 c_seg_need_delayed_compaction(c_seg, FALSE);
3441 } else {
3442 if (c_seg->c_state != C_ON_SWAPPEDOUTSPARSE_Q) {
3443
3444 c_seg_move_to_sparse_list(c_seg);
3445 consider_defragmenting = TRUE;
3446 }
3447 }
3448 } else if (c_seg->c_on_minorcompact_q) {
3449
3450 assert(c_seg->c_state != C_ON_BAD_Q);
3451
3452 if (C_SEG_SHOULD_MINORCOMPACT(c_seg)) {
3453 c_seg_try_minor_compaction_and_unlock(c_seg);
3454 need_unlock = FALSE;
3455 }
3456 } else if ( !(C_SEG_IS_ONDISK(c_seg))) {
3457
3458 if (c_seg->c_state != C_ON_BAD_Q && c_seg->c_state != C_ON_SWAPOUT_Q && C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE) {
3459 c_seg_need_delayed_compaction(c_seg, FALSE);
3460 }
3461 } else if (c_seg->c_state != C_ON_SWAPPEDOUTSPARSE_Q && C_SEG_ONDISK_IS_SPARSE(c_seg)) {
3462
3463 c_seg_move_to_sparse_list(c_seg);
3464 consider_defragmenting = TRUE;
3465 }
3466 }
3467 done:
3468 if (__improbable(kdp_mode)) {
3469 return retval;
3470 }
3471
3472 if (need_unlock == TRUE)
3473 lck_mtx_unlock_always(&c_seg->c_lock);
3474
3475 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3476
3477 if (consider_defragmenting == TRUE)
3478 vm_swap_consider_defragmenting();
3479
3480
3481 return (retval);
3482 }
3483
3484
3485 int
3486 vm_compressor_get(ppnum_t pn, int *slot, int flags)
3487 {
3488 c_slot_mapping_t slot_ptr;
3489 char *dst;
3490 int zeroslot = 1;
3491 int retval;
3492
3493 #if __x86_64__
3494 dst = PHYSMAP_PTOV((uint64_t)pn << (uint64_t)PAGE_SHIFT);
3495 #else
3496 #error "unsupported architecture"
3497 #endif
3498 slot_ptr = (c_slot_mapping_t)slot;
3499
3500 if (slot_ptr->s_cseg == C_SV_CSEG_ID) {
3501 int32_t data;
3502 int32_t *dptr;
3503
3504 /*
3505 * page was populated with a single value
3506 * that found a home in our hash table
3507 * grab that value from the hash and populate the page
3508 * that we need to populate the page with
3509 */
3510 dptr = (int32_t *)(uintptr_t)dst;
3511 data = c_segment_sv_hash_table[slot_ptr->s_cindx].he_data;
3512 #if __x86_64__
3513 memset_word(dptr, data, PAGE_SIZE / sizeof(int32_t));
3514 #else
3515 {
3516 int i;
3517
3518 for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++)
3519 *dptr++ = data;
3520 }
3521 #endif
3522 c_segment_sv_hash_drop_ref(slot_ptr->s_cindx);
3523
3524 if ( !(flags & C_KEEP)) {
3525 OSAddAtomic(-1, &c_segment_pages_compressed);
3526 *slot = 0;
3527 }
3528 if (data)
3529 OSAddAtomic(1, &c_segment_svp_nonzero_decompressions);
3530 else
3531 OSAddAtomic(1, &c_segment_svp_zero_decompressions);
3532
3533 return (0);
3534 }
3535
3536 retval = c_decompress_page(dst, slot_ptr, flags, &zeroslot);
3537
3538 /*
3539 * zeroslot will be set to 0 by c_decompress_page if (flags & C_KEEP)
3540 * or (flags & C_DONT_BLOCK) and we found 'c_busy' or 'C_SEG_IS_ONDISK' to be TRUE
3541 */
3542 if (zeroslot) {
3543 *slot = 0;
3544 }
3545 /*
3546 * returns 0 if we successfully decompressed a page from a segment already in memory
3547 * returns 1 if we had to first swap in the segment, before successfully decompressing the page
3548 * returns -1 if we encountered an error swapping in the segment - decompression failed
3549 * returns -2 if (flags & C_DONT_BLOCK) and we found 'c_busy' or 'C_SEG_IS_ONDISK' to be true
3550 */
3551 return (retval);
3552 }
3553
3554
3555 int
3556 vm_compressor_free(int *slot, int flags)
3557 {
3558 c_slot_mapping_t slot_ptr;
3559 int zeroslot = 1;
3560 int retval;
3561
3562 assert(flags == 0 || flags == C_DONT_BLOCK);
3563
3564 slot_ptr = (c_slot_mapping_t)slot;
3565
3566 if (slot_ptr->s_cseg == C_SV_CSEG_ID) {
3567
3568 c_segment_sv_hash_drop_ref(slot_ptr->s_cindx);
3569 OSAddAtomic(-1, &c_segment_pages_compressed);
3570
3571 *slot = 0;
3572 return (0);
3573 }
3574 retval = c_decompress_page(NULL, slot_ptr, flags, &zeroslot);
3575 /*
3576 * returns 0 if we successfully freed the specified compressed page
3577 * returns -2 if (flags & C_DONT_BLOCK) and we found 'c_busy' set
3578 */
3579
3580 if (retval == 0)
3581 *slot = 0;
3582 else
3583 assert(retval == -2);
3584
3585 return (retval);
3586 }
3587
3588
3589 int
3590 vm_compressor_put(ppnum_t pn, int *slot, void **current_chead, char *scratch_buf)
3591 {
3592 char *src;
3593 int retval;
3594
3595 #if __x86_64__
3596 src = PHYSMAP_PTOV((uint64_t)pn << (uint64_t)PAGE_SHIFT);
3597 #else
3598 #error "unsupported architecture"
3599 #endif
3600 retval = c_compress_page(src, (c_slot_mapping_t)slot, (c_segment_t *)current_chead, scratch_buf);
3601
3602 return (retval);
3603 }
3604
3605 void
3606 vm_compressor_transfer(
3607 int *dst_slot_p,
3608 int *src_slot_p)
3609 {
3610 c_slot_mapping_t dst_slot, src_slot;
3611 c_segment_t c_seg;
3612 int c_indx;
3613 c_slot_t cs;
3614
3615 src_slot = (c_slot_mapping_t) src_slot_p;
3616
3617 if (src_slot->s_cseg == C_SV_CSEG_ID) {
3618 *dst_slot_p = *src_slot_p;
3619 *src_slot_p = 0;
3620 return;
3621 }
3622 dst_slot = (c_slot_mapping_t) dst_slot_p;
3623 Retry:
3624 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3625 /* get segment for src_slot */
3626 c_seg = c_segments[src_slot->s_cseg -1].c_seg;
3627 /* lock segment */
3628 lck_mtx_lock_spin_always(&c_seg->c_lock);
3629 /* wait if it's busy */
3630 if (c_seg->c_busy && !c_seg->c_busy_swapping) {
3631 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3632 c_seg_wait_on_busy(c_seg);
3633 goto Retry;
3634 }
3635 /* find the c_slot */
3636 c_indx = src_slot->s_cindx;
3637 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
3638 /* point the c_slot back to dst_slot instead of src_slot */
3639 cs->c_packed_ptr = C_SLOT_PACK_PTR(dst_slot);
3640 /* transfer */
3641 *dst_slot_p = *src_slot_p;
3642 *src_slot_p = 0;
3643 lck_mtx_unlock_always(&c_seg->c_lock);
3644 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3645 }
3646
3647 #if CONFIG_FREEZE
3648
3649 int freezer_finished_filling = 0;
3650
3651 void
3652 vm_compressor_finished_filling(
3653 void **current_chead)
3654 {
3655 c_segment_t c_seg;
3656
3657 if ((c_seg = *(c_segment_t *)current_chead) == NULL)
3658 return;
3659
3660 assert(c_seg->c_state == C_IS_FILLING);
3661
3662 lck_mtx_lock_spin_always(&c_seg->c_lock);
3663
3664 c_current_seg_filled(c_seg, (c_segment_t *)current_chead);
3665
3666 lck_mtx_unlock_always(&c_seg->c_lock);
3667
3668 freezer_finished_filling++;
3669 }
3670
3671
3672 /*
3673 * This routine is used to transfer the compressed chunks from
3674 * the c_seg/cindx pointed to by slot_p into a new c_seg headed
3675 * by the current_chead and a new cindx within that c_seg.
3676 *
3677 * Currently, this routine is only used by the "freezer backed by
3678 * compressor with swap" mode to create a series of c_segs that
3679 * only contain compressed data belonging to one task. So, we
3680 * move a task's previously compressed data into a set of new
3681 * c_segs which will also hold the task's yet to be compressed data.
3682 */
3683
3684 kern_return_t
3685 vm_compressor_relocate(
3686 void **current_chead,
3687 int *slot_p)
3688 {
3689 c_slot_mapping_t slot_ptr;
3690 c_slot_mapping_t src_slot;
3691 uint32_t c_rounded_size;
3692 uint32_t c_size;
3693 uint16_t dst_slot;
3694 c_slot_t c_dst;
3695 c_slot_t c_src;
3696 int c_indx;
3697 c_segment_t c_seg_dst = NULL;
3698 c_segment_t c_seg_src = NULL;
3699 kern_return_t kr = KERN_SUCCESS;
3700
3701
3702 src_slot = (c_slot_mapping_t) slot_p;
3703
3704 if (src_slot->s_cseg == C_SV_CSEG_ID) {
3705 /*
3706 * no need to relocate... this is a page full of a single
3707 * value which is hashed to a single entry not contained
3708 * in a c_segment_t
3709 */
3710 return (kr);
3711 }
3712
3713 Relookup_dst:
3714 c_seg_dst = c_seg_allocate((c_segment_t *)current_chead);
3715 /*
3716 * returns with c_seg lock held
3717 * and PAGE_REPLACEMENT_DISALLOWED(TRUE)...
3718 * c_nextslot has been allocated and
3719 * c_store.c_buffer populated
3720 */
3721 if (c_seg_dst == NULL) {
3722 /*
3723 * Out of compression segments?
3724 */
3725 kr = KERN_RESOURCE_SHORTAGE;
3726 goto out;
3727 }
3728
3729 assert(c_seg_dst->c_busy == 0);
3730
3731 C_SEG_BUSY(c_seg_dst);
3732
3733 dst_slot = c_seg_dst->c_nextslot;
3734
3735 lck_mtx_unlock_always(&c_seg_dst->c_lock);
3736
3737 Relookup_src:
3738 c_seg_src = c_segments[src_slot->s_cseg - 1].c_seg;
3739
3740 assert(c_seg_dst != c_seg_src);
3741
3742 lck_mtx_lock_spin_always(&c_seg_src->c_lock);
3743
3744 if (C_SEG_IS_ONDISK(c_seg_src)) {
3745
3746 /*
3747 * A "thaw" can mark a process as eligible for
3748 * another freeze cycle without bringing any of
3749 * its swapped out c_segs back from disk (because
3750 * that is done on-demand).
3751 *
3752 * If the src c_seg we find for our pre-compressed
3753 * data is already on-disk, then we are dealing
3754 * with an app's data that is already packed and
3755 * swapped out. Don't do anything.
3756 */
3757
3758 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3759
3760 lck_mtx_unlock_always(&c_seg_src->c_lock);
3761
3762 c_seg_src = NULL;
3763
3764 goto out;
3765 }
3766
3767 if (c_seg_src->c_busy) {
3768
3769 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3770 c_seg_wait_on_busy(c_seg_src);
3771
3772 c_seg_src = NULL;
3773
3774 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3775
3776 goto Relookup_src;
3777 }
3778
3779 C_SEG_BUSY(c_seg_src);
3780
3781 lck_mtx_unlock_always(&c_seg_src->c_lock);
3782
3783 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3784
3785 /* find the c_slot */
3786 c_indx = src_slot->s_cindx;
3787
3788 c_src = C_SEG_SLOT_FROM_INDEX(c_seg_src, c_indx);
3789
3790 c_size = UNPACK_C_SIZE(c_src);
3791
3792 assert(c_size);
3793
3794 if (c_size > (uint32_t)(C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES((int32_t)c_seg_dst->c_nextoffset))) {
3795 /*
3796 * This segment is full. We need a new one.
3797 */
3798
3799 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3800
3801 lck_mtx_lock_spin_always(&c_seg_src->c_lock);
3802 C_SEG_WAKEUP_DONE(c_seg_src);
3803 lck_mtx_unlock_always(&c_seg_src->c_lock);
3804
3805 c_seg_src = NULL;
3806
3807 lck_mtx_lock_spin_always(&c_seg_dst->c_lock);
3808
3809 assert(c_seg_dst->c_busy);
3810 assert(c_seg_dst->c_state == C_IS_FILLING);
3811 assert(!c_seg_dst->c_on_minorcompact_q);
3812
3813 c_current_seg_filled(c_seg_dst, (c_segment_t *)current_chead);
3814 assert(*current_chead == NULL);
3815
3816 C_SEG_WAKEUP_DONE(c_seg_dst);
3817
3818 lck_mtx_unlock_always(&c_seg_dst->c_lock);
3819
3820 c_seg_dst = NULL;
3821
3822 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3823
3824 goto Relookup_dst;
3825 }
3826
3827 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, c_seg_dst->c_nextslot);
3828
3829 memcpy(&c_seg_dst->c_store.c_buffer[c_seg_dst->c_nextoffset], &c_seg_src->c_store.c_buffer[c_src->c_offset], c_size);
3830
3831 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
3832
3833 cslot_copy(c_dst, c_src);
3834 c_dst->c_offset = c_seg_dst->c_nextoffset;
3835
3836 if (c_seg_dst->c_firstemptyslot == c_seg_dst->c_nextslot)
3837 c_seg_dst->c_firstemptyslot++;
3838
3839 c_seg_dst->c_nextslot++;
3840 c_seg_dst->c_bytes_used += c_rounded_size;
3841 c_seg_dst->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
3842
3843
3844 PACK_C_SIZE(c_src, 0);
3845
3846 c_seg_src->c_bytes_used -= c_rounded_size;
3847 c_seg_src->c_bytes_unused += c_rounded_size;
3848
3849 if (c_indx < c_seg_src->c_firstemptyslot) {
3850 c_seg_src->c_firstemptyslot = c_indx;
3851 }
3852
3853 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, dst_slot);
3854
3855 PAGE_REPLACEMENT_ALLOWED(TRUE);
3856 slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
3857 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
3858 slot_ptr->s_cseg = c_seg_dst->c_mysegno + 1;
3859 slot_ptr->s_cindx = dst_slot;
3860
3861 PAGE_REPLACEMENT_ALLOWED(FALSE);
3862
3863 out:
3864 if (c_seg_src) {
3865
3866 lck_mtx_lock_spin_always(&c_seg_src->c_lock);
3867
3868 C_SEG_WAKEUP_DONE(c_seg_src);
3869
3870 if (c_seg_src->c_bytes_used == 0 && c_seg_src->c_state != C_IS_FILLING) {
3871 if (!c_seg_src->c_on_minorcompact_q)
3872 c_seg_need_delayed_compaction(c_seg_src, FALSE);
3873 }
3874
3875 lck_mtx_unlock_always(&c_seg_src->c_lock);
3876 }
3877
3878 if (c_seg_dst) {
3879
3880 PAGE_REPLACEMENT_DISALLOWED(TRUE);
3881
3882 lck_mtx_lock_spin_always(&c_seg_dst->c_lock);
3883
3884 if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX_INDEX) {
3885 /*
3886 * Nearing or exceeded maximum slot and offset capacity.
3887 */
3888 assert(c_seg_dst->c_busy);
3889 assert(c_seg_dst->c_state == C_IS_FILLING);
3890 assert(!c_seg_dst->c_on_minorcompact_q);
3891
3892 c_current_seg_filled(c_seg_dst, (c_segment_t *)current_chead);
3893 assert(*current_chead == NULL);
3894 }
3895
3896 C_SEG_WAKEUP_DONE(c_seg_dst);
3897
3898 lck_mtx_unlock_always(&c_seg_dst->c_lock);
3899
3900 c_seg_dst = NULL;
3901
3902 PAGE_REPLACEMENT_DISALLOWED(FALSE);
3903 }
3904
3905 return kr;
3906 }
3907 #endif /* CONFIG_FREEZE */
mirror of XNU