]> git.saurik.com Git - apple/xnu.git/blob - osfmk/vm/vm_compressor.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-2782.40.9.tar.gz
[apple/xnu.git] / osfmk / vm / vm_compressor.c
1 /*
2 * Copyright (c) 2000-2013 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <vm/vm_compressor.h>
30
31 #if CONFIG_PHANTOM_CACHE
32 #include <vm/vm_phantom_cache.h>
33 #endif
34
35 #include <vm/vm_map.h>
36 #include <vm/vm_pageout.h>
37 #include <vm/memory_object.h>
38 #include <mach/mach_host.h> /* for host_info() */
39 #include <kern/ledger.h>
40
41 #include <default_pager/default_pager_alerts.h>
42 #include <default_pager/default_pager_object_server.h>
43
44 #include <IOKit/IOHibernatePrivate.h>
45
46 /*
47 * vm_compressor_mode has a heirarchy of control to set its value.
48 * boot-args are checked first, then device-tree, and finally
49 * the default value that is defined below. See vm_fault_init() for
50 * the boot-arg & device-tree code.
51 */
52
53 extern ipc_port_t min_pages_trigger_port;
54 extern lck_mtx_t paging_segments_lock;
55 #define PSL_LOCK() lck_mtx_lock(&paging_segments_lock)
56 #define PSL_UNLOCK() lck_mtx_unlock(&paging_segments_lock)
57
58
59 int vm_compressor_mode = VM_PAGER_COMPRESSOR_WITH_SWAP;
60 int vm_scale = 16;
61
62
63 int vm_compressor_is_active = 0;
64 int vm_compression_limit = 0;
65
66 extern boolean_t vm_swap_up;
67 extern void vm_pageout_io_throttle(void);
68
69 #if CHECKSUM_THE_DATA || CHECKSUM_THE_SWAP || CHECKSUM_THE_COMPRESSED_DATA
70 extern unsigned int hash_string(char *cp, int len);
71 #endif
72
73
74 struct c_slot {
75 uint64_t c_offset:C_SEG_OFFSET_BITS,
76 c_size:12,
77 c_packed_ptr:36;
78 #if CHECKSUM_THE_DATA
79 unsigned int c_hash_data;
80 #endif
81 #if CHECKSUM_THE_COMPRESSED_DATA
82 unsigned int c_hash_compressed_data;
83 #endif
84
85 };
86
87 #define UNPACK_C_SIZE(cs) ((cs->c_size == (PAGE_SIZE-1)) ? PAGE_SIZE : cs->c_size)
88 #define PACK_C_SIZE(cs, size) (cs->c_size = ((size == PAGE_SIZE) ? PAGE_SIZE - 1 : size))
89
90
91 struct c_slot_mapping {
92 uint32_t s_cseg:22, /* segment number + 1 */
93 s_cindx:10; /* index in the segment */
94 };
95
96 typedef struct c_slot_mapping *c_slot_mapping_t;
97
98
99 union c_segu {
100 c_segment_t c_seg;
101 uint32_t c_segno;
102 };
103
104
105
106 #define C_SLOT_PACK_PTR(ptr) (((uintptr_t)ptr - (uintptr_t) VM_MIN_KERNEL_AND_KEXT_ADDRESS) >> 2)
107 #define C_SLOT_UNPACK_PTR(cslot) ((uintptr_t)(cslot->c_packed_ptr << 2) + (uintptr_t) VM_MIN_KERNEL_AND_KEXT_ADDRESS)
108
109
110 uint32_t c_segment_count = 0;
111
112 uint64_t c_generation_id = 0;
113 uint64_t c_generation_id_flush_barrier;
114
115
116 #define HIBERNATE_FLUSHING_SECS_TO_COMPLETE 120
117
118 boolean_t hibernate_no_swapspace = FALSE;
119 clock_sec_t hibernate_flushing_deadline = 0;
120
121
122 #if TRACK_BAD_C_SEGMENTS
123 queue_head_t c_bad_list_head;
124 uint32_t c_bad_count = 0;
125 #endif
126
127 queue_head_t c_age_list_head;
128 queue_head_t c_swapout_list_head;
129 queue_head_t c_swappedin_list_head;
130 queue_head_t c_swappedout_list_head;
131 queue_head_t c_swappedout_sparse_list_head;
132
133 uint32_t c_age_count = 0;
134 uint32_t c_swapout_count = 0;
135 uint32_t c_swappedin_count = 0;
136 uint32_t c_swappedout_count = 0;
137 uint32_t c_swappedout_sparse_count = 0;
138
139 queue_head_t c_minor_list_head;
140 uint32_t c_minor_count = 0;
141
142 union c_segu *c_segments;
143 caddr_t c_segments_next_page;
144 boolean_t c_segments_busy;
145 uint32_t c_segments_available;
146 uint32_t c_segments_limit;
147 uint32_t c_segments_nearing_limit;
148 uint32_t c_segment_pages_compressed;
149 uint32_t c_segment_pages_compressed_limit;
150 uint32_t c_segment_pages_compressed_nearing_limit;
151 uint32_t c_free_segno_head = (uint32_t)-1;
152
153 uint32_t vm_compressor_minorcompact_threshold_divisor = 10;
154 uint32_t vm_compressor_majorcompact_threshold_divisor = 10;
155 uint32_t vm_compressor_unthrottle_threshold_divisor = 10;
156 uint32_t vm_compressor_catchup_threshold_divisor = 10;
157
158 #define C_SEGMENTS_PER_PAGE (PAGE_SIZE / sizeof(union c_segu))
159
160
161 lck_grp_attr_t vm_compressor_lck_grp_attr;
162 lck_attr_t vm_compressor_lck_attr;
163 lck_grp_t vm_compressor_lck_grp;
164
165
166 #if __i386__ || __x86_64__
167 lck_mtx_t *c_list_lock;
168 #else /* __i386__ || __x86_64__ */
169 lck_spin_t *c_list_lock;
170 #endif /* __i386__ || __x86_64__ */
171
172 lck_rw_t c_master_lock;
173 boolean_t decompressions_blocked = FALSE;
174
175 zone_t compressor_segment_zone;
176 int c_compressor_swap_trigger = 0;
177
178 uint32_t compressor_cpus;
179 char *compressor_scratch_bufs;
180
181
182 clock_sec_t start_of_sample_period_sec = 0;
183 clock_nsec_t start_of_sample_period_nsec = 0;
184 clock_sec_t start_of_eval_period_sec = 0;
185 clock_nsec_t start_of_eval_period_nsec = 0;
186 uint32_t sample_period_decompression_count = 0;
187 uint32_t sample_period_compression_count = 0;
188 uint32_t last_eval_decompression_count = 0;
189 uint32_t last_eval_compression_count = 0;
190
191 #define DECOMPRESSION_SAMPLE_MAX_AGE (60 * 30)
192
193 uint32_t swapout_target_age = 0;
194 uint32_t age_of_decompressions_during_sample_period[DECOMPRESSION_SAMPLE_MAX_AGE];
195 uint32_t overage_decompressions_during_sample_period = 0;
196
197 void do_fastwake_warmup(void);
198 boolean_t fastwake_warmup = FALSE;
199 boolean_t fastwake_recording_in_progress = FALSE;
200 clock_sec_t dont_trim_until_ts = 0;
201
202 uint64_t c_segment_warmup_count;
203 uint64_t first_c_segment_to_warm_generation_id = 0;
204 uint64_t last_c_segment_to_warm_generation_id = 0;
205 boolean_t hibernate_flushing = FALSE;
206
207 int64_t c_segment_input_bytes __attribute__((aligned(8))) = 0;
208 int64_t c_segment_compressed_bytes __attribute__((aligned(8))) = 0;
209 int64_t compressor_bytes_used __attribute__((aligned(8))) = 0;
210 uint64_t compressor_kvspace_used __attribute__((aligned(8))) = 0;
211 uint64_t compressor_kvwaste_limit = 0;
212
213 static boolean_t compressor_needs_to_swap(void);
214 static void vm_compressor_swap_trigger_thread(void);
215 static void vm_compressor_do_delayed_compactions(boolean_t);
216 static void vm_compressor_compact_and_swap(boolean_t);
217 static void vm_compressor_age_swapped_in_segments(boolean_t);
218
219 boolean_t vm_compressor_low_on_space(void);
220
221 void compute_swapout_target_age(void);
222
223 boolean_t c_seg_major_compact(c_segment_t, c_segment_t);
224 boolean_t c_seg_major_compact_ok(c_segment_t, c_segment_t);
225
226 int c_seg_minor_compaction_and_unlock(c_segment_t, boolean_t);
227 int c_seg_do_minor_compaction_and_unlock(c_segment_t, boolean_t, boolean_t, boolean_t);
228 void c_seg_try_minor_compaction_and_unlock(c_segment_t c_seg);
229 void c_seg_need_delayed_compaction(c_segment_t);
230
231 void c_seg_move_to_sparse_list(c_segment_t);
232 void c_seg_insert_into_q(queue_head_t *, c_segment_t);
233
234 boolean_t c_seg_try_free(c_segment_t);
235 void c_seg_free(c_segment_t);
236 void c_seg_free_locked(c_segment_t);
237
238
239 uint64_t vm_available_memory(void);
240 uint64_t vm_compressor_pages_compressed(void);
241
242 extern unsigned int dp_pages_free, dp_pages_reserve;
243
244 uint64_t
245 vm_available_memory(void)
246 {
247 return (((uint64_t)AVAILABLE_NON_COMPRESSED_MEMORY) * PAGE_SIZE_64);
248 }
249
250
251 uint64_t
252 vm_compressor_pages_compressed(void)
253 {
254 return (c_segment_pages_compressed * PAGE_SIZE_64);
255 }
256
257
258 boolean_t
259 vm_compression_available(void)
260 {
261 if ( !(COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE))
262 return (FALSE);
263
264 if (c_segments_available >= c_segments_limit || c_segment_pages_compressed >= c_segment_pages_compressed_limit)
265 return (FALSE);
266
267 return (TRUE);
268 }
269
270
271 boolean_t
272 vm_compressor_low_on_space(void)
273 {
274 if ((c_segment_pages_compressed > c_segment_pages_compressed_nearing_limit) ||
275 (c_segment_count > c_segments_nearing_limit))
276 return (TRUE);
277
278 return (FALSE);
279 }
280
281
282 int
283 vm_wants_task_throttled(task_t task)
284 {
285 if (task == kernel_task)
286 return (0);
287
288 if (vm_compressor_mode == COMPRESSED_PAGER_IS_ACTIVE || vm_compressor_mode == DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE) {
289 if ((vm_compressor_low_on_space() || HARD_THROTTLE_LIMIT_REACHED()) &&
290 (unsigned int)pmap_compressed(task->map->pmap) > (c_segment_pages_compressed / 4))
291 return (1);
292 } else {
293 if (((dp_pages_free + dp_pages_reserve < 2000) && VM_DYNAMIC_PAGING_ENABLED(memory_manager_default)) &&
294 get_task_resident_size(task) > (((AVAILABLE_NON_COMPRESSED_MEMORY) * PAGE_SIZE) / 5))
295 return (1);
296 }
297 return (0);
298 }
299
300
301 void
302 vm_compressor_init_locks(void)
303 {
304 lck_grp_attr_setdefault(&vm_compressor_lck_grp_attr);
305 lck_grp_init(&vm_compressor_lck_grp, "vm_compressor", &vm_compressor_lck_grp_attr);
306 lck_attr_setdefault(&vm_compressor_lck_attr);
307
308 lck_rw_init(&c_master_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
309 }
310
311
312 void
313 vm_decompressor_lock(void)
314 {
315 PAGE_REPLACEMENT_ALLOWED(TRUE);
316
317 decompressions_blocked = TRUE;
318
319 PAGE_REPLACEMENT_ALLOWED(FALSE);
320 }
321
322 void
323 vm_decompressor_unlock(void)
324 {
325 PAGE_REPLACEMENT_ALLOWED(TRUE);
326
327 decompressions_blocked = FALSE;
328
329 PAGE_REPLACEMENT_ALLOWED(FALSE);
330
331 thread_wakeup((event_t)&decompressions_blocked);
332 }
333
334
335
336 void
337 vm_compressor_init(void)
338 {
339 thread_t thread;
340 struct c_slot cs_dummy;
341 c_slot_t cs = &cs_dummy;
342
343 /*
344 * ensure that any pointer that gets created from
345 * the vm_page zone can be packed properly
346 */
347 cs->c_packed_ptr = C_SLOT_PACK_PTR(zone_map_min_address);
348
349 if (C_SLOT_UNPACK_PTR(cs) != (uintptr_t)zone_map_min_address)
350 panic("C_SLOT_UNPACK_PTR failed on zone_map_min_address - %p", (void *)zone_map_min_address);
351
352 cs->c_packed_ptr = C_SLOT_PACK_PTR(zone_map_max_address);
353
354 if (C_SLOT_UNPACK_PTR(cs) != (uintptr_t)zone_map_max_address)
355 panic("C_SLOT_UNPACK_PTR failed on zone_map_max_address - %p", (void *)zone_map_max_address);
356
357
358 assert((C_SEGMENTS_PER_PAGE * sizeof(union c_segu)) == PAGE_SIZE);
359
360 PE_parse_boot_argn("vm_compression_limit", &vm_compression_limit, sizeof (vm_compression_limit));
361
362 if (max_mem <= (3ULL * 1024ULL * 1024ULL * 1024ULL)) {
363 vm_compressor_minorcompact_threshold_divisor = 11;
364 vm_compressor_majorcompact_threshold_divisor = 13;
365 vm_compressor_unthrottle_threshold_divisor = 20;
366 vm_compressor_catchup_threshold_divisor = 35;
367 } else {
368 vm_compressor_minorcompact_threshold_divisor = 20;
369 vm_compressor_majorcompact_threshold_divisor = 25;
370 vm_compressor_unthrottle_threshold_divisor = 35;
371 vm_compressor_catchup_threshold_divisor = 50;
372 }
373 /*
374 * vm_page_init_lck_grp is now responsible for calling vm_compressor_init_locks
375 * c_master_lock needs to be available early so that "vm_page_find_contiguous" can
376 * use PAGE_REPLACEMENT_ALLOWED to coordinate with the compressor.
377 */
378
379 #if __i386__ || __x86_64__
380 c_list_lock = lck_mtx_alloc_init(&vm_compressor_lck_grp, &vm_compressor_lck_attr);
381 #else /* __i386__ || __x86_64__ */
382 c_list_lock = lck_spin_alloc_init(&vm_compressor_lck_grp, &vm_compressor_lck_attr);
383 #endif /* __i386__ || __x86_64__ */
384
385 #if TRACK_BAD_C_SEGMENTS
386 queue_init(&c_bad_list_head);
387 #endif
388 queue_init(&c_age_list_head);
389 queue_init(&c_minor_list_head);
390 queue_init(&c_swapout_list_head);
391 queue_init(&c_swappedin_list_head);
392 queue_init(&c_swappedout_list_head);
393 queue_init(&c_swappedout_sparse_list_head);
394
395 compressor_segment_zone = zinit(sizeof (struct c_segment),
396 128000 * sizeof (struct c_segment),
397 8192, "compressor_segment");
398 zone_change(compressor_segment_zone, Z_CALLERACCT, FALSE);
399 zone_change(compressor_segment_zone, Z_NOENCRYPT, TRUE);
400
401
402 c_free_segno_head = -1;
403 c_segments_available = 0;
404
405 if (vm_compression_limit == 0) {
406 c_segment_pages_compressed_limit = (uint32_t)((max_mem / PAGE_SIZE)) * vm_scale;
407
408 #define OLD_SWAP_LIMIT (1024 * 1024 * 16)
409 #define MAX_SWAP_LIMIT (1024 * 1024 * 128)
410
411 if (c_segment_pages_compressed_limit > (OLD_SWAP_LIMIT))
412 c_segment_pages_compressed_limit = OLD_SWAP_LIMIT;
413
414 if (c_segment_pages_compressed_limit < (uint32_t)(max_mem / PAGE_SIZE_64))
415 c_segment_pages_compressed_limit = (uint32_t)(max_mem / PAGE_SIZE_64);
416 } else {
417 if (vm_compression_limit < MAX_SWAP_LIMIT)
418 c_segment_pages_compressed_limit = vm_compression_limit;
419 else
420 c_segment_pages_compressed_limit = MAX_SWAP_LIMIT;
421 }
422 if ((c_segments_limit = c_segment_pages_compressed_limit / (C_SEG_BUFSIZE / PAGE_SIZE)) > C_SEG_MAX_LIMIT)
423 c_segments_limit = C_SEG_MAX_LIMIT;
424
425 c_segment_pages_compressed_nearing_limit = (c_segment_pages_compressed_limit * 98) / 100;
426 c_segments_nearing_limit = (c_segments_limit * 98) / 100;
427
428 compressor_kvwaste_limit = (vm_map_max(kernel_map) - vm_map_min(kernel_map)) / 16;
429
430 c_segments_busy = FALSE;
431
432 if (kernel_memory_allocate(kernel_map, (vm_offset_t *)(&c_segments), (sizeof(union c_segu) * c_segments_limit), 0, KMA_KOBJECT | KMA_VAONLY) != KERN_SUCCESS)
433 panic("vm_compressor_init: kernel_memory_allocate failed\n");
434
435 c_segments_next_page = (caddr_t)c_segments;
436
437 {
438 host_basic_info_data_t hinfo;
439 mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
440
441 #define BSD_HOST 1
442 host_info((host_t)BSD_HOST, HOST_BASIC_INFO, (host_info_t)&hinfo, &count);
443
444 compressor_cpus = hinfo.max_cpus;
445
446 compressor_scratch_bufs = kalloc(compressor_cpus * WKdm_SCRATCH_BUF_SIZE);
447 }
448
449 if (kernel_thread_start_priority((thread_continue_t)vm_compressor_swap_trigger_thread, NULL,
450 BASEPRI_PREEMPT - 1, &thread) != KERN_SUCCESS) {
451 panic("vm_compressor_swap_trigger_thread: create failed");
452 }
453 thread->options |= TH_OPT_VMPRIV;
454
455 thread_deallocate(thread);
456
457 assert(default_pager_init_flag == 0);
458
459 if (vm_pageout_internal_start() != KERN_SUCCESS) {
460 panic("vm_compressor_init: Failed to start the internal pageout thread.\n");
461 }
462
463 if ((vm_compressor_mode == VM_PAGER_COMPRESSOR_WITH_SWAP) ||
464 (vm_compressor_mode == VM_PAGER_FREEZER_COMPRESSOR_WITH_SWAP)) {
465 vm_compressor_swap_init();
466 }
467
468 if (COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_SWAPBACKED)
469 vm_compressor_is_active = 1;
470
471 #if CONFIG_FREEZE
472 memorystatus_freeze_enabled = TRUE;
473 #endif /* CONFIG_FREEZE */
474
475 default_pager_init_flag = 1;
476
477 vm_page_reactivate_all_throttled();
478 }
479
480
481 #if VALIDATE_C_SEGMENTS
482
483 static void
484 c_seg_validate(c_segment_t c_seg, boolean_t must_be_compact)
485 {
486 int c_indx;
487 int32_t bytes_used;
488 int32_t bytes_unused;
489 uint32_t c_rounded_size;
490 uint32_t c_size;
491 c_slot_t cs;
492
493 if (c_seg->c_firstemptyslot < c_seg->c_nextslot) {
494 c_indx = c_seg->c_firstemptyslot;
495 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
496
497 if (cs == NULL)
498 panic("c_seg_validate: no slot backing c_firstemptyslot");
499
500 if (cs->c_size)
501 panic("c_seg_validate: c_firstemptyslot has non-zero size (%d)\n", cs->c_size);
502 }
503 bytes_used = 0;
504 bytes_unused = 0;
505
506 for (c_indx = 0; c_indx < c_seg->c_nextslot; c_indx++) {
507
508 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
509
510 c_size = UNPACK_C_SIZE(cs);
511
512 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
513
514 bytes_used += c_rounded_size;
515
516 #if CHECKSUM_THE_COMPRESSED_DATA
517 if (c_size && cs->c_hash_compressed_data != hash_string((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size))
518 panic("compressed data doesn't match original");
519 #endif
520 }
521
522 if (bytes_used != c_seg->c_bytes_used)
523 panic("c_seg_validate: bytes_used mismatch - found %d, segment has %d\n", bytes_used, c_seg->c_bytes_used);
524
525 if (c_seg->c_bytes_used > C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset))
526 panic("c_seg_validate: c_bytes_used > c_nextoffset - c_nextoffset = %d, c_bytes_used = %d\n",
527 (int32_t)C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset), c_seg->c_bytes_used);
528
529 if (must_be_compact) {
530 if (c_seg->c_bytes_used != C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset))
531 panic("c_seg_validate: c_bytes_used doesn't match c_nextoffset - c_nextoffset = %d, c_bytes_used = %d\n",
532 (int32_t)C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset), c_seg->c_bytes_used);
533 }
534 }
535
536 #endif
537
538
539 void
540 c_seg_need_delayed_compaction(c_segment_t c_seg)
541 {
542 boolean_t clear_busy = FALSE;
543
544 if ( !lck_mtx_try_lock_spin_always(c_list_lock)) {
545 C_SEG_BUSY(c_seg);
546
547 lck_mtx_unlock_always(&c_seg->c_lock);
548 lck_mtx_lock_spin_always(c_list_lock);
549 lck_mtx_lock_spin_always(&c_seg->c_lock);
550
551 clear_busy = TRUE;
552 }
553 if (!c_seg->c_on_minorcompact_q && !c_seg->c_ondisk && !c_seg->c_on_swapout_q) {
554 queue_enter(&c_minor_list_head, c_seg, c_segment_t, c_list);
555 c_seg->c_on_minorcompact_q = 1;
556 c_minor_count++;
557 }
558 lck_mtx_unlock_always(c_list_lock);
559
560 if (clear_busy == TRUE)
561 C_SEG_WAKEUP_DONE(c_seg);
562 }
563
564
565 unsigned int c_seg_moved_to_sparse_list = 0;
566
567 void
568 c_seg_move_to_sparse_list(c_segment_t c_seg)
569 {
570 boolean_t clear_busy = FALSE;
571
572 if ( !lck_mtx_try_lock_spin_always(c_list_lock)) {
573 C_SEG_BUSY(c_seg);
574
575 lck_mtx_unlock_always(&c_seg->c_lock);
576 lck_mtx_lock_spin_always(c_list_lock);
577 lck_mtx_lock_spin_always(&c_seg->c_lock);
578
579 clear_busy = TRUE;
580 }
581 assert(c_seg->c_ondisk);
582 assert(c_seg->c_on_swappedout_q);
583 assert(!c_seg->c_on_swappedout_sparse_q);
584
585 queue_remove(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
586 c_seg->c_on_swappedout_q = 0;
587 c_swappedout_count--;
588
589 c_seg_insert_into_q(&c_swappedout_sparse_list_head, c_seg);
590 c_seg->c_on_swappedout_sparse_q = 1;
591 c_swappedout_sparse_count++;
592
593 c_seg_moved_to_sparse_list++;
594
595 lck_mtx_unlock_always(c_list_lock);
596
597 if (clear_busy == TRUE)
598 C_SEG_WAKEUP_DONE(c_seg);
599 }
600
601
602 void
603 c_seg_insert_into_q(queue_head_t *qhead, c_segment_t c_seg)
604 {
605 c_segment_t c_seg_next;
606
607 if (queue_empty(qhead)) {
608 queue_enter(qhead, c_seg, c_segment_t, c_age_list);
609 } else {
610 c_seg_next = (c_segment_t)queue_first(qhead);
611
612 while (TRUE) {
613
614 if (c_seg->c_generation_id < c_seg_next->c_generation_id) {
615 queue_insert_before(qhead, c_seg, c_seg_next, c_segment_t, c_age_list);
616 break;
617 }
618 c_seg_next = (c_segment_t) queue_next(&c_seg_next->c_age_list);
619
620 if (queue_end(qhead, (queue_entry_t) c_seg_next)) {
621 queue_enter(qhead, c_seg, c_segment_t, c_age_list);
622 break;
623 }
624 }
625 }
626 }
627
628
629 int try_minor_compaction_failed = 0;
630 int try_minor_compaction_succeeded = 0;
631
632 void
633 c_seg_try_minor_compaction_and_unlock(c_segment_t c_seg)
634 {
635
636 assert(c_seg->c_on_minorcompact_q);
637 /*
638 * c_seg is currently on the delayed minor compaction
639 * queue and we have c_seg locked... if we can get the
640 * c_list_lock w/o blocking (if we blocked we could deadlock
641 * because the lock order is c_list_lock then c_seg's lock)
642 * we'll pull it from the delayed list and free it directly
643 */
644 if ( !lck_mtx_try_lock_spin_always(c_list_lock)) {
645 /*
646 * c_list_lock is held, we need to bail
647 */
648 try_minor_compaction_failed++;
649
650 lck_mtx_unlock_always(&c_seg->c_lock);
651 } else {
652 try_minor_compaction_succeeded++;
653
654 C_SEG_BUSY(c_seg);
655 c_seg_do_minor_compaction_and_unlock(c_seg, TRUE, FALSE, FALSE);
656 }
657 }
658
659
660 int
661 c_seg_do_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy, boolean_t need_list_lock, boolean_t disallow_page_replacement)
662 {
663 int c_seg_freed;
664
665 assert(c_seg->c_busy);
666
667 if (!c_seg->c_on_minorcompact_q) {
668 if (clear_busy == TRUE)
669 C_SEG_WAKEUP_DONE(c_seg);
670
671 lck_mtx_unlock_always(&c_seg->c_lock);
672
673 return (0);
674 }
675 queue_remove(&c_minor_list_head, c_seg, c_segment_t, c_list);
676 c_seg->c_on_minorcompact_q = 0;
677 c_minor_count--;
678
679 lck_mtx_unlock_always(c_list_lock);
680
681 if (disallow_page_replacement == TRUE) {
682 lck_mtx_unlock_always(&c_seg->c_lock);
683
684 PAGE_REPLACEMENT_DISALLOWED(TRUE);
685
686 lck_mtx_lock_spin_always(&c_seg->c_lock);
687 }
688 c_seg_freed = c_seg_minor_compaction_and_unlock(c_seg, clear_busy);
689
690 if (disallow_page_replacement == TRUE)
691 PAGE_REPLACEMENT_DISALLOWED(FALSE);
692
693 if (need_list_lock == TRUE)
694 lck_mtx_lock_spin_always(c_list_lock);
695
696 return (c_seg_freed);
697 }
698
699
700 void
701 c_seg_wait_on_busy(c_segment_t c_seg)
702 {
703 c_seg->c_wanted = 1;
704 assert_wait((event_t) (c_seg), THREAD_UNINT);
705
706 lck_mtx_unlock_always(&c_seg->c_lock);
707 thread_block(THREAD_CONTINUE_NULL);
708 }
709
710
711
712 int try_free_succeeded = 0;
713 int try_free_failed = 0;
714
715 boolean_t
716 c_seg_try_free(c_segment_t c_seg)
717 {
718 /*
719 * c_seg is currently on the delayed minor compaction
720 * or the spapped out sparse queue and we have c_seg locked...
721 * if we can get the c_list_lock w/o blocking (if we blocked we
722 * could deadlock because the lock order is c_list_lock then c_seg's lock)
723 * we'll pull it from the appropriate queue and free it
724 */
725 if ( !lck_mtx_try_lock_spin_always(c_list_lock)) {
726 /*
727 * c_list_lock is held, we need to bail
728 */
729 try_free_failed++;
730 return (FALSE);
731 }
732 if (c_seg->c_on_minorcompact_q) {
733 queue_remove(&c_minor_list_head, c_seg, c_segment_t, c_list);
734 c_seg->c_on_minorcompact_q = 0;
735 c_minor_count--;
736 } else {
737 assert(c_seg->c_on_swappedout_sparse_q);
738
739 /*
740 * c_seg_free_locked will remove it from the swappedout sparse list
741 */
742 }
743 if (!c_seg->c_busy_swapping)
744 C_SEG_BUSY(c_seg);
745
746 c_seg_free_locked(c_seg);
747
748 try_free_succeeded++;
749
750 return (TRUE);
751 }
752
753
754 void
755 c_seg_free(c_segment_t c_seg)
756 {
757 assert(c_seg->c_busy);
758
759 lck_mtx_unlock_always(&c_seg->c_lock);
760 lck_mtx_lock_spin_always(c_list_lock);
761 lck_mtx_lock_spin_always(&c_seg->c_lock);
762
763 c_seg_free_locked(c_seg);
764 }
765
766
767 void
768 c_seg_free_locked(c_segment_t c_seg)
769 {
770 int segno, i;
771 int pages_populated = 0;
772 int32_t *c_buffer = NULL;
773 uint64_t c_swap_handle = 0;
774
775 assert(!c_seg->c_on_minorcompact_q);
776
777 if (c_seg->c_on_age_q) {
778 queue_remove(&c_age_list_head, c_seg, c_segment_t, c_age_list);
779 c_seg->c_on_age_q = 0;
780 c_age_count--;
781 } else if (c_seg->c_on_swappedin_q) {
782 queue_remove(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
783 c_seg->c_on_swappedin_q = 0;
784 c_swappedin_count--;
785 } else if (c_seg->c_on_swapout_q) {
786 queue_remove(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
787 c_seg->c_on_swapout_q = 0;
788 c_swapout_count--;
789 thread_wakeup((event_t)&compaction_swapper_running);
790 } else if (c_seg->c_on_swappedout_q) {
791 queue_remove(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
792 c_seg->c_on_swappedout_q = 0;
793 c_swappedout_count--;
794 } else if (c_seg->c_on_swappedout_sparse_q) {
795 queue_remove(&c_swappedout_sparse_list_head, c_seg, c_segment_t, c_age_list);
796 c_seg->c_on_swappedout_sparse_q = 0;
797 c_swappedout_sparse_count--;
798 }
799 #if TRACK_BAD_C_SEGMENTS
800 else if (c_seg->c_on_bad_q) {
801 queue_remove(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
802 c_seg->c_on_bad_q = 0;
803 c_bad_count--;
804 }
805 #endif
806 segno = c_seg->c_mysegno;
807 c_segments[segno].c_segno = c_free_segno_head;
808 c_free_segno_head = segno;
809 c_segment_count--;
810
811 lck_mtx_unlock_always(c_list_lock);
812
813 if (c_seg->c_wanted) {
814 thread_wakeup((event_t) (c_seg));
815 c_seg->c_wanted = 0;
816 }
817 if (c_seg->c_busy_swapping) {
818 c_seg->c_must_free = 1;
819
820 lck_mtx_unlock_always(&c_seg->c_lock);
821 return;
822 }
823 if (c_seg->c_ondisk == 0) {
824 pages_populated = (round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE;
825
826 c_buffer = c_seg->c_store.c_buffer;
827 c_seg->c_store.c_buffer = NULL;
828 } else {
829 /*
830 * Free swap space on disk.
831 */
832 c_swap_handle = c_seg->c_store.c_swap_handle;
833 c_seg->c_store.c_swap_handle = (uint64_t)-1;
834 }
835 lck_mtx_unlock_always(&c_seg->c_lock);
836
837 if (c_buffer) {
838 if (pages_populated)
839 kernel_memory_depopulate(kernel_map, (vm_offset_t) c_buffer, pages_populated * PAGE_SIZE, KMA_COMPRESSOR);
840
841 kmem_free(kernel_map, (vm_offset_t) c_buffer, C_SEG_ALLOCSIZE);
842 OSAddAtomic64(-C_SEG_ALLOCSIZE, &compressor_kvspace_used);
843
844 } else if (c_swap_handle)
845 vm_swap_free(c_swap_handle);
846
847
848 #if __i386__ || __x86_64__
849 lck_mtx_destroy(&c_seg->c_lock, &vm_compressor_lck_grp);
850 #else /* __i386__ || __x86_64__ */
851 lck_spin_destroy(&c_seg->c_lock, &vm_compressor_lck_grp);
852 #endif /* __i386__ || __x86_64__ */
853
854 for (i = 0; i < C_SEG_SLOT_ARRAYS; i++) {
855 if (c_seg->c_slots[i] == 0)
856 break;
857
858 kfree((char *)c_seg->c_slots[i], sizeof(struct c_slot) * C_SEG_SLOT_ARRAY_SIZE);
859 }
860 zfree(compressor_segment_zone, c_seg);
861 }
862
863
864 int c_seg_trim_page_count = 0;
865
866 void
867 c_seg_trim_tail(c_segment_t c_seg)
868 {
869 c_slot_t cs;
870 uint32_t c_size;
871 uint32_t c_offset;
872 uint32_t c_rounded_size;
873 uint16_t current_nextslot;
874 uint32_t current_populated_offset;
875
876 if (c_seg->c_bytes_used == 0)
877 return;
878 current_nextslot = c_seg->c_nextslot;
879 current_populated_offset = c_seg->c_populated_offset;
880
881 while (c_seg->c_nextslot) {
882
883 cs = C_SEG_SLOT_FROM_INDEX(c_seg, (c_seg->c_nextslot - 1));
884
885 c_size = UNPACK_C_SIZE(cs);
886
887 if (c_size) {
888 if (current_nextslot != c_seg->c_nextslot) {
889 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
890 c_offset = cs->c_offset + C_SEG_BYTES_TO_OFFSET(c_rounded_size);
891
892 c_seg->c_nextoffset = c_offset;
893 c_seg->c_populated_offset = (c_offset + (C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1)) & ~(C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1);
894
895 if (c_seg->c_firstemptyslot > c_seg->c_nextslot)
896 c_seg->c_firstemptyslot = c_seg->c_nextslot;
897
898 c_seg_trim_page_count += ((round_page_32(C_SEG_OFFSET_TO_BYTES(current_populated_offset)) -
899 round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE);
900 }
901 break;
902 }
903 c_seg->c_nextslot--;
904 }
905 assert(c_seg->c_nextslot);
906 }
907
908
909 int
910 c_seg_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy)
911 {
912 c_slot_mapping_t slot_ptr;
913 uint32_t c_offset = 0;
914 uint32_t old_populated_offset;
915 uint32_t c_rounded_size;
916 uint32_t c_size;
917 int c_indx = 0;
918 int i;
919 c_slot_t c_dst;
920 c_slot_t c_src;
921 boolean_t need_unlock = TRUE;
922
923 assert(c_seg->c_busy);
924
925 #if VALIDATE_C_SEGMENTS
926 c_seg_validate(c_seg, FALSE);
927 #endif
928 if (c_seg->c_bytes_used == 0) {
929 c_seg_free(c_seg);
930 return (1);
931 }
932 if (c_seg->c_firstemptyslot >= c_seg->c_nextslot || C_SEG_UNUSED_BYTES(c_seg) < PAGE_SIZE)
933 goto done;
934
935 #if VALIDATE_C_SEGMENTS
936 c_seg->c_was_minor_compacted++;
937 #endif
938 c_indx = c_seg->c_firstemptyslot;
939 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
940
941 old_populated_offset = c_seg->c_populated_offset;
942 c_offset = c_dst->c_offset;
943
944 for (i = c_indx + 1; i < c_seg->c_nextslot && c_offset < c_seg->c_nextoffset; i++) {
945
946 c_src = C_SEG_SLOT_FROM_INDEX(c_seg, i);
947
948 c_size = UNPACK_C_SIZE(c_src);
949
950 if (c_size == 0)
951 continue;
952
953 memcpy(&c_seg->c_store.c_buffer[c_offset], &c_seg->c_store.c_buffer[c_src->c_offset], c_size);
954
955 #if CHECKSUM_THE_DATA
956 c_dst->c_hash_data = c_src->c_hash_data;
957 #endif
958 #if CHECKSUM_THE_COMPRESSED_DATA
959 c_dst->c_hash_compressed_data = c_src->c_hash_compressed_data;
960 #endif
961 c_dst->c_size = c_src->c_size;
962 c_dst->c_packed_ptr = c_src->c_packed_ptr;
963 c_dst->c_offset = c_offset;
964
965 slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
966 slot_ptr->s_cindx = c_indx;
967
968 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
969
970 c_offset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
971 PACK_C_SIZE(c_src, 0);
972 c_indx++;
973
974 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
975 }
976 c_seg->c_firstemptyslot = c_indx;
977 c_seg->c_nextslot = c_indx;
978 c_seg->c_nextoffset = c_offset;
979 c_seg->c_populated_offset = (c_offset + (C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1)) & ~(C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1);
980 c_seg->c_bytes_unused = 0;
981
982 #if VALIDATE_C_SEGMENTS
983 c_seg_validate(c_seg, TRUE);
984 #endif
985
986 if (old_populated_offset > c_seg->c_populated_offset) {
987 uint32_t gc_size;
988 int32_t *gc_ptr;
989
990 gc_size = C_SEG_OFFSET_TO_BYTES(old_populated_offset - c_seg->c_populated_offset);
991 gc_ptr = &c_seg->c_store.c_buffer[c_seg->c_populated_offset];
992
993 lck_mtx_unlock_always(&c_seg->c_lock);
994
995 kernel_memory_depopulate(kernel_map, (vm_offset_t)gc_ptr, gc_size, KMA_COMPRESSOR);
996
997 if (clear_busy == TRUE)
998 lck_mtx_lock_spin_always(&c_seg->c_lock);
999 else
1000 need_unlock = FALSE;
1001 }
1002 done:
1003 if (need_unlock == TRUE) {
1004 if (clear_busy == TRUE)
1005 C_SEG_WAKEUP_DONE(c_seg);
1006
1007 lck_mtx_unlock_always(&c_seg->c_lock);
1008 }
1009 return (0);
1010 }
1011
1012
1013
1014 struct {
1015 uint64_t asked_permission;
1016 uint64_t compactions;
1017 uint64_t moved_slots;
1018 uint64_t moved_bytes;
1019 uint64_t wasted_space_in_swapouts;
1020 uint64_t count_of_swapouts;
1021 } c_seg_major_compact_stats;
1022
1023
1024 #define C_MAJOR_COMPACTION_SIZE_APPROPRIATE ((C_SEG_BUFSIZE * 90) / 100)
1025
1026
1027 boolean_t
1028 c_seg_major_compact_ok(
1029 c_segment_t c_seg_dst,
1030 c_segment_t c_seg_src)
1031 {
1032
1033 c_seg_major_compact_stats.asked_permission++;
1034
1035 if (c_seg_src->c_filling) {
1036 /*
1037 * we're at or near the head... don't compact
1038 */
1039 return (FALSE);
1040 }
1041 if (c_seg_src->c_bytes_used >= C_MAJOR_COMPACTION_SIZE_APPROPRIATE &&
1042 c_seg_dst->c_bytes_used >= C_MAJOR_COMPACTION_SIZE_APPROPRIATE)
1043 return (FALSE);
1044
1045 if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX) {
1046 /*
1047 * destination segment is full... can't compact
1048 */
1049 return (FALSE);
1050 }
1051
1052 return (TRUE);
1053 }
1054
1055
1056 boolean_t
1057 c_seg_major_compact(
1058 c_segment_t c_seg_dst,
1059 c_segment_t c_seg_src)
1060 {
1061 c_slot_mapping_t slot_ptr;
1062 uint32_t c_rounded_size;
1063 uint32_t c_size;
1064 uint16_t dst_slot;
1065 int i;
1066 c_slot_t c_dst;
1067 c_slot_t c_src;
1068 int slotarray;
1069 boolean_t keep_compacting = TRUE;
1070
1071 /*
1072 * segments are not locked but they are both marked c_busy
1073 * which keeps c_decompress from working on them...
1074 * we can safely allocate new pages, move compressed data
1075 * from c_seg_src to c_seg_dst and update both c_segment's
1076 * state w/o holding the master lock
1077 */
1078
1079 #if VALIDATE_C_SEGMENTS
1080 c_seg_dst->c_was_major_compacted++;
1081 c_seg_src->c_was_major_donor++;
1082 #endif
1083 c_seg_major_compact_stats.compactions++;
1084
1085 dst_slot = c_seg_dst->c_nextslot;
1086
1087 for (i = 0; i < c_seg_src->c_nextslot; i++) {
1088
1089 c_src = C_SEG_SLOT_FROM_INDEX(c_seg_src, i);
1090
1091 c_size = UNPACK_C_SIZE(c_src);
1092
1093 if (c_size == 0) {
1094 /* BATCH: move what we have so far; */
1095 continue;
1096 }
1097
1098 if (C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset - c_seg_dst->c_nextoffset) < (unsigned) c_size) {
1099 /* doesn't fit */
1100 if ((C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset) == C_SEG_BUFSIZE)) {
1101 /* can't fit */
1102 keep_compacting = FALSE;
1103 break;
1104 }
1105 kernel_memory_populate(kernel_map,
1106 (vm_offset_t) &c_seg_dst->c_store.c_buffer[c_seg_dst->c_populated_offset],
1107 PAGE_SIZE,
1108 KMA_COMPRESSOR);
1109
1110 c_seg_dst->c_populated_offset += C_SEG_BYTES_TO_OFFSET(PAGE_SIZE);
1111 assert(C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset) <= C_SEG_BUFSIZE);
1112 }
1113
1114 slotarray = C_SEG_SLOTARRAY_FROM_INDEX(c_seg_dst, c_seg_dst->c_nextslot);
1115
1116 if (c_seg_dst->c_slots[slotarray] == 0) {
1117 KERNEL_DEBUG(0xe0400008 | DBG_FUNC_START, 0, 0, 0, 0, 0);
1118 c_seg_dst->c_slots[slotarray] = (struct c_slot *)
1119 kalloc(sizeof(struct c_slot) *
1120 C_SEG_SLOT_ARRAY_SIZE);
1121 KERNEL_DEBUG(0xe0400008 | DBG_FUNC_END, 0, 0, 0, 0, 0);
1122 }
1123 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, c_seg_dst->c_nextslot);
1124
1125 memcpy(&c_seg_dst->c_store.c_buffer[c_seg_dst->c_nextoffset], &c_seg_src->c_store.c_buffer[c_src->c_offset], c_size);
1126
1127 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
1128
1129 c_seg_major_compact_stats.moved_slots++;
1130 c_seg_major_compact_stats.moved_bytes += c_size;
1131
1132 #if CHECKSUM_THE_DATA
1133 c_dst->c_hash_data = c_src->c_hash_data;
1134 #endif
1135 #if CHECKSUM_THE_COMPRESSED_DATA
1136 c_dst->c_hash_compressed_data = c_src->c_hash_compressed_data;
1137 #endif
1138 c_dst->c_size = c_src->c_size;
1139 c_dst->c_packed_ptr = c_src->c_packed_ptr;
1140 c_dst->c_offset = c_seg_dst->c_nextoffset;
1141
1142 if (c_seg_dst->c_firstemptyslot == c_seg_dst->c_nextslot)
1143 c_seg_dst->c_firstemptyslot++;
1144 c_seg_dst->c_nextslot++;
1145 c_seg_dst->c_bytes_used += c_rounded_size;
1146 c_seg_dst->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
1147
1148 PACK_C_SIZE(c_src, 0);
1149
1150 c_seg_src->c_bytes_used -= c_rounded_size;
1151 c_seg_src->c_bytes_unused += c_rounded_size;
1152 c_seg_src->c_firstemptyslot = 0;
1153
1154 if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX) {
1155 /* dest segment is now full */
1156 keep_compacting = FALSE;
1157 break;
1158 }
1159 }
1160 if (dst_slot < c_seg_dst->c_nextslot) {
1161
1162 PAGE_REPLACEMENT_ALLOWED(TRUE);
1163 /*
1164 * we've now locked out c_decompress from
1165 * converting the slot passed into it into
1166 * a c_segment_t which allows us to use
1167 * the backptr to change which c_segment and
1168 * index the slot points to
1169 */
1170 while (dst_slot < c_seg_dst->c_nextslot) {
1171
1172 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, dst_slot);
1173
1174 slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
1175 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
1176 slot_ptr->s_cseg = c_seg_dst->c_mysegno + 1;
1177 slot_ptr->s_cindx = dst_slot++;
1178 }
1179 PAGE_REPLACEMENT_ALLOWED(FALSE);
1180 }
1181 return (keep_compacting);
1182 }
1183
1184
1185 uint64_t
1186 vm_compressor_compute_elapsed_msecs(clock_sec_t end_sec, clock_nsec_t end_nsec, clock_sec_t start_sec, clock_nsec_t start_nsec)
1187 {
1188 uint64_t end_msecs;
1189 uint64_t start_msecs;
1190
1191 end_msecs = (end_sec * 1000) + end_nsec / 1000000;
1192 start_msecs = (start_sec * 1000) + start_nsec / 1000000;
1193
1194 return (end_msecs - start_msecs);
1195 }
1196
1197
1198
1199 uint32_t compressor_eval_period_in_msecs = 250;
1200 uint32_t compressor_sample_min_in_msecs = 500;
1201 uint32_t compressor_sample_max_in_msecs = 10000;
1202 uint32_t compressor_thrashing_threshold_per_10msecs = 50;
1203 uint32_t compressor_thrashing_min_per_10msecs = 20;
1204
1205 /* When true, reset sample data next chance we get. */
1206 static boolean_t compressor_need_sample_reset = FALSE;
1207
1208 extern uint32_t vm_page_filecache_min;
1209
1210
1211 void
1212 compute_swapout_target_age(void)
1213 {
1214 clock_sec_t cur_ts_sec;
1215 clock_nsec_t cur_ts_nsec;
1216 uint32_t min_operations_needed_in_this_sample;
1217 uint64_t elapsed_msecs_in_eval;
1218 uint64_t elapsed_msecs_in_sample;
1219 boolean_t need_eval_reset = FALSE;
1220
1221 clock_get_system_nanotime(&cur_ts_sec, &cur_ts_nsec);
1222
1223 elapsed_msecs_in_sample = vm_compressor_compute_elapsed_msecs(cur_ts_sec, cur_ts_nsec, start_of_sample_period_sec, start_of_sample_period_nsec);
1224
1225 if (compressor_need_sample_reset ||
1226 elapsed_msecs_in_sample >= compressor_sample_max_in_msecs) {
1227 compressor_need_sample_reset = TRUE;
1228 need_eval_reset = TRUE;
1229 goto done;
1230 }
1231 elapsed_msecs_in_eval = vm_compressor_compute_elapsed_msecs(cur_ts_sec, cur_ts_nsec, start_of_eval_period_sec, start_of_eval_period_nsec);
1232
1233 if (elapsed_msecs_in_eval < compressor_eval_period_in_msecs)
1234 goto done;
1235 need_eval_reset = TRUE;
1236
1237 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_START, elapsed_msecs_in_eval, sample_period_compression_count, sample_period_decompression_count, 0, 0);
1238
1239 min_operations_needed_in_this_sample = (compressor_thrashing_min_per_10msecs * (uint32_t)elapsed_msecs_in_eval) / 10;
1240
1241 if ((sample_period_compression_count - last_eval_compression_count) < min_operations_needed_in_this_sample ||
1242 (sample_period_decompression_count - last_eval_decompression_count) < min_operations_needed_in_this_sample) {
1243
1244 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, sample_period_compression_count - last_eval_compression_count,
1245 sample_period_decompression_count - last_eval_decompression_count, 0, 1, 0);
1246
1247 swapout_target_age = 0;
1248
1249 compressor_need_sample_reset = TRUE;
1250 need_eval_reset = TRUE;
1251 goto done;
1252 }
1253 last_eval_compression_count = sample_period_compression_count;
1254 last_eval_decompression_count = sample_period_decompression_count;
1255
1256 if (elapsed_msecs_in_sample < compressor_sample_min_in_msecs) {
1257
1258 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, swapout_target_age, 0, 0, 5, 0);
1259 goto done;
1260 }
1261 if (sample_period_decompression_count > ((compressor_thrashing_threshold_per_10msecs * elapsed_msecs_in_sample) / 10)) {
1262
1263 uint64_t running_total;
1264 uint64_t working_target;
1265 uint64_t aging_target;
1266 uint32_t oldest_age_of_csegs_sampled = 0;
1267 uint64_t working_set_approximation = 0;
1268
1269 swapout_target_age = 0;
1270
1271 working_target = (sample_period_decompression_count / 100) * 95; /* 95 percent */
1272 aging_target = (sample_period_decompression_count / 100) * 1; /* 1 percent */
1273 running_total = 0;
1274
1275 for (oldest_age_of_csegs_sampled = 0; oldest_age_of_csegs_sampled < DECOMPRESSION_SAMPLE_MAX_AGE; oldest_age_of_csegs_sampled++) {
1276
1277 running_total += age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
1278
1279 working_set_approximation += oldest_age_of_csegs_sampled * age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
1280
1281 if (running_total >= working_target)
1282 break;
1283 }
1284 if (oldest_age_of_csegs_sampled < DECOMPRESSION_SAMPLE_MAX_AGE) {
1285
1286 working_set_approximation = (working_set_approximation * 1000) / elapsed_msecs_in_sample;
1287
1288 if (working_set_approximation < VM_PAGE_COMPRESSOR_COUNT) {
1289
1290 running_total = overage_decompressions_during_sample_period;
1291
1292 for (oldest_age_of_csegs_sampled = DECOMPRESSION_SAMPLE_MAX_AGE - 1; oldest_age_of_csegs_sampled; oldest_age_of_csegs_sampled--) {
1293 running_total += age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
1294
1295 if (running_total >= aging_target)
1296 break;
1297 }
1298 swapout_target_age = (uint32_t)cur_ts_sec - oldest_age_of_csegs_sampled;
1299
1300 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, swapout_target_age, working_set_approximation, VM_PAGE_COMPRESSOR_COUNT, 2, 0);
1301 } else {
1302 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, working_set_approximation, VM_PAGE_COMPRESSOR_COUNT, 0, 3, 0);
1303 }
1304 } else
1305 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, working_target, running_total, 0, 4, 0);
1306
1307 compressor_need_sample_reset = TRUE;
1308 need_eval_reset = TRUE;
1309 } else
1310 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, sample_period_decompression_count, (compressor_thrashing_threshold_per_10msecs * elapsed_msecs_in_sample) / 10, 0, 6, 0);
1311 done:
1312 if (compressor_need_sample_reset == TRUE) {
1313 bzero(age_of_decompressions_during_sample_period, sizeof(age_of_decompressions_during_sample_period));
1314 overage_decompressions_during_sample_period = 0;
1315
1316 start_of_sample_period_sec = cur_ts_sec;
1317 start_of_sample_period_nsec = cur_ts_nsec;
1318 sample_period_decompression_count = 0;
1319 sample_period_compression_count = 0;
1320 last_eval_decompression_count = 0;
1321 last_eval_compression_count = 0;
1322 compressor_need_sample_reset = FALSE;
1323 }
1324 if (need_eval_reset == TRUE) {
1325 start_of_eval_period_sec = cur_ts_sec;
1326 start_of_eval_period_nsec = cur_ts_nsec;
1327 }
1328 }
1329
1330
1331 int compaction_swapper_inited = 0;
1332 int compaction_swapper_init_now = 0;
1333 int compaction_swapper_running = 0;
1334 int compaction_swapper_abort = 0;
1335
1336
1337 #if CONFIG_JETSAM
1338 boolean_t memorystatus_kill_on_VM_thrashing(boolean_t);
1339 boolean_t memorystatus_kill_on_FC_thrashing(boolean_t);
1340 int compressor_thrashing_induced_jetsam = 0;
1341 int filecache_thrashing_induced_jetsam = 0;
1342 static boolean_t vm_compressor_thrashing_detected = FALSE;
1343 #endif /* CONFIG_JETSAM */
1344
1345 static boolean_t
1346 compressor_needs_to_swap(void)
1347 {
1348 boolean_t should_swap = FALSE;
1349
1350 if (vm_swap_up == TRUE) {
1351 if (COMPRESSOR_NEEDS_TO_SWAP()) {
1352 return (TRUE);
1353 }
1354 if (VM_PAGE_Q_THROTTLED(&vm_pageout_queue_external) && vm_page_anonymous_count < (vm_page_inactive_count / 20)) {
1355 return (TRUE);
1356 }
1357 if (vm_page_free_count < (vm_page_free_reserved - COMPRESSOR_FREE_RESERVED_LIMIT))
1358 return (TRUE);
1359 }
1360 compute_swapout_target_age();
1361
1362 if (swapout_target_age) {
1363 c_segment_t c_seg;
1364
1365 lck_mtx_lock_spin_always(c_list_lock);
1366
1367 if (!queue_empty(&c_age_list_head)) {
1368
1369 c_seg = (c_segment_t) queue_first(&c_age_list_head);
1370
1371 if (c_seg->c_creation_ts > swapout_target_age)
1372 swapout_target_age = 0;
1373 }
1374 lck_mtx_unlock_always(c_list_lock);
1375 }
1376 #if CONFIG_PHANTOM_CACHE
1377 if (vm_phantom_cache_check_pressure())
1378 should_swap = TRUE;
1379 #endif
1380 if (swapout_target_age)
1381 should_swap = TRUE;
1382
1383 if (vm_swap_up == FALSE) {
1384
1385 if (should_swap) {
1386 #if CONFIG_JETSAM
1387 if (vm_compressor_thrashing_detected == FALSE) {
1388 vm_compressor_thrashing_detected = TRUE;
1389
1390 if (swapout_target_age) {
1391 memorystatus_kill_on_VM_thrashing(TRUE /* async */);
1392 compressor_thrashing_induced_jetsam++;
1393 } else {
1394 memorystatus_kill_on_FC_thrashing(TRUE /* async */);
1395 filecache_thrashing_induced_jetsam++;
1396 }
1397 /*
1398 * let the jetsam take precedence over
1399 * any major compactions we might have
1400 * been able to do... otherwise we run
1401 * the risk of doing major compactions
1402 * on segments we're about to free up
1403 * due to the jetsam activity.
1404 */
1405 should_swap = FALSE;
1406 }
1407 #endif /* CONFIG_JETSAM */
1408 } else
1409 should_swap = COMPRESSOR_NEEDS_TO_MAJOR_COMPACT();
1410 }
1411
1412 /*
1413 * returning TRUE when swap_supported == FALSE
1414 * will cause the major compaction engine to
1415 * run, but will not trigger any swapping...
1416 * segments that have been major compacted
1417 * will be moved to the swapped_out_q
1418 * but will not have the c_ondisk flag set
1419 */
1420 return (should_swap);
1421 }
1422
1423 #if CONFIG_JETSAM
1424 /*
1425 * This function is called from the jetsam thread after killing something to
1426 * mitigate thrashing.
1427 *
1428 * We need to restart our thrashing detection heuristics since memory pressure
1429 * has potentially changed significantly, and we don't want to detect on old
1430 * data from before the jetsam.
1431 */
1432 void
1433 vm_thrashing_jetsam_done(void)
1434 {
1435 vm_compressor_thrashing_detected = FALSE;
1436
1437 /* Were we compressor-thrashing or filecache-thrashing? */
1438 if (swapout_target_age) {
1439 swapout_target_age = 0;
1440 compressor_need_sample_reset = TRUE;
1441 }
1442 #if CONFIG_PHANTOM_CACHE
1443 else {
1444 vm_phantom_cache_restart_sample();
1445 }
1446 #endif
1447 }
1448 #endif /* CONFIG_JETSAM */
1449
1450 uint32_t vm_wake_compactor_swapper_calls = 0;
1451
1452 void
1453 vm_wake_compactor_swapper(void)
1454 {
1455 boolean_t need_major_compaction = FALSE;
1456
1457 if (compaction_swapper_running)
1458 return;
1459
1460 if (c_minor_count == 0 && need_major_compaction == FALSE)
1461 return;
1462
1463 lck_mtx_lock_spin_always(c_list_lock);
1464
1465 fastwake_warmup = FALSE;
1466
1467 if (compaction_swapper_running == 0) {
1468 vm_wake_compactor_swapper_calls++;
1469
1470 thread_wakeup((event_t)&c_compressor_swap_trigger);
1471
1472 compaction_swapper_running = 1;
1473 }
1474 lck_mtx_unlock_always(c_list_lock);
1475 }
1476
1477
1478 void
1479 vm_consider_waking_compactor_swapper(void)
1480 {
1481 boolean_t need_wakeup = FALSE;
1482
1483 if (compaction_swapper_running)
1484 return;
1485
1486 if (!compaction_swapper_inited && !compaction_swapper_init_now) {
1487 compaction_swapper_init_now = 1;
1488 need_wakeup = TRUE;
1489 }
1490
1491 if (c_minor_count && (COMPRESSOR_NEEDS_TO_MINOR_COMPACT())) {
1492
1493 need_wakeup = TRUE;
1494
1495 } else if (compressor_needs_to_swap()) {
1496
1497 need_wakeup = TRUE;
1498
1499 } else if (c_minor_count) {
1500 uint64_t total_bytes;
1501
1502 total_bytes = compressor_object->resident_page_count * PAGE_SIZE_64;
1503
1504 if ((total_bytes - compressor_bytes_used) > total_bytes / 10)
1505 need_wakeup = TRUE;
1506 }
1507 if (need_wakeup == TRUE) {
1508
1509 lck_mtx_lock_spin_always(c_list_lock);
1510
1511 fastwake_warmup = FALSE;
1512
1513 if (compaction_swapper_running == 0) {
1514 memoryshot(VM_WAKEUP_COMPACTOR_SWAPPER, DBG_FUNC_NONE);
1515
1516 thread_wakeup((event_t)&c_compressor_swap_trigger);
1517
1518 compaction_swapper_running = 1;
1519 }
1520 lck_mtx_unlock_always(c_list_lock);
1521 }
1522 }
1523
1524
1525 #define C_SWAPOUT_LIMIT 4
1526 #define DELAYED_COMPACTIONS_PER_PASS 30
1527
1528 void
1529 vm_compressor_do_delayed_compactions(boolean_t flush_all)
1530 {
1531 c_segment_t c_seg;
1532 int number_compacted = 0;
1533 boolean_t needs_to_swap = FALSE;
1534
1535
1536 lck_mtx_assert(c_list_lock, LCK_MTX_ASSERT_OWNED);
1537
1538 while (!queue_empty(&c_minor_list_head) && needs_to_swap == FALSE) {
1539
1540 c_seg = (c_segment_t)queue_first(&c_minor_list_head);
1541
1542 lck_mtx_lock_spin_always(&c_seg->c_lock);
1543
1544 if (c_seg->c_busy) {
1545
1546 lck_mtx_unlock_always(c_list_lock);
1547 c_seg_wait_on_busy(c_seg);
1548 lck_mtx_lock_spin_always(c_list_lock);
1549
1550 continue;
1551 }
1552 C_SEG_BUSY(c_seg);
1553
1554 c_seg_do_minor_compaction_and_unlock(c_seg, TRUE, FALSE, TRUE);
1555
1556 if (vm_swap_up == TRUE && (number_compacted++ > DELAYED_COMPACTIONS_PER_PASS)) {
1557
1558 if ((flush_all == TRUE || compressor_needs_to_swap() == TRUE) && c_swapout_count < C_SWAPOUT_LIMIT)
1559 needs_to_swap = TRUE;
1560
1561 number_compacted = 0;
1562 }
1563 lck_mtx_lock_spin_always(c_list_lock);
1564 }
1565 }
1566
1567
1568 #define C_SEGMENT_SWAPPEDIN_AGE_LIMIT 10
1569
1570 static void
1571 vm_compressor_age_swapped_in_segments(boolean_t flush_all)
1572 {
1573 c_segment_t c_seg;
1574 clock_sec_t now;
1575 clock_nsec_t nsec;
1576
1577 clock_get_system_nanotime(&now, &nsec);
1578
1579 while (!queue_empty(&c_swappedin_list_head)) {
1580
1581 c_seg = (c_segment_t)queue_first(&c_swappedin_list_head);
1582
1583 if (flush_all == FALSE && (now - c_seg->c_swappedin_ts) < C_SEGMENT_SWAPPEDIN_AGE_LIMIT)
1584 break;
1585
1586 lck_mtx_lock_spin_always(&c_seg->c_lock);
1587
1588 queue_remove(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
1589 c_seg->c_on_swappedin_q = 0;
1590 c_swappedin_count--;
1591
1592 c_seg_insert_into_q(&c_age_list_head, c_seg);
1593 c_seg->c_on_age_q = 1;
1594 c_age_count++;
1595
1596 lck_mtx_unlock_always(&c_seg->c_lock);
1597 }
1598 }
1599
1600
1601 void
1602 vm_compressor_flush(void)
1603 {
1604 uint64_t vm_swap_put_failures_at_start;
1605 wait_result_t wait_result = 0;
1606 AbsoluteTime startTime, endTime;
1607 clock_sec_t now_sec;
1608 clock_nsec_t now_nsec;
1609 uint64_t nsec;
1610
1611 HIBLOG("vm_compressor_flush - starting\n");
1612
1613 clock_get_uptime(&startTime);
1614
1615 lck_mtx_lock_spin_always(c_list_lock);
1616
1617 fastwake_warmup = FALSE;
1618 compaction_swapper_abort = 1;
1619
1620 while (compaction_swapper_running) {
1621 assert_wait((event_t)&compaction_swapper_running, THREAD_UNINT);
1622
1623 lck_mtx_unlock_always(c_list_lock);
1624
1625 thread_block(THREAD_CONTINUE_NULL);
1626
1627 lck_mtx_lock_spin_always(c_list_lock);
1628 }
1629 compaction_swapper_abort = 0;
1630 compaction_swapper_running = 1;
1631
1632 hibernate_flushing = TRUE;
1633 hibernate_no_swapspace = FALSE;
1634 c_generation_id_flush_barrier = c_generation_id + 1000;
1635
1636 clock_get_system_nanotime(&now_sec, &now_nsec);
1637 hibernate_flushing_deadline = now_sec + HIBERNATE_FLUSHING_SECS_TO_COMPLETE;
1638
1639 vm_swap_put_failures_at_start = vm_swap_put_failures;
1640
1641 vm_compressor_compact_and_swap(TRUE);
1642
1643 while (!queue_empty(&c_swapout_list_head)) {
1644
1645 assert_wait_timeout((event_t) &compaction_swapper_running, THREAD_INTERRUPTIBLE, 5000, 1000*NSEC_PER_USEC);
1646
1647 lck_mtx_unlock_always(c_list_lock);
1648
1649 wait_result = thread_block(THREAD_CONTINUE_NULL);
1650
1651 lck_mtx_lock_spin_always(c_list_lock);
1652
1653 if (wait_result == THREAD_TIMED_OUT)
1654 break;
1655 }
1656 hibernate_flushing = FALSE;
1657 compaction_swapper_running = 0;
1658
1659 if (vm_swap_put_failures > vm_swap_put_failures_at_start)
1660 HIBLOG("vm_compressor_flush failed to clean %llu segments - vm_page_compressor_count(%d)\n",
1661 vm_swap_put_failures - vm_swap_put_failures_at_start, VM_PAGE_COMPRESSOR_COUNT);
1662
1663 lck_mtx_unlock_always(c_list_lock);
1664
1665 clock_get_uptime(&endTime);
1666 SUB_ABSOLUTETIME(&endTime, &startTime);
1667 absolutetime_to_nanoseconds(endTime, &nsec);
1668
1669 HIBLOG("vm_compressor_flush completed - took %qd msecs\n", nsec / 1000000ULL);
1670 }
1671
1672
1673 extern void vm_swap_file_set_tuneables(void);
1674 int compaction_swap_trigger_thread_awakened = 0;
1675
1676
1677 static void
1678 vm_compressor_swap_trigger_thread(void)
1679 {
1680 /*
1681 * compaction_swapper_init_now is set when the first call to
1682 * vm_consider_waking_compactor_swapper is made from
1683 * vm_pageout_scan... since this function is called upon
1684 * thread creation, we want to make sure to delay adjusting
1685 * the tuneables until we are awakened via vm_pageout_scan
1686 * so that we are at a point where the vm_swapfile_open will
1687 * be operating on the correct directory (in case the default
1688 * of /var/vm/ is overridden by the dymanic_pager
1689 */
1690 if (compaction_swapper_init_now && !compaction_swapper_inited) {
1691 if (vm_compressor_mode == VM_PAGER_COMPRESSOR_WITH_SWAP)
1692 vm_swap_file_set_tuneables();
1693
1694 compaction_swapper_inited = 1;
1695 }
1696 lck_mtx_lock_spin_always(c_list_lock);
1697
1698 compaction_swap_trigger_thread_awakened++;
1699
1700 vm_compressor_compact_and_swap(FALSE);
1701
1702 assert_wait((event_t)&c_compressor_swap_trigger, THREAD_UNINT);
1703
1704 compaction_swapper_running = 0;
1705 thread_wakeup((event_t)&compaction_swapper_running);
1706
1707 lck_mtx_unlock_always(c_list_lock);
1708
1709 thread_block((thread_continue_t)vm_compressor_swap_trigger_thread);
1710
1711 /* NOTREACHED */
1712 }
1713
1714
1715 void
1716 vm_compressor_record_warmup_start(void)
1717 {
1718 c_segment_t c_seg;
1719
1720 lck_mtx_lock_spin_always(c_list_lock);
1721
1722 if (first_c_segment_to_warm_generation_id == 0) {
1723 if (!queue_empty(&c_age_list_head)) {
1724
1725 c_seg = (c_segment_t)queue_last(&c_age_list_head);
1726
1727 first_c_segment_to_warm_generation_id = c_seg->c_generation_id;
1728 } else
1729 first_c_segment_to_warm_generation_id = 0;
1730
1731 fastwake_recording_in_progress = TRUE;
1732 }
1733 lck_mtx_unlock_always(c_list_lock);
1734 }
1735
1736
1737 void
1738 vm_compressor_record_warmup_end(void)
1739 {
1740 c_segment_t c_seg;
1741
1742 lck_mtx_lock_spin_always(c_list_lock);
1743
1744 if (fastwake_recording_in_progress == TRUE) {
1745
1746 if (!queue_empty(&c_age_list_head)) {
1747
1748 c_seg = (c_segment_t)queue_last(&c_age_list_head);
1749
1750 last_c_segment_to_warm_generation_id = c_seg->c_generation_id;
1751 } else
1752 last_c_segment_to_warm_generation_id = first_c_segment_to_warm_generation_id;
1753
1754 fastwake_recording_in_progress = FALSE;
1755
1756 HIBLOG("vm_compressor_record_warmup (%qd - %qd)\n", first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id);
1757 }
1758 lck_mtx_unlock_always(c_list_lock);
1759 }
1760
1761
1762 #define DELAY_TRIM_ON_WAKE_SECS 4
1763
1764 void
1765 vm_compressor_delay_trim(void)
1766 {
1767 clock_sec_t sec;
1768 clock_nsec_t nsec;
1769
1770 clock_get_system_nanotime(&sec, &nsec);
1771 dont_trim_until_ts = sec + DELAY_TRIM_ON_WAKE_SECS;
1772 }
1773
1774
1775 void
1776 vm_compressor_do_warmup(void)
1777 {
1778 lck_mtx_lock_spin_always(c_list_lock);
1779
1780 if (first_c_segment_to_warm_generation_id == last_c_segment_to_warm_generation_id) {
1781 first_c_segment_to_warm_generation_id = last_c_segment_to_warm_generation_id = 0;
1782
1783 lck_mtx_unlock_always(c_list_lock);
1784 return;
1785 }
1786
1787 if (compaction_swapper_running == 0) {
1788
1789 fastwake_warmup = TRUE;
1790 compaction_swapper_running = 1;
1791 thread_wakeup((event_t)&c_compressor_swap_trigger);
1792 }
1793 lck_mtx_unlock_always(c_list_lock);
1794 }
1795
1796
1797 void
1798 do_fastwake_warmup(void)
1799 {
1800 uint64_t my_thread_id;
1801 c_segment_t c_seg = NULL;
1802 AbsoluteTime startTime, endTime;
1803 uint64_t nsec;
1804
1805
1806 HIBLOG("vm_compressor_fastwake_warmup (%qd - %qd) - starting\n", first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id);
1807
1808 clock_get_uptime(&startTime);
1809
1810 lck_mtx_unlock_always(c_list_lock);
1811
1812 my_thread_id = current_thread()->thread_id;
1813 proc_set_task_policy_thread(kernel_task, my_thread_id,
1814 TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER2);
1815
1816 PAGE_REPLACEMENT_DISALLOWED(TRUE);
1817
1818 lck_mtx_lock_spin_always(c_list_lock);
1819
1820 while (!queue_empty(&c_swappedout_list_head) && fastwake_warmup == TRUE) {
1821
1822 c_seg = (c_segment_t) queue_first(&c_swappedout_list_head);
1823
1824 if (c_seg->c_generation_id < first_c_segment_to_warm_generation_id ||
1825 c_seg->c_generation_id > last_c_segment_to_warm_generation_id)
1826 break;
1827
1828 lck_mtx_lock_spin_always(&c_seg->c_lock);
1829 lck_mtx_unlock_always(c_list_lock);
1830
1831 if (c_seg->c_busy) {
1832 PAGE_REPLACEMENT_DISALLOWED(FALSE);
1833 c_seg_wait_on_busy(c_seg);
1834 PAGE_REPLACEMENT_DISALLOWED(TRUE);
1835 } else {
1836 c_seg_swapin(c_seg, TRUE);
1837
1838 lck_mtx_unlock_always(&c_seg->c_lock);
1839 c_segment_warmup_count++;
1840
1841 PAGE_REPLACEMENT_DISALLOWED(FALSE);
1842 vm_pageout_io_throttle();
1843 PAGE_REPLACEMENT_DISALLOWED(TRUE);
1844 }
1845 lck_mtx_lock_spin_always(c_list_lock);
1846 }
1847 lck_mtx_unlock_always(c_list_lock);
1848
1849 PAGE_REPLACEMENT_DISALLOWED(FALSE);
1850
1851 proc_set_task_policy_thread(kernel_task, my_thread_id,
1852 TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER0);
1853
1854 clock_get_uptime(&endTime);
1855 SUB_ABSOLUTETIME(&endTime, &startTime);
1856 absolutetime_to_nanoseconds(endTime, &nsec);
1857
1858 HIBLOG("vm_compressor_fastwake_warmup completed - took %qd msecs\n", nsec / 1000000ULL);
1859
1860 lck_mtx_lock_spin_always(c_list_lock);
1861
1862 first_c_segment_to_warm_generation_id = last_c_segment_to_warm_generation_id = 0;
1863 }
1864
1865
1866 void
1867 vm_compressor_compact_and_swap(boolean_t flush_all)
1868 {
1869 c_segment_t c_seg, c_seg_next;
1870 boolean_t keep_compacting;
1871
1872
1873 if (fastwake_warmup == TRUE) {
1874 uint64_t starting_warmup_count;
1875
1876 starting_warmup_count = c_segment_warmup_count;
1877
1878 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 11) | DBG_FUNC_START, c_segment_warmup_count,
1879 first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id, 0, 0);
1880 do_fastwake_warmup();
1881 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 11) | DBG_FUNC_END, c_segment_warmup_count, c_segment_warmup_count - starting_warmup_count, 0, 0, 0);
1882
1883 fastwake_warmup = FALSE;
1884 }
1885
1886 /*
1887 * it's possible for the c_age_list_head to be empty if we
1888 * hit our limits for growing the compressor pool and we subsequently
1889 * hibernated... on the next hibernation we could see the queue as
1890 * empty and not proceeed even though we have a bunch of segments on
1891 * the swapped in queue that need to be dealt with.
1892 */
1893 vm_compressor_do_delayed_compactions(flush_all);
1894
1895 vm_compressor_age_swapped_in_segments(flush_all);
1896
1897
1898 while (!queue_empty(&c_age_list_head) && compaction_swapper_abort == 0) {
1899
1900 if (hibernate_flushing == TRUE) {
1901 clock_sec_t sec;
1902 clock_nsec_t nsec;
1903
1904 if (hibernate_should_abort()) {
1905 HIBLOG("vm_compressor_flush - hibernate_should_abort returned TRUE\n");
1906 break;
1907 }
1908 if (hibernate_no_swapspace == TRUE) {
1909 HIBLOG("vm_compressor_flush - out of swap space\n");
1910 break;
1911 }
1912 clock_get_system_nanotime(&sec, &nsec);
1913
1914 if (sec > hibernate_flushing_deadline) {
1915 HIBLOG("vm_compressor_flush - failed to finish before deadline\n");
1916 break;
1917 }
1918 }
1919 if (c_swapout_count >= C_SWAPOUT_LIMIT) {
1920
1921 assert_wait_timeout((event_t) &compaction_swapper_running, THREAD_INTERRUPTIBLE, 100, 1000*NSEC_PER_USEC);
1922
1923 lck_mtx_unlock_always(c_list_lock);
1924
1925 thread_block(THREAD_CONTINUE_NULL);
1926
1927 lck_mtx_lock_spin_always(c_list_lock);
1928 }
1929 /*
1930 * Minor compactions
1931 */
1932 vm_compressor_do_delayed_compactions(flush_all);
1933
1934 vm_compressor_age_swapped_in_segments(flush_all);
1935
1936 if (c_swapout_count >= C_SWAPOUT_LIMIT) {
1937 /*
1938 * we timed out on the above thread_block
1939 * let's loop around and try again
1940 * the timeout allows us to continue
1941 * to do minor compactions to make
1942 * more memory available
1943 */
1944 continue;
1945 }
1946
1947 /*
1948 * Swap out segments?
1949 */
1950 if (flush_all == FALSE) {
1951 boolean_t needs_to_swap;
1952
1953 lck_mtx_unlock_always(c_list_lock);
1954
1955 needs_to_swap = compressor_needs_to_swap();
1956
1957 lck_mtx_lock_spin_always(c_list_lock);
1958
1959 if (needs_to_swap == FALSE)
1960 break;
1961 }
1962 if (queue_empty(&c_age_list_head))
1963 break;
1964 c_seg = (c_segment_t) queue_first(&c_age_list_head);
1965
1966 if (flush_all == TRUE && c_seg->c_generation_id > c_generation_id_flush_barrier)
1967 break;
1968
1969 if (c_seg->c_filling) {
1970 /*
1971 * we're at or near the head... no more work to do
1972 */
1973 break;
1974 }
1975 lck_mtx_lock_spin_always(&c_seg->c_lock);
1976
1977 if (c_seg->c_busy) {
1978
1979 lck_mtx_unlock_always(c_list_lock);
1980 c_seg_wait_on_busy(c_seg);
1981 lck_mtx_lock_spin_always(c_list_lock);
1982
1983 continue;
1984 }
1985 C_SEG_BUSY(c_seg);
1986
1987 if (c_seg_do_minor_compaction_and_unlock(c_seg, FALSE, TRUE, TRUE)) {
1988 /*
1989 * found an empty c_segment and freed it
1990 * so go grab the next guy in the queue
1991 */
1992 continue;
1993 }
1994 /*
1995 * Major compaction
1996 */
1997 keep_compacting = TRUE;
1998
1999 while (keep_compacting == TRUE) {
2000
2001 assert(c_seg->c_busy);
2002
2003 /* look for another segment to consolidate */
2004
2005 c_seg_next = (c_segment_t) queue_next(&c_seg->c_age_list);
2006
2007 if (queue_end(&c_age_list_head, (queue_entry_t)c_seg_next))
2008 break;
2009
2010 if (c_seg_major_compact_ok(c_seg, c_seg_next) == FALSE)
2011 break;
2012
2013 lck_mtx_lock_spin_always(&c_seg_next->c_lock);
2014
2015 if (c_seg_next->c_busy) {
2016
2017 lck_mtx_unlock_always(c_list_lock);
2018 c_seg_wait_on_busy(c_seg_next);
2019 lck_mtx_lock_spin_always(c_list_lock);
2020
2021 continue;
2022 }
2023 /* grab that segment */
2024 C_SEG_BUSY(c_seg_next);
2025
2026 if (c_seg_do_minor_compaction_and_unlock(c_seg_next, FALSE, TRUE, TRUE)) {
2027 /*
2028 * found an empty c_segment and freed it
2029 * so we can't continue to use c_seg_next
2030 */
2031 continue;
2032 }
2033
2034 /* unlock the list ... */
2035 lck_mtx_unlock_always(c_list_lock);
2036
2037 /* do the major compaction */
2038
2039 keep_compacting = c_seg_major_compact(c_seg, c_seg_next);
2040
2041 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2042
2043 lck_mtx_lock_spin_always(&c_seg_next->c_lock);
2044 /*
2045 * run a minor compaction on the donor segment
2046 * since we pulled at least some of it's
2047 * data into our target... if we've emptied
2048 * it, now is a good time to free it which
2049 * c_seg_minor_compaction_and_unlock also takes care of
2050 *
2051 * by passing TRUE, we ask for c_busy to be cleared
2052 * and c_wanted to be taken care of
2053 */
2054 c_seg_minor_compaction_and_unlock(c_seg_next, TRUE);
2055
2056 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2057
2058 /* relock the list */
2059 lck_mtx_lock_spin_always(c_list_lock);
2060
2061 } /* major compaction */
2062
2063 c_seg_major_compact_stats.wasted_space_in_swapouts += C_SEG_BUFSIZE - c_seg->c_bytes_used;
2064 c_seg_major_compact_stats.count_of_swapouts++;
2065
2066 lck_mtx_lock_spin_always(&c_seg->c_lock);
2067
2068 assert(c_seg->c_busy);
2069 assert(c_seg->c_on_age_q);
2070 assert(!c_seg->c_on_minorcompact_q);
2071
2072 queue_remove(&c_age_list_head, c_seg, c_segment_t, c_age_list);
2073 c_seg->c_on_age_q = 0;
2074 c_age_count--;
2075
2076 if (vm_swap_up == TRUE) {
2077 queue_enter(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
2078 c_seg->c_on_swapout_q = 1;
2079 c_swapout_count++;
2080 } else {
2081 queue_enter(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
2082 c_seg->c_on_swappedout_q = 1;
2083 c_swappedout_count++;
2084 }
2085 C_SEG_WAKEUP_DONE(c_seg);
2086
2087 lck_mtx_unlock_always(&c_seg->c_lock);
2088
2089 if (c_swapout_count) {
2090 lck_mtx_unlock_always(c_list_lock);
2091
2092 thread_wakeup((event_t)&c_swapout_list_head);
2093
2094 lck_mtx_lock_spin_always(c_list_lock);
2095 }
2096 }
2097 }
2098
2099
2100 static uint32_t no_paging_space_action_in_progress = 0;
2101 extern void memorystatus_send_low_swap_note(void);
2102
2103
2104 static c_segment_t
2105 c_seg_allocate(c_segment_t *current_chead)
2106 {
2107 clock_sec_t sec;
2108 clock_nsec_t nsec;
2109 c_segment_t c_seg;
2110 int slotarray;
2111
2112 if ( (c_seg = *current_chead) == NULL ) {
2113 uint32_t c_segno;
2114
2115 if (vm_compressor_low_on_space() || vm_swap_low_on_space()) {
2116
2117 if (no_paging_space_action_in_progress == 0) {
2118
2119 if (OSCompareAndSwap(0, 1, (UInt32 *)&no_paging_space_action_in_progress)) {
2120
2121 if (no_paging_space_action()) {
2122 memorystatus_send_low_swap_note();
2123 }
2124
2125 no_paging_space_action_in_progress = 0;
2126 }
2127 }
2128 }
2129 KERNEL_DEBUG(0xe0400004 | DBG_FUNC_START, 0, 0, 0, 0, 0);
2130
2131 lck_mtx_lock_spin_always(c_list_lock);
2132
2133 while (c_segments_busy == TRUE) {
2134 assert_wait((event_t) (&c_segments_busy), THREAD_UNINT);
2135
2136 lck_mtx_unlock_always(c_list_lock);
2137
2138 thread_block(THREAD_CONTINUE_NULL);
2139
2140 lck_mtx_lock_spin_always(c_list_lock);
2141 }
2142 if (c_free_segno_head == (uint32_t)-1) {
2143
2144 if (c_segments_available >= c_segments_limit || c_segment_pages_compressed >= c_segment_pages_compressed_limit) {
2145 lck_mtx_unlock_always(c_list_lock);
2146
2147 KERNEL_DEBUG(0xe0400004 | DBG_FUNC_END, 0, 0, 0, 1, 0);
2148 return (NULL);
2149 }
2150 c_segments_busy = TRUE;
2151 lck_mtx_unlock_always(c_list_lock);
2152
2153 kernel_memory_populate(kernel_map, (vm_offset_t)c_segments_next_page, PAGE_SIZE, KMA_KOBJECT);
2154 c_segments_next_page += PAGE_SIZE;
2155
2156 for (c_segno = c_segments_available + 1; c_segno < (c_segments_available + C_SEGMENTS_PER_PAGE); c_segno++)
2157 c_segments[c_segno - 1].c_segno = c_segno;
2158
2159 lck_mtx_lock_spin_always(c_list_lock);
2160
2161 c_segments[c_segno - 1].c_segno = c_free_segno_head;
2162 c_free_segno_head = c_segments_available;
2163 c_segments_available += C_SEGMENTS_PER_PAGE;
2164
2165 c_segments_busy = FALSE;
2166 thread_wakeup((event_t) (&c_segments_busy));
2167 }
2168 c_segno = c_free_segno_head;
2169 c_free_segno_head = c_segments[c_segno].c_segno;
2170
2171 lck_mtx_unlock_always(c_list_lock);
2172
2173 c_seg = (c_segment_t)zalloc(compressor_segment_zone);
2174 bzero((char *)c_seg, sizeof(struct c_segment));
2175
2176 if (kernel_memory_allocate(kernel_map, (vm_offset_t *)(&c_seg->c_store.c_buffer), C_SEG_ALLOCSIZE, 0, KMA_COMPRESSOR | KMA_VAONLY) != KERN_SUCCESS) {
2177 zfree(compressor_segment_zone, c_seg);
2178
2179 lck_mtx_lock_spin_always(c_list_lock);
2180
2181 c_segments[c_segno].c_segno = c_free_segno_head;
2182 c_free_segno_head = c_segno;
2183
2184 lck_mtx_unlock_always(c_list_lock);
2185
2186 KERNEL_DEBUG(0xe0400004 | DBG_FUNC_END, 0, 0, 0, 2, 0);
2187
2188 return (NULL);
2189 }
2190 OSAddAtomic64(C_SEG_ALLOCSIZE, &compressor_kvspace_used);
2191
2192 #if __i386__ || __x86_64__
2193 lck_mtx_init(&c_seg->c_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
2194 #else /* __i386__ || __x86_64__ */
2195 lck_spin_init(&c_seg->c_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
2196 #endif /* __i386__ || __x86_64__ */
2197
2198 kernel_memory_populate(kernel_map, (vm_offset_t)(c_seg->c_store.c_buffer), 3 * PAGE_SIZE, KMA_COMPRESSOR);
2199
2200 c_seg->c_populated_offset = C_SEG_BYTES_TO_OFFSET(3 * PAGE_SIZE);
2201 c_seg->c_firstemptyslot = C_SLOT_MAX;
2202 c_seg->c_mysegno = c_segno;
2203 c_seg->c_filling = 1;
2204
2205 lck_mtx_lock_spin_always(c_list_lock);
2206
2207 c_segment_count++;
2208 c_segments[c_segno].c_seg = c_seg;
2209
2210 c_seg->c_generation_id = c_generation_id++;
2211
2212 queue_enter(&c_age_list_head, c_seg, c_segment_t, c_age_list);
2213 c_seg->c_on_age_q = 1;
2214 c_age_count++;
2215
2216 lck_mtx_unlock_always(c_list_lock);
2217
2218 clock_get_system_nanotime(&sec, &nsec);
2219 c_seg->c_creation_ts = (uint32_t)sec;
2220
2221 *current_chead = c_seg;
2222
2223 KERNEL_DEBUG(0xe0400004 | DBG_FUNC_END, c_seg, 0, 0, 3, 0);
2224 }
2225 slotarray = C_SEG_SLOTARRAY_FROM_INDEX(c_seg, c_seg->c_nextslot);
2226
2227 if (c_seg->c_slots[slotarray] == 0) {
2228 KERNEL_DEBUG(0xe0400008 | DBG_FUNC_START, 0, 0, 0, 0, 0);
2229
2230 c_seg->c_slots[slotarray] = (struct c_slot *)kalloc(sizeof(struct c_slot) * C_SEG_SLOT_ARRAY_SIZE);
2231
2232 KERNEL_DEBUG(0xe0400008 | DBG_FUNC_END, 0, 0, 0, 0, 0);
2233 }
2234
2235 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2236
2237 lck_mtx_lock_spin_always(&c_seg->c_lock);
2238
2239 return (c_seg);
2240 }
2241
2242
2243
2244 static void
2245 c_current_seg_filled(c_segment_t c_seg, c_segment_t *current_chead)
2246 {
2247 uint32_t unused_bytes;
2248 uint32_t offset_to_depopulate;
2249
2250 unused_bytes = trunc_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset - c_seg->c_nextoffset));
2251
2252 if (unused_bytes) {
2253
2254 offset_to_depopulate = C_SEG_BYTES_TO_OFFSET(round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_nextoffset)));
2255
2256 /*
2257 * release the extra physical page(s) at the end of the segment
2258 */
2259 lck_mtx_unlock_always(&c_seg->c_lock);
2260
2261 kernel_memory_depopulate(
2262 kernel_map,
2263 (vm_offset_t) &c_seg->c_store.c_buffer[offset_to_depopulate],
2264 unused_bytes,
2265 KMA_COMPRESSOR);
2266
2267 lck_mtx_lock_spin_always(&c_seg->c_lock);
2268
2269 c_seg->c_populated_offset = offset_to_depopulate;
2270 }
2271 c_seg->c_filling = 0;
2272
2273 if (C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE)
2274 c_seg_need_delayed_compaction(c_seg);
2275
2276 lck_mtx_unlock_always(&c_seg->c_lock);
2277
2278 *current_chead = NULL;
2279 }
2280
2281
2282 /*
2283 * returns with c_seg locked
2284 */
2285 void
2286 c_seg_swapin_requeue(c_segment_t c_seg)
2287 {
2288 clock_sec_t sec;
2289 clock_nsec_t nsec;
2290
2291 clock_get_system_nanotime(&sec, &nsec);
2292
2293 lck_mtx_lock_spin_always(c_list_lock);
2294 lck_mtx_lock_spin_always(&c_seg->c_lock);
2295
2296 if (c_seg->c_on_swappedout_q) {
2297 queue_remove(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
2298 c_seg->c_on_swappedout_q = 0;
2299 c_swappedout_count--;
2300 } else {
2301 assert(c_seg->c_on_swappedout_sparse_q);
2302
2303 queue_remove(&c_swappedout_sparse_list_head, c_seg, c_segment_t, c_age_list);
2304 c_seg->c_on_swappedout_sparse_q = 0;
2305 c_swappedout_sparse_count--;
2306 }
2307 if (c_seg->c_store.c_buffer) {
2308 queue_enter(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
2309 c_seg->c_on_swappedin_q = 1;
2310 c_swappedin_count++;
2311 }
2312 #if TRACK_BAD_C_SEGMENTS
2313 else {
2314 queue_enter(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
2315 c_seg->c_on_bad_q = 1;
2316 c_bad_count++;
2317 }
2318 #endif
2319 c_seg->c_swappedin_ts = (uint32_t)sec;
2320 c_seg->c_ondisk = 0;
2321 c_seg->c_was_swapped_in = 1;
2322
2323 lck_mtx_unlock_always(c_list_lock);
2324 }
2325
2326
2327
2328 /*
2329 * c_seg has to be locked and is returned locked.
2330 * PAGE_REPLACMENT_DISALLOWED has to be TRUE on entry and is returned TRUE
2331 */
2332
2333 void
2334 c_seg_swapin(c_segment_t c_seg, boolean_t force_minor_compaction)
2335 {
2336 vm_offset_t addr = 0;
2337 uint32_t io_size = 0;
2338 uint64_t f_offset;
2339
2340 #if !CHECKSUM_THE_SWAP
2341 if (c_seg->c_ondisk)
2342 c_seg_trim_tail(c_seg);
2343 #endif
2344 io_size = round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset));
2345 f_offset = c_seg->c_store.c_swap_handle;
2346
2347 C_SEG_BUSY(c_seg);
2348 lck_mtx_unlock_always(&c_seg->c_lock);
2349
2350 if (c_seg->c_ondisk) {
2351
2352 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2353
2354 if (kernel_memory_allocate(kernel_map, &addr, C_SEG_ALLOCSIZE, 0, KMA_COMPRESSOR | KMA_VAONLY) != KERN_SUCCESS)
2355 panic("c_seg_swapin: kernel_memory_allocate failed\n");
2356
2357 kernel_memory_populate(kernel_map, addr, io_size, KMA_COMPRESSOR);
2358
2359 if (vm_swap_get(addr, f_offset, io_size) != KERN_SUCCESS) {
2360 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2361
2362 kernel_memory_depopulate(kernel_map, addr, io_size, KMA_COMPRESSOR);
2363 kmem_free(kernel_map, addr, C_SEG_ALLOCSIZE);
2364
2365 c_seg->c_store.c_buffer = (int32_t*) NULL;
2366 c_seg->c_populated_offset = C_SEG_BYTES_TO_OFFSET(0);
2367 } else {
2368 c_seg->c_store.c_buffer = (int32_t*) addr;
2369 #if ENCRYPTED_SWAP
2370 vm_swap_decrypt(c_seg);
2371 #endif /* ENCRYPTED_SWAP */
2372
2373 #if CHECKSUM_THE_SWAP
2374 if (c_seg->cseg_swap_size != io_size)
2375 panic("swapin size doesn't match swapout size");
2376
2377 if (c_seg->cseg_hash != hash_string((char*) c_seg->c_store.c_buffer, (int)io_size)) {
2378 panic("c_seg_swapin - Swap hash mismatch\n");
2379 }
2380 #endif /* CHECKSUM_THE_SWAP */
2381
2382 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2383
2384 if (force_minor_compaction == TRUE) {
2385 lck_mtx_lock_spin_always(&c_seg->c_lock);
2386
2387 c_seg_minor_compaction_and_unlock(c_seg, FALSE);
2388 }
2389 OSAddAtomic64(c_seg->c_bytes_used, &compressor_bytes_used);
2390 OSAddAtomic64(C_SEG_ALLOCSIZE, &compressor_kvspace_used);
2391 }
2392 }
2393 c_seg_swapin_requeue(c_seg);
2394
2395 C_SEG_WAKEUP_DONE(c_seg);
2396 }
2397
2398
2399 static int
2400 c_compress_page(char *src, c_slot_mapping_t slot_ptr, c_segment_t *current_chead, char *scratch_buf)
2401 {
2402 int c_size;
2403 int c_rounded_size = 0;
2404 int max_csize;
2405 c_slot_t cs;
2406 c_segment_t c_seg;
2407
2408 KERNEL_DEBUG(0xe0400000 | DBG_FUNC_START, *current_chead, 0, 0, 0, 0);
2409 retry:
2410 if ((c_seg = c_seg_allocate(current_chead)) == NULL)
2411 return (1);
2412 /*
2413 * returns with c_seg lock held
2414 * and PAGE_REPLACEMENT_DISALLOWED(TRUE)
2415 */
2416 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_seg->c_nextslot);
2417
2418 cs->c_packed_ptr = C_SLOT_PACK_PTR(slot_ptr);
2419 assert(slot_ptr == (c_slot_mapping_t)C_SLOT_UNPACK_PTR(cs));
2420
2421 cs->c_offset = c_seg->c_nextoffset;
2422
2423 max_csize = C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES((int32_t)cs->c_offset);
2424
2425 if (max_csize > PAGE_SIZE)
2426 max_csize = PAGE_SIZE;
2427
2428 if (C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset -
2429 c_seg->c_nextoffset)
2430 < (unsigned) max_csize + PAGE_SIZE &&
2431 (C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset)
2432 < C_SEG_ALLOCSIZE)) {
2433 lck_mtx_unlock_always(&c_seg->c_lock);
2434
2435 kernel_memory_populate(kernel_map,
2436 (vm_offset_t) &c_seg->c_store.c_buffer[c_seg->c_populated_offset],
2437 PAGE_SIZE,
2438 KMA_COMPRESSOR);
2439
2440 lck_mtx_lock_spin_always(&c_seg->c_lock);
2441
2442 c_seg->c_populated_offset += C_SEG_BYTES_TO_OFFSET(PAGE_SIZE);
2443 }
2444
2445 #if CHECKSUM_THE_DATA
2446 cs->c_hash_data = hash_string(src, PAGE_SIZE);
2447 #endif
2448
2449 c_size = WKdm_compress_new((WK_word *)(uintptr_t)src, (WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
2450 (WK_word *)(uintptr_t)scratch_buf, max_csize - 4);
2451 assert(c_size <= (max_csize - 4) && c_size >= -1);
2452
2453 if (c_size == -1) {
2454
2455 if (max_csize < PAGE_SIZE) {
2456 c_current_seg_filled(c_seg, current_chead);
2457
2458 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2459
2460 goto retry;
2461 }
2462 c_size = PAGE_SIZE;
2463
2464 memcpy(&c_seg->c_store.c_buffer[cs->c_offset], src, c_size);
2465 }
2466 #if CHECKSUM_THE_COMPRESSED_DATA
2467 cs->c_hash_compressed_data = hash_string((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size);
2468 #endif
2469 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
2470
2471 PACK_C_SIZE(cs, c_size);
2472 c_seg->c_bytes_used += c_rounded_size;
2473 c_seg->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
2474
2475 slot_ptr->s_cindx = c_seg->c_nextslot++;
2476 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
2477 slot_ptr->s_cseg = c_seg->c_mysegno + 1;
2478
2479 if (c_seg->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg->c_nextslot >= C_SLOT_MAX)
2480 c_current_seg_filled(c_seg, current_chead);
2481 else
2482 lck_mtx_unlock_always(&c_seg->c_lock);
2483
2484 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2485
2486 OSAddAtomic64(c_rounded_size, &compressor_bytes_used);
2487 OSAddAtomic64(PAGE_SIZE, &c_segment_input_bytes);
2488 OSAddAtomic64(c_size, &c_segment_compressed_bytes);
2489
2490 OSAddAtomic(1, &c_segment_pages_compressed);
2491 OSAddAtomic(1, &sample_period_compression_count);
2492
2493 KERNEL_DEBUG(0xe0400000 | DBG_FUNC_END, *current_chead, c_size, c_segment_input_bytes, c_segment_compressed_bytes, 0);
2494
2495 return (0);
2496 }
2497
2498
2499 static int
2500 c_decompress_page(char *dst, volatile c_slot_mapping_t slot_ptr, int flags, int *zeroslot)
2501 {
2502 c_slot_t cs;
2503 c_segment_t c_seg;
2504 int c_indx;
2505 int c_rounded_size;
2506 uint32_t c_size;
2507 int retval = 0;
2508 boolean_t c_seg_has_data = TRUE;
2509 boolean_t c_seg_swappedin = FALSE;
2510 boolean_t need_unlock = TRUE;
2511 boolean_t consider_defragmenting = FALSE;
2512
2513 ReTry:
2514 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2515
2516 #if HIBERNATION
2517 /*
2518 * if hibernation is enabled, it indicates (via a call
2519 * to 'vm_decompressor_lock' that no further
2520 * decompressions are allowed once it reaches
2521 * the point of flushing all of the currently dirty
2522 * anonymous memory through the compressor and out
2523 * to disk... in this state we allow freeing of compressed
2524 * pages and must honor the C_DONT_BLOCK case
2525 */
2526 if (dst && decompressions_blocked == TRUE) {
2527 if (flags & C_DONT_BLOCK) {
2528
2529 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2530
2531 *zeroslot = 0;
2532 return (-2);
2533 }
2534 /*
2535 * it's safe to atomically assert and block behind the
2536 * lock held in shared mode because "decompressions_blocked" is
2537 * only set and cleared and the thread_wakeup done when the lock
2538 * is held exclusively
2539 */
2540 assert_wait((event_t)&decompressions_blocked, THREAD_UNINT);
2541
2542 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2543
2544 thread_block(THREAD_CONTINUE_NULL);
2545
2546 goto ReTry;
2547 }
2548 #endif
2549 /* s_cseg is actually "segno+1" */
2550 c_seg = c_segments[slot_ptr->s_cseg - 1].c_seg;
2551
2552 lck_mtx_lock_spin_always(&c_seg->c_lock);
2553
2554 if (flags & C_DONT_BLOCK) {
2555 if (c_seg->c_busy || (c_seg->c_ondisk && dst)) {
2556
2557 retval = -2;
2558 *zeroslot = 0;
2559
2560 goto done;
2561 }
2562 }
2563 if (c_seg->c_busy) {
2564
2565 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2566
2567 c_seg_wait_on_busy(c_seg);
2568
2569 goto ReTry;
2570 }
2571 c_indx = slot_ptr->s_cindx;
2572
2573 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
2574
2575 c_size = UNPACK_C_SIZE(cs);
2576
2577 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
2578
2579 if (dst) {
2580 uint32_t age_of_cseg;
2581 clock_sec_t cur_ts_sec;
2582 clock_nsec_t cur_ts_nsec;
2583
2584 if (c_seg->c_on_swappedout_q || c_seg->c_on_swappedout_sparse_q) {
2585 if (c_seg->c_ondisk)
2586 c_seg_swappedin = TRUE;
2587 c_seg_swapin(c_seg, FALSE);
2588 }
2589 if (c_seg->c_store.c_buffer == NULL) {
2590 c_seg_has_data = FALSE;
2591 goto c_seg_invalid_data;
2592 }
2593 #if CHECKSUM_THE_COMPRESSED_DATA
2594 if (cs->c_hash_compressed_data != hash_string((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size))
2595 panic("compressed data doesn't match original");
2596 #endif
2597 if (c_rounded_size == PAGE_SIZE) {
2598 /*
2599 * page wasn't compressible... just copy it out
2600 */
2601 memcpy(dst, &c_seg->c_store.c_buffer[cs->c_offset], PAGE_SIZE);
2602 } else {
2603 uint32_t my_cpu_no;
2604 char *scratch_buf;
2605
2606 /*
2607 * we're behind the c_seg lock held in spin mode
2608 * which means pre-emption is disabled... therefore
2609 * the following sequence is atomic and safe
2610 */
2611 my_cpu_no = cpu_number();
2612
2613 assert(my_cpu_no < compressor_cpus);
2614
2615 scratch_buf = &compressor_scratch_bufs[my_cpu_no * WKdm_SCRATCH_BUF_SIZE];
2616 WKdm_decompress_new((WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
2617 (WK_word *)(uintptr_t)dst, (WK_word *)(uintptr_t)scratch_buf, c_size);
2618 }
2619
2620 #if CHECKSUM_THE_DATA
2621 if (cs->c_hash_data != hash_string(dst, PAGE_SIZE))
2622 panic("decompressed data doesn't match original");
2623 #endif
2624 if (!c_seg->c_was_swapped_in) {
2625
2626 clock_get_system_nanotime(&cur_ts_sec, &cur_ts_nsec);
2627
2628 age_of_cseg = (uint32_t)cur_ts_sec - c_seg->c_creation_ts;
2629
2630 if (age_of_cseg < DECOMPRESSION_SAMPLE_MAX_AGE)
2631 OSAddAtomic(1, &age_of_decompressions_during_sample_period[age_of_cseg]);
2632 else
2633 OSAddAtomic(1, &overage_decompressions_during_sample_period);
2634
2635 OSAddAtomic(1, &sample_period_decompression_count);
2636 }
2637 } else {
2638 if (c_seg->c_store.c_buffer == NULL)
2639 c_seg_has_data = FALSE;
2640 }
2641 c_seg_invalid_data:
2642
2643 if (c_seg_has_data == TRUE) {
2644 if (c_seg_swappedin == TRUE)
2645 retval = 1;
2646 else
2647 retval = 0;
2648 } else
2649 retval = -1;
2650
2651 if (flags & C_KEEP) {
2652 *zeroslot = 0;
2653 goto done;
2654 }
2655 c_seg->c_bytes_unused += c_rounded_size;
2656 c_seg->c_bytes_used -= c_rounded_size;
2657 PACK_C_SIZE(cs, 0);
2658
2659 if (c_indx < c_seg->c_firstemptyslot)
2660 c_seg->c_firstemptyslot = c_indx;
2661
2662 OSAddAtomic(-1, &c_segment_pages_compressed);
2663
2664 if (c_seg_has_data == TRUE && !c_seg->c_ondisk) {
2665 /*
2666 * c_ondisk == TRUE can occur when we're doing a
2667 * free of a compressed page (i.e. dst == NULL)
2668 */
2669 OSAddAtomic64(-c_rounded_size, &compressor_bytes_used);
2670 }
2671 if (!c_seg->c_filling) {
2672 if (c_seg->c_bytes_used == 0) {
2673 if (!c_seg->c_ondisk) {
2674 int pages_populated;
2675
2676 pages_populated = (round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE;
2677 c_seg->c_populated_offset = C_SEG_BYTES_TO_OFFSET(0);
2678
2679 if (pages_populated) {
2680 assert(c_seg->c_store.c_buffer != NULL);
2681
2682 C_SEG_BUSY(c_seg);
2683 lck_mtx_unlock_always(&c_seg->c_lock);
2684
2685 kernel_memory_depopulate(kernel_map, (vm_offset_t) c_seg->c_store.c_buffer, pages_populated * PAGE_SIZE, KMA_COMPRESSOR);
2686
2687 lck_mtx_lock_spin_always(&c_seg->c_lock);
2688 C_SEG_WAKEUP_DONE(c_seg);
2689 }
2690 if (!c_seg->c_on_minorcompact_q && !c_seg->c_on_swapout_q)
2691 c_seg_need_delayed_compaction(c_seg);
2692 } else
2693 assert(c_seg->c_on_swappedout_sparse_q);
2694
2695 } else if (c_seg->c_on_minorcompact_q) {
2696
2697 if (C_SEG_INCORE_IS_SPARSE(c_seg)) {
2698 c_seg_try_minor_compaction_and_unlock(c_seg);
2699 need_unlock = FALSE;
2700 }
2701 } else if (!c_seg->c_ondisk) {
2702
2703 if (c_seg_has_data == TRUE && !c_seg->c_on_swapout_q && C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE) {
2704 c_seg_need_delayed_compaction(c_seg);
2705 }
2706 } else if (!c_seg->c_on_swappedout_sparse_q && C_SEG_ONDISK_IS_SPARSE(c_seg)) {
2707
2708 c_seg_move_to_sparse_list(c_seg);
2709 consider_defragmenting = TRUE;
2710 }
2711 }
2712 done:
2713 if (need_unlock == TRUE)
2714 lck_mtx_unlock_always(&c_seg->c_lock);
2715
2716 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2717
2718 if (consider_defragmenting == TRUE)
2719 vm_swap_consider_defragmenting();
2720
2721
2722 return (retval);
2723 }
2724
2725
2726 int
2727 vm_compressor_get(ppnum_t pn, int *slot, int flags)
2728 {
2729 char *dst;
2730 int zeroslot = 1;
2731 int retval;
2732
2733 #if __x86_64__
2734 dst = PHYSMAP_PTOV((uint64_t)pn << (uint64_t)PAGE_SHIFT);
2735 #else
2736 #error "unsupported architecture"
2737 #endif
2738
2739 retval = c_decompress_page(dst, (c_slot_mapping_t)slot, flags, &zeroslot);
2740
2741 /*
2742 * zeroslot will be set to 0 by c_decompress_page if (flags & C_KEEP)
2743 * or (flags & C_DONT_BLOCK) and we found 'c_busy' or 'c_ondisk' set
2744 */
2745 if (zeroslot) {
2746 *slot = 0;
2747 }
2748 /*
2749 * returns 0 if we successfully decompressed a page from a segment already in memory
2750 * returns 1 if we had to first swap in the segment, before successfully decompressing the page
2751 * returns -1 if we encountered an error swapping in the segment - decompression failed
2752 * returns -2 if (flags & C_DONT_BLOCK) and we found 'c_busy' or 'c_ondisk' set
2753 */
2754 return (retval);
2755 }
2756
2757
2758 int
2759 vm_compressor_free(int *slot, int flags)
2760 {
2761 int zeroslot = 1;
2762 int retval;
2763
2764 assert(flags == 0 || flags == C_DONT_BLOCK);
2765
2766 retval = c_decompress_page(NULL, (c_slot_mapping_t)slot, flags, &zeroslot);
2767 /*
2768 * returns 0 if we successfully freed the specified compressed page
2769 * returns -2 if (flags & C_DONT_BLOCK) and we found 'c_busy' set
2770 */
2771
2772 if (retval == 0)
2773 *slot = 0;
2774
2775 return (retval);
2776 }
2777
2778
2779 int
2780 vm_compressor_put(ppnum_t pn, int *slot, void **current_chead, char *scratch_buf)
2781 {
2782 char *src;
2783 int retval;
2784
2785 #if __x86_64__
2786 src = PHYSMAP_PTOV((uint64_t)pn << (uint64_t)PAGE_SHIFT);
2787 #else
2788 #error "unsupported architecture"
2789 #endif
2790 retval = c_compress_page(src, (c_slot_mapping_t)slot, (c_segment_t *)current_chead, scratch_buf);
2791
2792 return (retval);
2793 }
2794
2795 void
2796 vm_compressor_transfer(
2797 int *dst_slot_p,
2798 int *src_slot_p)
2799 {
2800 c_slot_mapping_t dst_slot, src_slot;
2801 c_segment_t c_seg;
2802 int c_indx;
2803 c_slot_t cs;
2804
2805 dst_slot = (c_slot_mapping_t) dst_slot_p;
2806 src_slot = (c_slot_mapping_t) src_slot_p;
2807
2808 Retry:
2809 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2810 /* get segment for src_slot */
2811 c_seg = c_segments[src_slot->s_cseg -1].c_seg;
2812 /* lock segment */
2813 lck_mtx_lock_spin_always(&c_seg->c_lock);
2814 /* wait if it's busy */
2815 if (c_seg->c_busy) {
2816 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2817 c_seg_wait_on_busy(c_seg);
2818 goto Retry;
2819 }
2820 /* find the c_slot */
2821 c_indx = src_slot->s_cindx;
2822 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
2823 /* point the c_slot back to dst_slot instead of src_slot */
2824 cs->c_packed_ptr = C_SLOT_PACK_PTR(dst_slot);
2825 /* transfer */
2826 *dst_slot_p = *src_slot_p;
2827 *src_slot_p = 0;
2828 lck_mtx_unlock_always(&c_seg->c_lock);
2829 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2830 }
mirror of XNU