]> git.saurik.com Git - apple/xnu.git/blob - osfmk/vm/vm_compressor.c
projects / apple / xnu.git / blob
? search:


ca9ff6c6d69a97e2b08ecf91274db0c9bb93bab5
[apple/xnu.git] / osfmk / vm / vm_compressor.c
1 /*
2 * Copyright (c) 2000-2013 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <vm/vm_compressor.h>
30 #include <vm/vm_map.h>
31 #include <vm/vm_pageout.h>
32 #include <vm/memory_object.h>
33 #include <mach/mach_host.h> /* for host_info() */
34 #include <kern/ledger.h>
35
36 #include <default_pager/default_pager_alerts.h>
37 #include <default_pager/default_pager_object_server.h>
38
39 #include <IOKit/IOHibernatePrivate.h>
40
41 /*
42 * vm_compressor_mode has a heirarchy of control to set its value.
43 * boot-args are checked first, then device-tree, and finally
44 * the default value that is defined below. See vm_fault_init() for
45 * the boot-arg & device-tree code.
46 */
47
48 extern ipc_port_t min_pages_trigger_port;
49 extern lck_mtx_t paging_segments_lock;
50 #define PSL_LOCK() lck_mtx_lock(&paging_segments_lock)
51 #define PSL_UNLOCK() lck_mtx_unlock(&paging_segments_lock)
52
53
54 int vm_compressor_mode = VM_PAGER_COMPRESSOR_WITH_SWAP;
55 int vm_scale = 16;
56
57
58 int vm_compression_limit = 0;
59
60 extern boolean_t vm_swap_up;
61 extern void vm_pageout_io_throttle(void);
62
63 #if CHECKSUM_THE_DATA || CHECKSUM_THE_SWAP || CHECKSUM_THE_COMPRESSED_DATA
64 extern unsigned int hash_string(char *cp, int len);
65 #endif
66
67 struct c_slot {
68 uint64_t c_offset:C_SEG_OFFSET_BITS,
69 c_size:12,
70 c_packed_ptr:36;
71 #if CHECKSUM_THE_DATA
72 unsigned int c_hash_data;
73 #endif
74 #if CHECKSUM_THE_COMPRESSED_DATA
75 unsigned int c_hash_compressed_data;
76 #endif
77
78 };
79
80 #define UNPACK_C_SIZE(cs) ((cs->c_size == (PAGE_SIZE-1)) ? 4096 : cs->c_size)
81 #define PACK_C_SIZE(cs, size) (cs->c_size = ((size == PAGE_SIZE) ? PAGE_SIZE - 1 : size))
82
83
84 struct c_slot_mapping {
85 uint32_t s_cseg:22, /* segment number + 1 */
86 s_cindx:10; /* index in the segment */
87 };
88
89 typedef struct c_slot_mapping *c_slot_mapping_t;
90
91
92 union c_segu {
93 c_segment_t c_seg;
94 uint32_t c_segno;
95 };
96
97
98
99 #define C_SLOT_PACK_PTR(ptr) (((uintptr_t)ptr - (uintptr_t) VM_MIN_KERNEL_AND_KEXT_ADDRESS) >> 2)
100 #define C_SLOT_UNPACK_PTR(cslot) ((uintptr_t)(cslot->c_packed_ptr << 2) + (uintptr_t) VM_MIN_KERNEL_AND_KEXT_ADDRESS)
101
102
103 uint32_t c_segment_count = 0;
104
105 uint64_t c_generation_id = 0;
106 uint64_t c_generation_id_flush_barrier;
107
108
109 #define HIBERNATE_FLUSHING_SECS_TO_COMPLETE 120
110
111 boolean_t hibernate_no_swapspace = FALSE;
112 clock_sec_t hibernate_flushing_deadline = 0;
113
114
115 #if TRACK_BAD_C_SEGMENTS
116 queue_head_t c_bad_list_head;
117 uint32_t c_bad_count = 0;
118 #endif
119
120 queue_head_t c_age_list_head;
121 queue_head_t c_swapout_list_head;
122 queue_head_t c_swappedin_list_head;
123 queue_head_t c_swappedout_list_head;
124 queue_head_t c_swappedout_sparse_list_head;
125
126 uint32_t c_age_count = 0;
127 uint32_t c_swapout_count = 0;
128 uint32_t c_swappedin_count = 0;
129 uint32_t c_swappedout_count = 0;
130 uint32_t c_swappedout_sparse_count = 0;
131
132 queue_head_t c_minor_list_head;
133 uint32_t c_minor_count = 0;
134
135 union c_segu *c_segments;
136 caddr_t c_segments_next_page;
137 boolean_t c_segments_busy;
138 uint32_t c_segments_available;
139 uint32_t c_segments_limit;
140 uint32_t c_segment_pages_compressed;
141 uint32_t c_segment_pages_compressed_limit;
142 uint32_t c_free_segno_head = (uint32_t)-1;
143
144 uint32_t vm_compressor_minorcompact_threshold_divisor = 10;
145 uint32_t vm_compressor_majorcompact_threshold_divisor = 10;
146 uint32_t vm_compressor_unthrottle_threshold_divisor = 10;
147 uint32_t vm_compressor_catchup_threshold_divisor = 10;
148
149 #define C_SEGMENTS_PER_PAGE (PAGE_SIZE / sizeof(union c_segu))
150
151
152 lck_grp_attr_t vm_compressor_lck_grp_attr;
153 lck_attr_t vm_compressor_lck_attr;
154 lck_grp_t vm_compressor_lck_grp;
155
156
157 #if __i386__ || __x86_64__
158 lck_mtx_t *c_list_lock;
159 #else /* __i386__ || __x86_64__ */
160 lck_spin_t *c_list_lock;
161 #endif /* __i386__ || __x86_64__ */
162
163 lck_rw_t c_master_lock;
164 lck_rw_t c_decompressor_lock;
165
166 zone_t compressor_segment_zone;
167 int c_compressor_swap_trigger = 0;
168
169 uint32_t compressor_cpus;
170 char *compressor_scratch_bufs;
171
172
173 clock_sec_t start_of_sample_period_sec = 0;
174 clock_nsec_t start_of_sample_period_nsec = 0;
175 clock_sec_t start_of_eval_period_sec = 0;
176 clock_nsec_t start_of_eval_period_nsec = 0;
177 uint32_t sample_period_decompression_count = 0;
178 uint32_t sample_period_compression_count = 0;
179 uint32_t last_eval_decompression_count = 0;
180 uint32_t last_eval_compression_count = 0;
181
182 #define DECOMPRESSION_SAMPLE_MAX_AGE (60 * 30)
183
184 uint32_t swapout_target_age = 0;
185 uint32_t age_of_decompressions_during_sample_period[DECOMPRESSION_SAMPLE_MAX_AGE];
186 uint32_t overage_decompressions_during_sample_period = 0;
187
188 void do_fastwake_warmup(void);
189 boolean_t fastwake_warmup = FALSE;
190 boolean_t fastwake_recording_in_progress = FALSE;
191 clock_sec_t dont_trim_until_ts = 0;
192
193 uint64_t c_segment_warmup_count;
194 uint64_t first_c_segment_to_warm_generation_id = 0;
195 uint64_t last_c_segment_to_warm_generation_id = 0;
196 boolean_t hibernate_flushing = FALSE;
197
198 int64_t c_segment_input_bytes = 0;
199 int64_t c_segment_compressed_bytes = 0;
200 int64_t compressor_bytes_used = 0;
201
202 static boolean_t compressor_needs_to_swap(void);
203 static void vm_compressor_swap_trigger_thread(void);
204 static void vm_compressor_do_delayed_compactions(boolean_t);
205 static void vm_compressor_compact_and_swap(boolean_t);
206 static void vm_compressor_age_swapped_in_segments(boolean_t);
207 static uint64_t compute_elapsed_msecs(clock_sec_t, clock_nsec_t, clock_sec_t, clock_nsec_t);
208
209 boolean_t vm_compressor_low_on_space(void);
210
211 void compute_swapout_target_age(void);
212
213 boolean_t c_seg_major_compact(c_segment_t, c_segment_t);
214 boolean_t c_seg_major_compact_ok(c_segment_t, c_segment_t);
215
216 int c_seg_minor_compaction_and_unlock(c_segment_t, boolean_t);
217 int c_seg_do_minor_compaction_and_unlock(c_segment_t, boolean_t, boolean_t, boolean_t);
218 void c_seg_try_minor_compaction_and_unlock(c_segment_t c_seg);
219 void c_seg_need_delayed_compaction(c_segment_t);
220
221 void c_seg_move_to_sparse_list(c_segment_t);
222 void c_seg_insert_into_q(queue_head_t *, c_segment_t);
223
224 boolean_t c_seg_try_free(c_segment_t);
225 void c_seg_free(c_segment_t);
226 void c_seg_free_locked(c_segment_t);
227
228
229 uint64_t vm_available_memory(void);
230
231 extern unsigned int dp_pages_free, dp_pages_reserve;
232
233 uint64_t
234 vm_available_memory(void)
235 {
236 return (((uint64_t)AVAILABLE_NON_COMPRESSED_MEMORY) * PAGE_SIZE_64);
237 }
238
239
240 boolean_t
241 vm_compression_available(void)
242 {
243 if ( !(COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE))
244 return (FALSE);
245
246 if (c_segments_available >= c_segments_limit || c_segment_pages_compressed >= c_segment_pages_compressed_limit)
247 return (FALSE);
248
249 return (TRUE);
250 }
251
252
253 boolean_t
254 vm_compressor_low_on_space(void)
255 {
256 if ((c_segment_pages_compressed > (c_segment_pages_compressed_limit - 20000)) ||
257 (c_segment_count > (c_segments_limit - 250)))
258 return (TRUE);
259
260 return (FALSE);
261 }
262
263
264 int
265 vm_low_on_space(void)
266 {
267 if (vm_compressor_mode == COMPRESSED_PAGER_IS_ACTIVE || vm_compressor_mode == DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE) {
268 if (vm_compressor_low_on_space() || HARD_THROTTLE_LIMIT_REACHED())
269 return (1);
270 } else {
271 if (((dp_pages_free + dp_pages_reserve < 2000) && VM_DYNAMIC_PAGING_ENABLED(memory_manager_default)))
272 return (1);
273 }
274 return (0);
275 }
276
277
278 void
279 vm_compressor_init_locks(void)
280 {
281 lck_grp_attr_setdefault(&vm_compressor_lck_grp_attr);
282 lck_grp_init(&vm_compressor_lck_grp, "vm_compressor", &vm_compressor_lck_grp_attr);
283 lck_attr_setdefault(&vm_compressor_lck_attr);
284
285 lck_rw_init(&c_master_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
286 lck_rw_init(&c_decompressor_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
287 }
288
289
290 void
291 vm_decompressor_lock(void)
292 {
293 lck_rw_lock_exclusive(&c_decompressor_lock);
294 }
295
296 void
297 vm_decompressor_unlock(void)
298 {
299 lck_rw_done(&c_decompressor_lock);
300
301 }
302
303
304
305 void
306 vm_compressor_init(void)
307 {
308 thread_t thread;
309
310 assert((C_SEGMENTS_PER_PAGE * sizeof(union c_segu)) == PAGE_SIZE);
311
312 PE_parse_boot_argn("vm_compression_limit", &vm_compression_limit, sizeof (vm_compression_limit));
313
314 if (max_mem <= (3ULL * 1024ULL * 1024ULL * 1024ULL)) {
315 vm_compressor_minorcompact_threshold_divisor = 11;
316 vm_compressor_majorcompact_threshold_divisor = 13;
317 vm_compressor_unthrottle_threshold_divisor = 20;
318 vm_compressor_catchup_threshold_divisor = 35;
319 } else {
320 vm_compressor_minorcompact_threshold_divisor = 20;
321 vm_compressor_majorcompact_threshold_divisor = 25;
322 vm_compressor_unthrottle_threshold_divisor = 35;
323 vm_compressor_catchup_threshold_divisor = 50;
324 }
325 /*
326 * vm_page_init_lck_grp is now responsible for calling vm_compressor_init_locks
327 * c_master_lock needs to be available early so that "vm_page_find_contiguous" can
328 * use PAGE_REPLACEMENT_ALLOWED to coordinate with the compressor.
329 */
330
331 #if __i386__ || __x86_64__
332 c_list_lock = lck_mtx_alloc_init(&vm_compressor_lck_grp, &vm_compressor_lck_attr);
333 #else /* __i386__ || __x86_64__ */
334 c_list_lock = lck_spin_alloc_init(&vm_compressor_lck_grp, &vm_compressor_lck_attr);
335 #endif /* __i386__ || __x86_64__ */
336
337 #if TRACK_BAD_C_SEGMENTS
338 queue_init(&c_bad_list_head);
339 #endif
340 queue_init(&c_age_list_head);
341 queue_init(&c_minor_list_head);
342 queue_init(&c_swapout_list_head);
343 queue_init(&c_swappedin_list_head);
344 queue_init(&c_swappedout_list_head);
345 queue_init(&c_swappedout_sparse_list_head);
346
347 compressor_segment_zone = zinit(sizeof (struct c_segment),
348 128000 * sizeof (struct c_segment),
349 8192, "compressor_segment");
350 zone_change(compressor_segment_zone, Z_CALLERACCT, FALSE);
351 zone_change(compressor_segment_zone, Z_NOENCRYPT, TRUE);
352
353
354 c_free_segno_head = -1;
355 c_segments_available = 0;
356
357 if (vm_compression_limit == 0) {
358 c_segment_pages_compressed_limit = (uint32_t)((max_mem / PAGE_SIZE)) * vm_scale;
359
360 #define OLD_SWAP_LIMIT (1024 * 1024 * 16)
361 #define MAX_SWAP_LIMIT (1024 * 1024 * 128)
362
363 if (c_segment_pages_compressed_limit > (OLD_SWAP_LIMIT))
364 c_segment_pages_compressed_limit = OLD_SWAP_LIMIT;
365
366 if (c_segment_pages_compressed_limit < (uint32_t)(max_mem / PAGE_SIZE_64))
367 c_segment_pages_compressed_limit = (uint32_t)(max_mem / PAGE_SIZE_64);
368 } else {
369 if (vm_compression_limit < MAX_SWAP_LIMIT)
370 c_segment_pages_compressed_limit = vm_compression_limit;
371 else
372 c_segment_pages_compressed_limit = MAX_SWAP_LIMIT;
373 }
374 if ((c_segments_limit = c_segment_pages_compressed_limit / (C_SEG_BUFSIZE / PAGE_SIZE)) > C_SEG_MAX_LIMIT)
375 c_segments_limit = C_SEG_MAX_LIMIT;
376
377 c_segments_busy = FALSE;
378
379 if (kernel_memory_allocate(kernel_map, (vm_offset_t *)(&c_segments), (sizeof(union c_segu) * c_segments_limit), 0, KMA_KOBJECT | KMA_VAONLY) != KERN_SUCCESS)
380 panic("vm_compressor_init: kernel_memory_allocate failed\n");
381
382 c_segments_next_page = (caddr_t)c_segments;
383
384 {
385 host_basic_info_data_t hinfo;
386 mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
387
388 #define BSD_HOST 1
389 host_info((host_t)BSD_HOST, HOST_BASIC_INFO, (host_info_t)&hinfo, &count);
390
391 compressor_cpus = hinfo.max_cpus;
392
393 compressor_scratch_bufs = kalloc(compressor_cpus * WKdm_SCRATCH_BUF_SIZE);
394 }
395
396 if (kernel_thread_start_priority((thread_continue_t)vm_compressor_swap_trigger_thread, NULL,
397 BASEPRI_PREEMPT - 1, &thread) != KERN_SUCCESS) {
398 panic("vm_compressor_swap_trigger_thread: create failed");
399 }
400 thread->options |= TH_OPT_VMPRIV;
401
402 thread_deallocate(thread);
403
404 assert(default_pager_init_flag == 0);
405
406 if (vm_pageout_internal_start() != KERN_SUCCESS) {
407 panic("vm_compressor_init: Failed to start the internal pageout thread.\n");
408 }
409
410 #if CONFIG_FREEZE
411 memorystatus_freeze_enabled = TRUE;
412 #endif /* CONFIG_FREEZE */
413
414 default_pager_init_flag = 1;
415
416 vm_page_reactivate_all_throttled();
417 }
418
419
420 #if VALIDATE_C_SEGMENTS
421
422 static void
423 c_seg_validate(c_segment_t c_seg, boolean_t must_be_compact)
424 {
425 int c_indx;
426 int32_t bytes_used;
427 int32_t bytes_unused;
428 uint32_t c_rounded_size;
429 uint32_t c_size;
430 c_slot_t cs;
431
432 if (c_seg->c_firstemptyslot < c_seg->c_nextslot) {
433 c_indx = c_seg->c_firstemptyslot;
434 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
435
436 if (cs == NULL)
437 panic("c_seg_validate: no slot backing c_firstemptyslot");
438
439 if (cs->c_size)
440 panic("c_seg_validate: c_firstemptyslot has non-zero size (%d)\n", cs->c_size);
441 }
442 bytes_used = 0;
443 bytes_unused = 0;
444
445 for (c_indx = 0; c_indx < c_seg->c_nextslot; c_indx++) {
446
447 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
448
449 c_size = UNPACK_C_SIZE(cs);
450
451 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
452
453 bytes_used += c_rounded_size;
454
455 #if CHECKSUM_THE_COMPRESSED_DATA
456 if (c_size && cs->c_hash_compressed_data != hash_string((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size))
457 panic("compressed data doesn't match original");
458 #endif
459 }
460
461 if (bytes_used != c_seg->c_bytes_used)
462 panic("c_seg_validate: bytes_used mismatch - found %d, segment has %d\n", bytes_used, c_seg->c_bytes_used);
463
464 if (c_seg->c_bytes_used > C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset))
465 panic("c_seg_validate: c_bytes_used > c_nextoffset - c_nextoffset = %d, c_bytes_used = %d\n",
466 (int32_t)C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset), c_seg->c_bytes_used);
467
468 if (must_be_compact) {
469 if (c_seg->c_bytes_used != C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset))
470 panic("c_seg_validate: c_bytes_used doesn't match c_nextoffset - c_nextoffset = %d, c_bytes_used = %d\n",
471 (int32_t)C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset), c_seg->c_bytes_used);
472 }
473 }
474
475 #endif
476
477
478 void
479 c_seg_need_delayed_compaction(c_segment_t c_seg)
480 {
481 boolean_t clear_busy = FALSE;
482
483 if ( !lck_mtx_try_lock_spin_always(c_list_lock)) {
484 c_seg->c_busy = 1;
485
486 lck_mtx_unlock_always(&c_seg->c_lock);
487 lck_mtx_lock_spin_always(c_list_lock);
488 lck_mtx_lock_spin_always(&c_seg->c_lock);
489
490 clear_busy = TRUE;
491 }
492 if (!c_seg->c_on_minorcompact_q && !c_seg->c_ondisk && !c_seg->c_on_swapout_q) {
493 queue_enter(&c_minor_list_head, c_seg, c_segment_t, c_list);
494 c_seg->c_on_minorcompact_q = 1;
495 c_minor_count++;
496 }
497 lck_mtx_unlock_always(c_list_lock);
498
499 if (clear_busy == TRUE)
500 C_SEG_WAKEUP_DONE(c_seg);
501 }
502
503
504 unsigned int c_seg_moved_to_sparse_list = 0;
505
506 void
507 c_seg_move_to_sparse_list(c_segment_t c_seg)
508 {
509 boolean_t clear_busy = FALSE;
510
511 if ( !lck_mtx_try_lock_spin_always(c_list_lock)) {
512 c_seg->c_busy = 1;
513
514 lck_mtx_unlock_always(&c_seg->c_lock);
515 lck_mtx_lock_spin_always(c_list_lock);
516 lck_mtx_lock_spin_always(&c_seg->c_lock);
517
518 clear_busy = TRUE;
519 }
520 assert(c_seg->c_ondisk);
521 assert(c_seg->c_on_swappedout_q);
522 assert(!c_seg->c_on_swappedout_sparse_q);
523
524 queue_remove(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
525 c_seg->c_on_swappedout_q = 0;
526 c_swappedout_count--;
527
528 c_seg_insert_into_q(&c_swappedout_sparse_list_head, c_seg);
529 c_seg->c_on_swappedout_sparse_q = 1;
530 c_swappedout_sparse_count++;
531
532 c_seg_moved_to_sparse_list++;
533
534 lck_mtx_unlock_always(c_list_lock);
535
536 if (clear_busy == TRUE)
537 C_SEG_WAKEUP_DONE(c_seg);
538 }
539
540
541 void
542 c_seg_insert_into_q(queue_head_t *qhead, c_segment_t c_seg)
543 {
544 c_segment_t c_seg_next;
545
546 if (queue_empty(qhead)) {
547 queue_enter(qhead, c_seg, c_segment_t, c_age_list);
548 } else {
549 c_seg_next = (c_segment_t)queue_first(qhead);
550
551 while (TRUE) {
552
553 if (c_seg->c_generation_id < c_seg_next->c_generation_id) {
554 queue_insert_before(qhead, c_seg, c_seg_next, c_segment_t, c_age_list);
555 break;
556 }
557 c_seg_next = (c_segment_t) queue_next(&c_seg_next->c_age_list);
558
559 if (queue_end(qhead, (queue_entry_t) c_seg_next)) {
560 queue_enter(qhead, c_seg, c_segment_t, c_age_list);
561 break;
562 }
563 }
564 }
565 }
566
567
568 int try_minor_compaction_failed = 0;
569 int try_minor_compaction_succeeded = 0;
570
571 void
572 c_seg_try_minor_compaction_and_unlock(c_segment_t c_seg)
573 {
574
575 assert(c_seg->c_on_minorcompact_q);
576 /*
577 * c_seg is currently on the delayed minor compaction
578 * queue and we have c_seg locked... if we can get the
579 * c_list_lock w/o blocking (if we blocked we could deadlock
580 * because the lock order is c_list_lock then c_seg's lock)
581 * we'll pull it from the delayed list and free it directly
582 */
583 if ( !lck_mtx_try_lock_spin_always(c_list_lock)) {
584 /*
585 * c_list_lock is held, we need to bail
586 */
587 try_minor_compaction_failed++;
588
589 lck_mtx_unlock_always(&c_seg->c_lock);
590 } else {
591 try_minor_compaction_succeeded++;
592
593 c_seg->c_busy = 1;
594 c_seg_do_minor_compaction_and_unlock(c_seg, TRUE, FALSE, FALSE);
595 }
596 }
597
598
599 int
600 c_seg_do_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy, boolean_t need_list_lock, boolean_t disallow_page_replacement)
601 {
602 int c_seg_freed;
603
604 assert(c_seg->c_busy);
605
606 if (!c_seg->c_on_minorcompact_q) {
607 if (clear_busy == TRUE)
608 C_SEG_WAKEUP_DONE(c_seg);
609
610 lck_mtx_unlock_always(&c_seg->c_lock);
611
612 return (0);
613 }
614 queue_remove(&c_minor_list_head, c_seg, c_segment_t, c_list);
615 c_seg->c_on_minorcompact_q = 0;
616 c_minor_count--;
617
618 lck_mtx_unlock_always(c_list_lock);
619
620 if (disallow_page_replacement == TRUE) {
621 lck_mtx_unlock_always(&c_seg->c_lock);
622
623 PAGE_REPLACEMENT_DISALLOWED(TRUE);
624
625 lck_mtx_lock_spin_always(&c_seg->c_lock);
626 }
627 c_seg_freed = c_seg_minor_compaction_and_unlock(c_seg, clear_busy);
628
629 if (disallow_page_replacement == TRUE)
630 PAGE_REPLACEMENT_DISALLOWED(FALSE);
631
632 if (need_list_lock == TRUE)
633 lck_mtx_lock_spin_always(c_list_lock);
634
635 return (c_seg_freed);
636 }
637
638
639 void
640 c_seg_wait_on_busy(c_segment_t c_seg)
641 {
642 c_seg->c_wanted = 1;
643 assert_wait((event_t) (c_seg), THREAD_UNINT);
644
645 lck_mtx_unlock_always(&c_seg->c_lock);
646 thread_block(THREAD_CONTINUE_NULL);
647 }
648
649
650
651 int try_free_succeeded = 0;
652 int try_free_failed = 0;
653
654 boolean_t
655 c_seg_try_free(c_segment_t c_seg)
656 {
657 /*
658 * c_seg is currently on the delayed minor compaction
659 * or the spapped out sparse queue and we have c_seg locked...
660 * if we can get the c_list_lock w/o blocking (if we blocked we
661 * could deadlock because the lock order is c_list_lock then c_seg's lock)
662 * we'll pull it from the appropriate queue and free it
663 */
664 if ( !lck_mtx_try_lock_spin_always(c_list_lock)) {
665 /*
666 * c_list_lock is held, we need to bail
667 */
668 try_free_failed++;
669 return (FALSE);
670 }
671 if (c_seg->c_on_minorcompact_q) {
672 queue_remove(&c_minor_list_head, c_seg, c_segment_t, c_list);
673 c_seg->c_on_minorcompact_q = 0;
674 c_minor_count--;
675 } else {
676 assert(c_seg->c_on_swappedout_sparse_q);
677
678 /*
679 * c_seg_free_locked will remove it from the swappedout sparse list
680 */
681 }
682 if (!c_seg->c_busy_swapping)
683 c_seg->c_busy = 1;
684
685 c_seg_free_locked(c_seg);
686
687 try_free_succeeded++;
688
689 return (TRUE);
690 }
691
692
693 void
694 c_seg_free(c_segment_t c_seg)
695 {
696 if (!c_seg->c_busy_swapping)
697 c_seg->c_busy = 1;
698
699 lck_mtx_unlock_always(&c_seg->c_lock);
700 lck_mtx_lock_spin_always(c_list_lock);
701 lck_mtx_lock_spin_always(&c_seg->c_lock);
702
703 c_seg_free_locked(c_seg);
704 }
705
706
707 void
708 c_seg_free_locked(c_segment_t c_seg)
709 {
710 int segno, i;
711 int pages_populated;
712 int32_t *c_buffer = NULL;
713 uint64_t c_swap_handle;
714
715 assert(!c_seg->c_on_minorcompact_q);
716
717 if (c_seg->c_on_age_q) {
718 queue_remove(&c_age_list_head, c_seg, c_segment_t, c_age_list);
719 c_seg->c_on_age_q = 0;
720 c_age_count--;
721 } else if (c_seg->c_on_swappedin_q) {
722 queue_remove(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
723 c_seg->c_on_swappedin_q = 0;
724 c_swappedin_count--;
725 } else if (c_seg->c_on_swapout_q) {
726 queue_remove(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
727 c_seg->c_on_swapout_q = 0;
728 c_swapout_count--;
729 thread_wakeup((event_t)&compaction_swapper_running);
730 } else if (c_seg->c_on_swappedout_q) {
731 queue_remove(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
732 c_seg->c_on_swappedout_q = 0;
733 c_swappedout_count--;
734 } else if (c_seg->c_on_swappedout_sparse_q) {
735 queue_remove(&c_swappedout_sparse_list_head, c_seg, c_segment_t, c_age_list);
736 c_seg->c_on_swappedout_sparse_q = 0;
737 c_swappedout_sparse_count--;
738 }
739 #if TRACK_BAD_C_SEGMENTS
740 else if (c_seg->c_on_bad_q) {
741 queue_remove(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
742 c_seg->c_on_bad_q = 0;
743 c_bad_count--;
744 }
745 #endif
746 segno = c_seg->c_mysegno;
747 c_segments[segno].c_segno = c_free_segno_head;
748 c_free_segno_head = segno;
749 c_segment_count--;
750
751 lck_mtx_unlock_always(c_list_lock);
752
753 if (c_seg->c_wanted) {
754 thread_wakeup((event_t) (c_seg));
755 c_seg->c_wanted = 0;
756 }
757 if (c_seg->c_busy_swapping) {
758 c_seg->c_must_free = 1;
759
760 lck_mtx_unlock_always(&c_seg->c_lock);
761 return;
762 }
763 if (c_seg->c_ondisk == 0) {
764 pages_populated = (round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE;
765
766 c_buffer = c_seg->c_store.c_buffer;
767 c_seg->c_store.c_buffer = NULL;
768 } else {
769 /*
770 * Free swap space on disk.
771 */
772 c_swap_handle = c_seg->c_store.c_swap_handle;
773 c_seg->c_store.c_swap_handle = (uint64_t)-1;
774 }
775 lck_mtx_unlock_always(&c_seg->c_lock);
776
777 if (c_buffer) {
778 kernel_memory_depopulate(kernel_map, (vm_offset_t) c_buffer, pages_populated * PAGE_SIZE, KMA_COMPRESSOR);
779
780 kmem_free(kernel_map, (vm_offset_t) c_buffer, C_SEG_ALLOCSIZE);
781 } else if (c_swap_handle)
782 vm_swap_free(c_swap_handle);
783
784
785 #if __i386__ || __x86_64__
786 lck_mtx_destroy(&c_seg->c_lock, &vm_compressor_lck_grp);
787 #else /* __i386__ || __x86_64__ */
788 lck_spin_destroy(&c_seg->c_lock, &vm_compressor_lck_grp);
789 #endif /* __i386__ || __x86_64__ */
790
791 for (i = 0; i < C_SEG_SLOT_ARRAYS; i++) {
792 if (c_seg->c_slots[i] == 0)
793 break;
794
795 kfree((char *)c_seg->c_slots[i], sizeof(struct c_slot) * C_SEG_SLOT_ARRAY_SIZE);
796 }
797 zfree(compressor_segment_zone, c_seg);
798 }
799
800
801 int c_seg_trim_page_count = 0;
802
803 void
804 c_seg_trim_tail(c_segment_t c_seg)
805 {
806 c_slot_t cs;
807 uint32_t c_size;
808 uint32_t c_offset;
809 uint32_t c_rounded_size;
810 uint16_t current_nextslot;
811 uint32_t current_populated_offset;
812
813 if (c_seg->c_bytes_used == 0)
814 return;
815 current_nextslot = c_seg->c_nextslot;
816 current_populated_offset = c_seg->c_populated_offset;
817
818 while (c_seg->c_nextslot) {
819
820 cs = C_SEG_SLOT_FROM_INDEX(c_seg, (c_seg->c_nextslot - 1));
821
822 c_size = UNPACK_C_SIZE(cs);
823
824 if (c_size) {
825 if (current_nextslot != c_seg->c_nextslot) {
826 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
827 c_offset = cs->c_offset + C_SEG_BYTES_TO_OFFSET(c_rounded_size);
828
829 c_seg->c_nextoffset = c_offset;
830 c_seg->c_populated_offset = (c_offset + (C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1)) & ~(C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1);
831
832 if (c_seg->c_firstemptyslot > c_seg->c_nextslot)
833 c_seg->c_firstemptyslot = c_seg->c_nextslot;
834
835 c_seg_trim_page_count += ((round_page_32(C_SEG_OFFSET_TO_BYTES(current_populated_offset)) -
836 round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE);
837 }
838 break;
839 }
840 c_seg->c_nextslot--;
841 }
842 assert(c_seg->c_nextslot);
843 }
844
845
846 int
847 c_seg_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy)
848 {
849 c_slot_mapping_t slot_ptr;
850 uint32_t c_offset = 0;
851 uint32_t old_populated_offset;
852 uint32_t c_rounded_size;
853 uint32_t c_size;
854 int c_indx = 0;
855 int i;
856 c_slot_t c_dst;
857 c_slot_t c_src;
858 boolean_t need_unlock = TRUE;
859
860 assert(c_seg->c_busy);
861
862 #if VALIDATE_C_SEGMENTS
863 c_seg_validate(c_seg, FALSE);
864 #endif
865 if (c_seg->c_bytes_used == 0) {
866 c_seg_free(c_seg);
867 return (1);
868 }
869 if (c_seg->c_firstemptyslot >= c_seg->c_nextslot || C_SEG_UNUSED_BYTES(c_seg) < PAGE_SIZE)
870 goto done;
871
872 #if VALIDATE_C_SEGMENTS
873 c_seg->c_was_minor_compacted++;
874 #endif
875 c_indx = c_seg->c_firstemptyslot;
876 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
877
878 old_populated_offset = c_seg->c_populated_offset;
879 c_offset = c_dst->c_offset;
880
881 for (i = c_indx + 1; i < c_seg->c_nextslot && c_offset < c_seg->c_nextoffset; i++) {
882
883 c_src = C_SEG_SLOT_FROM_INDEX(c_seg, i);
884
885 c_size = UNPACK_C_SIZE(c_src);
886
887 if (c_size == 0)
888 continue;
889
890 memcpy(&c_seg->c_store.c_buffer[c_offset], &c_seg->c_store.c_buffer[c_src->c_offset], c_size);
891
892 #if CHECKSUM_THE_DATA
893 c_dst->c_hash_data = c_src->c_hash_data;
894 #endif
895 #if CHECKSUM_THE_COMPRESSED_DATA
896 c_dst->c_hash_compressed_data = c_src->c_hash_compressed_data;
897 #endif
898 c_dst->c_size = c_src->c_size;
899 c_dst->c_packed_ptr = c_src->c_packed_ptr;
900 c_dst->c_offset = c_offset;
901
902 slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
903 slot_ptr->s_cindx = c_indx;
904
905 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
906
907 c_offset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
908 PACK_C_SIZE(c_src, 0);
909 c_indx++;
910
911 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
912 }
913 c_seg->c_firstemptyslot = c_indx;
914 c_seg->c_nextslot = c_indx;
915 c_seg->c_nextoffset = c_offset;
916 c_seg->c_populated_offset = (c_offset + (C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1)) & ~(C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1);
917 c_seg->c_bytes_unused = 0;
918
919 #if VALIDATE_C_SEGMENTS
920 c_seg_validate(c_seg, TRUE);
921 #endif
922
923 if (old_populated_offset > c_seg->c_populated_offset) {
924 uint32_t gc_size;
925 int32_t *gc_ptr;
926
927 gc_size = C_SEG_OFFSET_TO_BYTES(old_populated_offset - c_seg->c_populated_offset);
928 gc_ptr = &c_seg->c_store.c_buffer[c_seg->c_populated_offset];
929
930 lck_mtx_unlock_always(&c_seg->c_lock);
931
932 kernel_memory_depopulate(kernel_map, (vm_offset_t)gc_ptr, gc_size, KMA_COMPRESSOR);
933
934 if (clear_busy == TRUE)
935 lck_mtx_lock_spin_always(&c_seg->c_lock);
936 else
937 need_unlock = FALSE;
938 }
939 done:
940 if (need_unlock == TRUE) {
941 if (clear_busy == TRUE)
942 C_SEG_WAKEUP_DONE(c_seg);
943
944 lck_mtx_unlock_always(&c_seg->c_lock);
945 }
946 return (0);
947 }
948
949
950
951 struct {
952 uint64_t asked_permission;
953 uint64_t compactions;
954 uint64_t moved_slots;
955 uint64_t moved_bytes;
956 uint64_t wasted_space_in_swapouts;
957 uint64_t count_of_swapouts;
958 } c_seg_major_compact_stats;
959
960
961 #define C_MAJOR_COMPACTION_AGE_APPROPRIATE 30
962 #define C_MAJOR_COMPACTION_OLD_ENOUGH 300
963 #define C_MAJOR_COMPACTION_SIZE_APPROPRIATE ((C_SEG_BUFSIZE * 80) / 100)
964
965
966 boolean_t
967 c_seg_major_compact_ok(
968 c_segment_t c_seg_dst,
969 c_segment_t c_seg_src)
970 {
971
972 c_seg_major_compact_stats.asked_permission++;
973
974 if (c_seg_src->c_filling) {
975 /*
976 * we're at or near the head... don't compact
977 */
978 return (FALSE);
979 }
980 if (c_seg_src->c_bytes_used >= C_MAJOR_COMPACTION_SIZE_APPROPRIATE &&
981 c_seg_dst->c_bytes_used >= C_MAJOR_COMPACTION_SIZE_APPROPRIATE)
982 return (FALSE);
983
984 if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX) {
985 /*
986 * destination segment is full... can't compact
987 */
988 return (FALSE);
989 }
990
991 return (TRUE);
992 }
993
994
995 boolean_t
996 c_seg_major_compact(
997 c_segment_t c_seg_dst,
998 c_segment_t c_seg_src)
999 {
1000 c_slot_mapping_t slot_ptr;
1001 uint32_t c_rounded_size;
1002 uint32_t c_size;
1003 uint16_t dst_slot;
1004 int i;
1005 c_slot_t c_dst;
1006 c_slot_t c_src;
1007 int slotarray;
1008 boolean_t keep_compacting = TRUE;
1009
1010 /*
1011 * segments are not locked but they are both marked c_busy
1012 * which keeps c_decompress from working on them...
1013 * we can safely allocate new pages, move compressed data
1014 * from c_seg_src to c_seg_dst and update both c_segment's
1015 * state w/o holding the master lock
1016 */
1017
1018 #if VALIDATE_C_SEGMENTS
1019 c_seg_dst->c_was_major_compacted++;
1020 c_seg_src->c_was_major_donor++;
1021 #endif
1022 c_seg_major_compact_stats.compactions++;
1023
1024 dst_slot = c_seg_dst->c_nextslot;
1025
1026 for (i = 0; i < c_seg_src->c_nextslot; i++) {
1027
1028 c_src = C_SEG_SLOT_FROM_INDEX(c_seg_src, i);
1029
1030 c_size = UNPACK_C_SIZE(c_src);
1031
1032 if (c_size == 0) {
1033 /* BATCH: move what we have so far; */
1034 continue;
1035 }
1036
1037 if (C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset - c_seg_dst->c_nextoffset) < (unsigned) c_size) {
1038 /* doesn't fit */
1039 if ((C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset) == C_SEG_BUFSIZE)) {
1040 /* can't fit */
1041 keep_compacting = FALSE;
1042 break;
1043 }
1044 kernel_memory_populate(kernel_map,
1045 (vm_offset_t) &c_seg_dst->c_store.c_buffer[c_seg_dst->c_populated_offset],
1046 PAGE_SIZE,
1047 KMA_COMPRESSOR);
1048
1049 c_seg_dst->c_populated_offset += C_SEG_BYTES_TO_OFFSET(PAGE_SIZE);
1050 assert(C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset) <= C_SEG_BUFSIZE);
1051 }
1052
1053 slotarray = C_SEG_SLOTARRAY_FROM_INDEX(c_seg_dst, c_seg_dst->c_nextslot);
1054
1055 if (c_seg_dst->c_slots[slotarray] == 0) {
1056 KERNEL_DEBUG(0xe0400008 | DBG_FUNC_START, 0, 0, 0, 0, 0);
1057 c_seg_dst->c_slots[slotarray] = (struct c_slot *)
1058 kalloc(sizeof(struct c_slot) *
1059 C_SEG_SLOT_ARRAY_SIZE);
1060 KERNEL_DEBUG(0xe0400008 | DBG_FUNC_END, 0, 0, 0, 0, 0);
1061 }
1062 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, c_seg_dst->c_nextslot);
1063
1064 memcpy(&c_seg_dst->c_store.c_buffer[c_seg_dst->c_nextoffset], &c_seg_src->c_store.c_buffer[c_src->c_offset], c_size);
1065
1066 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
1067
1068 c_seg_major_compact_stats.moved_slots++;
1069 c_seg_major_compact_stats.moved_bytes += c_size;
1070
1071 #if CHECKSUM_THE_DATA
1072 c_dst->c_hash_data = c_src->c_hash_data;
1073 #endif
1074 #if CHECKSUM_THE_COMPRESSED_DATA
1075 c_dst->c_hash_compressed_data = c_src->c_hash_compressed_data;
1076 #endif
1077 c_dst->c_size = c_src->c_size;
1078 c_dst->c_packed_ptr = c_src->c_packed_ptr;
1079 c_dst->c_offset = c_seg_dst->c_nextoffset;
1080
1081 if (c_seg_dst->c_firstemptyslot == c_seg_dst->c_nextslot)
1082 c_seg_dst->c_firstemptyslot++;
1083 c_seg_dst->c_nextslot++;
1084 c_seg_dst->c_bytes_used += c_rounded_size;
1085 c_seg_dst->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
1086
1087 PACK_C_SIZE(c_src, 0);
1088
1089 c_seg_src->c_bytes_used -= c_rounded_size;
1090 c_seg_src->c_bytes_unused += c_rounded_size;
1091 c_seg_src->c_firstemptyslot = 0;
1092
1093 if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX) {
1094 /* dest segment is now full */
1095 keep_compacting = FALSE;
1096 break;
1097 }
1098 }
1099 if (dst_slot < c_seg_dst->c_nextslot) {
1100
1101 PAGE_REPLACEMENT_ALLOWED(TRUE);
1102 /*
1103 * we've now locked out c_decompress from
1104 * converting the slot passed into it into
1105 * a c_segment_t which allows us to use
1106 * the backptr to change which c_segment and
1107 * index the slot points to
1108 */
1109 while (dst_slot < c_seg_dst->c_nextslot) {
1110
1111 c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, dst_slot);
1112
1113 slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
1114 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
1115 slot_ptr->s_cseg = c_seg_dst->c_mysegno + 1;
1116 slot_ptr->s_cindx = dst_slot++;
1117 }
1118 PAGE_REPLACEMENT_ALLOWED(FALSE);
1119 }
1120 return (keep_compacting);
1121 }
1122
1123
1124 static uint64_t
1125 compute_elapsed_msecs(clock_sec_t end_sec, clock_nsec_t end_nsec, clock_sec_t start_sec, clock_nsec_t start_nsec)
1126 {
1127 uint64_t end_msecs;
1128 uint64_t start_msecs;
1129
1130 end_msecs = (end_sec * 1000) + end_nsec / 1000000;
1131 start_msecs = (start_sec * 1000) + start_nsec / 1000000;
1132
1133 return (end_msecs - start_msecs);
1134 }
1135
1136
1137
1138 uint32_t compressor_eval_period_in_msecs = 250;
1139 uint32_t compressor_sample_min_in_msecs = 500;
1140 uint32_t compressor_sample_max_in_msecs = 10000;
1141 uint32_t compressor_thrashing_threshold_per_10msecs = 50;
1142 uint32_t compressor_thrashing_min_per_10msecs = 20;
1143
1144 extern uint32_t vm_page_filecache_min;
1145
1146
1147 void
1148 compute_swapout_target_age(void)
1149 {
1150 clock_sec_t cur_ts_sec;
1151 clock_nsec_t cur_ts_nsec;
1152 uint32_t min_operations_needed_in_this_sample;
1153 uint64_t elapsed_msecs_in_eval;
1154 uint64_t elapsed_msecs_in_sample;
1155 boolean_t need_sample_reset = FALSE;
1156 boolean_t need_eval_reset = FALSE;
1157
1158 clock_get_system_nanotime(&cur_ts_sec, &cur_ts_nsec);
1159
1160 elapsed_msecs_in_sample = compute_elapsed_msecs(cur_ts_sec, cur_ts_nsec, start_of_sample_period_sec, start_of_sample_period_nsec);
1161
1162 if (elapsed_msecs_in_sample >= compressor_sample_max_in_msecs) {
1163 need_sample_reset = TRUE;
1164 need_eval_reset = TRUE;
1165 goto done;
1166 }
1167 elapsed_msecs_in_eval = compute_elapsed_msecs(cur_ts_sec, cur_ts_nsec, start_of_eval_period_sec, start_of_eval_period_nsec);
1168
1169 if (elapsed_msecs_in_eval < compressor_eval_period_in_msecs)
1170 goto done;
1171 need_eval_reset = TRUE;
1172
1173 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_START, elapsed_msecs_in_eval, sample_period_compression_count, sample_period_decompression_count, 0, 0);
1174
1175 min_operations_needed_in_this_sample = (compressor_thrashing_min_per_10msecs * (uint32_t)elapsed_msecs_in_eval) / 10;
1176
1177 if ((sample_period_compression_count - last_eval_compression_count) < min_operations_needed_in_this_sample ||
1178 (sample_period_decompression_count - last_eval_decompression_count) < min_operations_needed_in_this_sample) {
1179
1180 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, sample_period_compression_count - last_eval_compression_count,
1181 sample_period_decompression_count - last_eval_decompression_count, 0, 1, 0);
1182
1183 swapout_target_age = 0;
1184
1185 need_sample_reset = TRUE;
1186 need_eval_reset = TRUE;
1187 goto done;
1188 }
1189 last_eval_compression_count = sample_period_compression_count;
1190 last_eval_decompression_count = sample_period_decompression_count;
1191
1192 if (elapsed_msecs_in_sample < compressor_sample_min_in_msecs) {
1193
1194 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, swapout_target_age, 0, 0, 5, 0);
1195 goto done;
1196 }
1197 if (sample_period_decompression_count > ((compressor_thrashing_threshold_per_10msecs * elapsed_msecs_in_sample) / 10)) {
1198
1199 uint64_t running_total;
1200 uint64_t working_target;
1201 uint64_t aging_target;
1202 uint32_t oldest_age_of_csegs_sampled = 0;
1203 uint64_t working_set_approximation = 0;
1204
1205 swapout_target_age = 0;
1206
1207 working_target = (sample_period_decompression_count / 100) * 95; /* 95 percent */
1208 aging_target = (sample_period_decompression_count / 100) * 1; /* 1 percent */
1209 running_total = 0;
1210
1211 for (oldest_age_of_csegs_sampled = 0; oldest_age_of_csegs_sampled < DECOMPRESSION_SAMPLE_MAX_AGE; oldest_age_of_csegs_sampled++) {
1212
1213 running_total += age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
1214
1215 working_set_approximation += oldest_age_of_csegs_sampled * age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
1216
1217 if (running_total >= working_target)
1218 break;
1219 }
1220 if (oldest_age_of_csegs_sampled < DECOMPRESSION_SAMPLE_MAX_AGE) {
1221
1222 working_set_approximation = (working_set_approximation * 1000) / elapsed_msecs_in_sample;
1223
1224 if (working_set_approximation < VM_PAGE_COMPRESSOR_COUNT) {
1225
1226 running_total = overage_decompressions_during_sample_period;
1227
1228 for (oldest_age_of_csegs_sampled = DECOMPRESSION_SAMPLE_MAX_AGE - 1; oldest_age_of_csegs_sampled; oldest_age_of_csegs_sampled--) {
1229 running_total += age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
1230
1231 if (running_total >= aging_target)
1232 break;
1233 }
1234 swapout_target_age = (uint32_t)cur_ts_sec - oldest_age_of_csegs_sampled;
1235
1236 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, swapout_target_age, working_set_approximation, VM_PAGE_COMPRESSOR_COUNT, 2, 0);
1237 } else {
1238 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, working_set_approximation, VM_PAGE_COMPRESSOR_COUNT, 0, 3, 0);
1239 }
1240 } else
1241 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, working_target, running_total, 0, 4, 0);
1242
1243 need_sample_reset = TRUE;
1244 need_eval_reset = TRUE;
1245 } else
1246 KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, sample_period_decompression_count, (compressor_thrashing_threshold_per_10msecs * elapsed_msecs_in_sample) / 10, 0, 6, 0);
1247 done:
1248 if (need_sample_reset == TRUE) {
1249 bzero(age_of_decompressions_during_sample_period, sizeof(age_of_decompressions_during_sample_period));
1250 overage_decompressions_during_sample_period = 0;
1251
1252 start_of_sample_period_sec = cur_ts_sec;
1253 start_of_sample_period_nsec = cur_ts_nsec;
1254 sample_period_decompression_count = 0;
1255 sample_period_compression_count = 0;
1256 last_eval_decompression_count = 0;
1257 last_eval_compression_count = 0;
1258 }
1259 if (need_eval_reset == TRUE) {
1260 start_of_eval_period_sec = cur_ts_sec;
1261 start_of_eval_period_nsec = cur_ts_nsec;
1262 }
1263 }
1264
1265
1266
1267 int calls_since_last_considered = 0;
1268 int compaction_swapper_running = 0;
1269 int compaction_swapper_abort = 0;
1270
1271
1272 #if CONFIG_JETSAM
1273 boolean_t memorystatus_kill_on_VM_thrashing(boolean_t);
1274 int compressor_thrashing_induced_jetsam = 0;
1275 boolean_t vm_compressor_thrashing_detected = FALSE;
1276 #endif /* CONFIG_JETSAM */
1277
1278 static boolean_t
1279 compressor_needs_to_swap(void)
1280 {
1281 boolean_t should_swap = FALSE;
1282
1283 if (vm_swap_up == TRUE) {
1284 if (COMPRESSOR_NEEDS_TO_SWAP()) {
1285 return (TRUE);
1286 }
1287 if (VM_PAGE_Q_THROTTLED(&vm_pageout_queue_external) && vm_page_anonymous_count < (vm_page_inactive_count / 20)) {
1288 return (TRUE);
1289 }
1290 if (vm_page_free_count < (vm_page_free_reserved - COMPRESSOR_FREE_RESERVED_LIMIT))
1291 return (TRUE);
1292 }
1293 compute_swapout_target_age();
1294
1295 if (swapout_target_age) {
1296 c_segment_t c_seg;
1297
1298 lck_mtx_lock_spin_always(c_list_lock);
1299
1300 if (!queue_empty(&c_age_list_head)) {
1301
1302 c_seg = (c_segment_t) queue_first(&c_age_list_head);
1303
1304 if (c_seg->c_creation_ts <= swapout_target_age)
1305 should_swap = TRUE;
1306 else
1307 swapout_target_age = 0;
1308 }
1309 lck_mtx_unlock_always(c_list_lock);
1310 }
1311
1312 if (vm_swap_up == FALSE) {
1313 #if CONFIG_JETSAM
1314 if (should_swap) {
1315 if (vm_compressor_thrashing_detected == FALSE) {
1316 vm_compressor_thrashing_detected = TRUE;
1317 memorystatus_kill_on_VM_thrashing(TRUE /* async */);
1318 compressor_thrashing_induced_jetsam++;
1319 /*
1320 * let the jetsam take precedence over
1321 * any major compactions we might have
1322 * been able to do... otherwise we run
1323 * the risk of doing major compactions
1324 * on segments we're about to free up
1325 * due to the jetsam activity.
1326 */
1327 should_swap = FALSE;
1328 }
1329 } else
1330 #endif /* CONFIG_JETSAM */
1331 if (COMPRESSOR_NEEDS_TO_MAJOR_COMPACT())
1332 should_swap = TRUE;
1333 }
1334 /*
1335 * returning TRUE when swap_supported == FALSE
1336 * will cause the major compaction engine to
1337 * run, but will not trigger any swapping...
1338 * segments that have been major compacted
1339 * will be moved to the swapped_out_q
1340 * but will not have the c_ondisk flag set
1341 */
1342 return (should_swap);
1343 }
1344
1345 uint64_t
1346 vm_compressor_total_compressions(void)
1347 {
1348 processor_t processor = processor_list;
1349 vm_statistics64_t stat = &PROCESSOR_DATA(processor, vm_stat);
1350
1351 uint64_t compressions = stat->compressions;
1352
1353 if (processor_count > 1) {
1354 simple_lock(&processor_list_lock);
1355
1356 while ((processor = processor->processor_list) != NULL) {
1357 stat = &PROCESSOR_DATA(processor, vm_stat);
1358 compressions += stat->compressions;
1359 }
1360
1361 simple_unlock(&processor_list_lock);
1362 }
1363
1364 return compressions;
1365 }
1366
1367 uint32_t vm_wake_compactor_swapper_calls = 0;
1368
1369 void
1370 vm_wake_compactor_swapper(void)
1371 {
1372 if (compaction_swapper_running)
1373 return;
1374
1375 if (c_minor_count == 0)
1376 return;
1377
1378 lck_mtx_lock_spin_always(c_list_lock);
1379
1380 fastwake_warmup = FALSE;
1381
1382 if (compaction_swapper_running == 0) {
1383 vm_wake_compactor_swapper_calls++;
1384
1385 thread_wakeup((event_t)&c_compressor_swap_trigger);
1386
1387 compaction_swapper_running = 1;
1388 }
1389 lck_mtx_unlock_always(c_list_lock);
1390 }
1391
1392 void
1393 vm_consider_waking_compactor_swapper(void)
1394 {
1395 boolean_t need_wakeup = FALSE;
1396
1397 if (calls_since_last_considered++ < 1000 || compaction_swapper_running)
1398 return;
1399 calls_since_last_considered = 0;
1400
1401 if (c_minor_count && (COMPRESSOR_NEEDS_TO_MINOR_COMPACT())) {
1402
1403 need_wakeup = TRUE;
1404
1405 } else if (compressor_needs_to_swap()) {
1406
1407 need_wakeup = TRUE;
1408
1409 } else if (c_minor_count) {
1410 uint64_t total_bytes;
1411
1412 total_bytes = compressor_object->resident_page_count * PAGE_SIZE_64;
1413
1414 if ((total_bytes - compressor_bytes_used) > total_bytes / 10)
1415 need_wakeup = TRUE;
1416 }
1417 if (need_wakeup == TRUE) {
1418
1419 lck_mtx_lock_spin_always(c_list_lock);
1420
1421 fastwake_warmup = FALSE;
1422
1423 if (compaction_swapper_running == 0) {
1424 memoryshot(VM_WAKEUP_COMPACTOR_SWAPPER, DBG_FUNC_NONE);
1425
1426 thread_wakeup((event_t)&c_compressor_swap_trigger);
1427
1428 compaction_swapper_running = 1;
1429 }
1430 lck_mtx_unlock_always(c_list_lock);
1431 }
1432 }
1433
1434
1435 #define C_SWAPOUT_LIMIT 4
1436 #define DELAYED_COMPACTIONS_PER_PASS 30
1437
1438 void
1439 vm_compressor_do_delayed_compactions(boolean_t flush_all)
1440 {
1441 c_segment_t c_seg;
1442 int number_compacted = 0;
1443 boolean_t needs_to_swap = FALSE;
1444
1445
1446 lck_mtx_assert(c_list_lock, LCK_MTX_ASSERT_OWNED);
1447
1448 while (!queue_empty(&c_minor_list_head) && needs_to_swap == FALSE) {
1449
1450 c_seg = (c_segment_t)queue_first(&c_minor_list_head);
1451
1452 lck_mtx_lock_spin_always(&c_seg->c_lock);
1453 c_seg->c_busy = 1;
1454
1455 c_seg_do_minor_compaction_and_unlock(c_seg, TRUE, FALSE, TRUE);
1456
1457 if (vm_swap_up == TRUE && (number_compacted++ > DELAYED_COMPACTIONS_PER_PASS)) {
1458
1459 if ((flush_all == TRUE || compressor_needs_to_swap() == TRUE) && c_swapout_count < C_SWAPOUT_LIMIT)
1460 needs_to_swap = TRUE;
1461
1462 number_compacted = 0;
1463 }
1464 lck_mtx_lock_spin_always(c_list_lock);
1465 }
1466 }
1467
1468
1469 #define C_SEGMENT_SWAPPEDIN_AGE_LIMIT 10
1470
1471 static void
1472 vm_compressor_age_swapped_in_segments(boolean_t flush_all)
1473 {
1474 c_segment_t c_seg;
1475 clock_sec_t now;
1476 clock_nsec_t nsec;
1477
1478 clock_get_system_nanotime(&now, &nsec);
1479
1480 while (!queue_empty(&c_swappedin_list_head)) {
1481
1482 c_seg = (c_segment_t)queue_first(&c_swappedin_list_head);
1483
1484 if (flush_all == FALSE && (now - c_seg->c_swappedin_ts) < C_SEGMENT_SWAPPEDIN_AGE_LIMIT)
1485 break;
1486
1487 lck_mtx_lock_spin_always(&c_seg->c_lock);
1488
1489 queue_remove(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
1490 c_seg->c_on_swappedin_q = 0;
1491 c_swappedin_count--;
1492
1493 c_seg_insert_into_q(&c_age_list_head, c_seg);
1494 c_seg->c_on_age_q = 1;
1495 c_age_count++;
1496
1497 lck_mtx_unlock_always(&c_seg->c_lock);
1498 }
1499 }
1500
1501
1502 void
1503 vm_compressor_flush(void)
1504 {
1505 uint64_t vm_swap_put_failures_at_start;
1506 wait_result_t wait_result = 0;
1507 AbsoluteTime startTime, endTime;
1508 clock_sec_t now_sec;
1509 clock_nsec_t now_nsec;
1510 uint64_t nsec;
1511
1512 HIBLOG("vm_compressor_flush - starting\n");
1513
1514 clock_get_uptime(&startTime);
1515
1516 lck_mtx_lock_spin_always(c_list_lock);
1517
1518 fastwake_warmup = FALSE;
1519 compaction_swapper_abort = 1;
1520
1521 while (compaction_swapper_running) {
1522 assert_wait((event_t)&compaction_swapper_running, THREAD_UNINT);
1523
1524 lck_mtx_unlock_always(c_list_lock);
1525
1526 thread_block(THREAD_CONTINUE_NULL);
1527
1528 lck_mtx_lock_spin_always(c_list_lock);
1529 }
1530 compaction_swapper_abort = 0;
1531 compaction_swapper_running = 1;
1532
1533 hibernate_flushing = TRUE;
1534 hibernate_no_swapspace = FALSE;
1535 c_generation_id_flush_barrier = c_generation_id + 1000;
1536
1537 clock_get_system_nanotime(&now_sec, &now_nsec);
1538 hibernate_flushing_deadline = now_sec + HIBERNATE_FLUSHING_SECS_TO_COMPLETE;
1539
1540 vm_swap_put_failures_at_start = vm_swap_put_failures;
1541
1542 vm_compressor_compact_and_swap(TRUE);
1543
1544 while (!queue_empty(&c_swapout_list_head)) {
1545
1546 assert_wait_timeout((event_t) &compaction_swapper_running, THREAD_INTERRUPTIBLE, 5000, 1000*NSEC_PER_USEC);
1547
1548 lck_mtx_unlock_always(c_list_lock);
1549
1550 wait_result = thread_block(THREAD_CONTINUE_NULL);
1551
1552 lck_mtx_lock_spin_always(c_list_lock);
1553
1554 if (wait_result == THREAD_TIMED_OUT)
1555 break;
1556 }
1557 hibernate_flushing = FALSE;
1558 compaction_swapper_running = 0;
1559
1560 if (vm_swap_put_failures > vm_swap_put_failures_at_start)
1561 HIBLOG("vm_compressor_flush failed to clean %llu segments - vm_page_compressor_count(%d)\n",
1562 vm_swap_put_failures - vm_swap_put_failures_at_start, VM_PAGE_COMPRESSOR_COUNT);
1563
1564 lck_mtx_unlock_always(c_list_lock);
1565
1566 clock_get_uptime(&endTime);
1567 SUB_ABSOLUTETIME(&endTime, &startTime);
1568 absolutetime_to_nanoseconds(endTime, &nsec);
1569
1570 HIBLOG("vm_compressor_flush completed - took %qd msecs\n", nsec / 1000000ULL);
1571 }
1572
1573
1574
1575 int compaction_swap_trigger_thread_awakened = 0;
1576
1577 static void
1578 vm_compressor_swap_trigger_thread(void)
1579 {
1580
1581 lck_mtx_lock_spin_always(c_list_lock);
1582
1583 compaction_swap_trigger_thread_awakened++;
1584
1585 vm_compressor_compact_and_swap(FALSE);
1586
1587 assert_wait((event_t)&c_compressor_swap_trigger, THREAD_UNINT);
1588
1589 compaction_swapper_running = 0;
1590 thread_wakeup((event_t)&compaction_swapper_running);
1591
1592 lck_mtx_unlock_always(c_list_lock);
1593
1594 thread_block((thread_continue_t)vm_compressor_swap_trigger_thread);
1595
1596 /* NOTREACHED */
1597 }
1598
1599
1600 void
1601 vm_compressor_record_warmup_start(void)
1602 {
1603 c_segment_t c_seg;
1604
1605 lck_mtx_lock_spin_always(c_list_lock);
1606
1607 if (!queue_empty(&c_age_list_head)) {
1608
1609 c_seg = (c_segment_t)queue_last(&c_age_list_head);
1610
1611 first_c_segment_to_warm_generation_id = c_seg->c_generation_id;
1612 } else
1613 first_c_segment_to_warm_generation_id = 0;
1614
1615 fastwake_recording_in_progress = TRUE;
1616
1617 lck_mtx_unlock_always(c_list_lock);
1618 }
1619
1620
1621 void
1622 vm_compressor_record_warmup_end(void)
1623 {
1624 c_segment_t c_seg;
1625
1626 lck_mtx_lock_spin_always(c_list_lock);
1627
1628 if (!queue_empty(&c_age_list_head)) {
1629
1630 c_seg = (c_segment_t)queue_last(&c_age_list_head);
1631
1632 last_c_segment_to_warm_generation_id = c_seg->c_generation_id;
1633 } else
1634 last_c_segment_to_warm_generation_id = first_c_segment_to_warm_generation_id;
1635
1636 fastwake_recording_in_progress = FALSE;
1637
1638 lck_mtx_unlock_always(c_list_lock);
1639 }
1640
1641
1642 #define DELAY_TRIM_ON_WAKE_SECS 4
1643
1644 void
1645 vm_compressor_do_warmup(void)
1646 {
1647 clock_sec_t sec;
1648 clock_nsec_t nsec;
1649
1650 clock_get_system_nanotime(&sec, &nsec);
1651 dont_trim_until_ts = sec + DELAY_TRIM_ON_WAKE_SECS;
1652
1653 if (first_c_segment_to_warm_generation_id == last_c_segment_to_warm_generation_id)
1654 return;
1655
1656 lck_mtx_lock_spin_always(c_list_lock);
1657
1658 if (compaction_swapper_running == 0) {
1659
1660 fastwake_warmup = TRUE;
1661 compaction_swapper_running = 1;
1662 thread_wakeup((event_t)&c_compressor_swap_trigger);
1663 }
1664 lck_mtx_unlock_always(c_list_lock);
1665 }
1666
1667
1668 void
1669 do_fastwake_warmup(void)
1670 {
1671 uint64_t my_thread_id;
1672 c_segment_t c_seg = NULL;
1673
1674 lck_mtx_unlock_always(c_list_lock);
1675
1676 my_thread_id = current_thread()->thread_id;
1677 proc_set_task_policy_thread(kernel_task, my_thread_id,
1678 TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER2);
1679
1680 PAGE_REPLACEMENT_DISALLOWED(TRUE);
1681
1682 lck_mtx_lock_spin_always(c_list_lock);
1683
1684 while (!queue_empty(&c_swappedout_list_head) && fastwake_warmup == TRUE) {
1685
1686 c_seg = (c_segment_t) queue_first(&c_swappedout_list_head);
1687
1688 if (c_seg->c_generation_id < first_c_segment_to_warm_generation_id ||
1689 c_seg->c_generation_id > last_c_segment_to_warm_generation_id)
1690 break;
1691
1692 lck_mtx_lock_spin_always(&c_seg->c_lock);
1693 lck_mtx_unlock_always(c_list_lock);
1694
1695 if (c_seg->c_busy)
1696 c_seg_wait_on_busy(c_seg);
1697 else {
1698 c_seg_swapin(c_seg, TRUE);
1699
1700 lck_mtx_unlock_always(&c_seg->c_lock);
1701
1702 c_segment_warmup_count++;
1703 vm_pageout_io_throttle();
1704 }
1705 lck_mtx_lock_spin_always(c_list_lock);
1706 }
1707 lck_mtx_unlock_always(c_list_lock);
1708
1709 PAGE_REPLACEMENT_DISALLOWED(FALSE);
1710
1711 proc_set_task_policy_thread(kernel_task, my_thread_id,
1712 TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER0);
1713
1714 lck_mtx_lock_spin_always(c_list_lock);
1715 }
1716
1717
1718 void
1719 vm_compressor_compact_and_swap(boolean_t flush_all)
1720 {
1721 c_segment_t c_seg, c_seg_next;
1722 boolean_t keep_compacting;
1723
1724
1725 if (fastwake_warmup == TRUE) {
1726 uint64_t starting_warmup_count;
1727
1728 starting_warmup_count = c_segment_warmup_count;
1729
1730 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 11) | DBG_FUNC_START, c_segment_warmup_count,
1731 first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id, 0, 0);
1732 do_fastwake_warmup();
1733 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 11) | DBG_FUNC_END, c_segment_warmup_count, c_segment_warmup_count - starting_warmup_count, 0, 0, 0);
1734
1735 fastwake_warmup = FALSE;
1736 }
1737
1738 while (!queue_empty(&c_age_list_head) && compaction_swapper_abort == 0) {
1739
1740 if (hibernate_flushing == TRUE) {
1741 clock_sec_t sec;
1742 clock_nsec_t nsec;
1743
1744 if (hibernate_should_abort()) {
1745 HIBLOG("vm_compressor_flush - hibernate_should_abort returned TRUE\n");
1746 break;
1747 }
1748 if (hibernate_no_swapspace == TRUE) {
1749 HIBLOG("vm_compressor_flush - out of swap space\n");
1750 break;
1751 }
1752 clock_get_system_nanotime(&sec, &nsec);
1753
1754 if (sec > hibernate_flushing_deadline) {
1755 HIBLOG("vm_compressor_flush - failed to finish before deadline\n");
1756 break;
1757 }
1758 }
1759 if (c_swapout_count >= C_SWAPOUT_LIMIT) {
1760
1761 assert_wait_timeout((event_t) &compaction_swapper_running, THREAD_INTERRUPTIBLE, 100, 1000*NSEC_PER_USEC);
1762
1763 lck_mtx_unlock_always(c_list_lock);
1764
1765 thread_block(THREAD_CONTINUE_NULL);
1766
1767 lck_mtx_lock_spin_always(c_list_lock);
1768 }
1769 /*
1770 * Minor compactions
1771 */
1772 vm_compressor_do_delayed_compactions(flush_all);
1773
1774 vm_compressor_age_swapped_in_segments(flush_all);
1775
1776 if (c_swapout_count >= C_SWAPOUT_LIMIT) {
1777 /*
1778 * we timed out on the above thread_block
1779 * let's loop around and try again
1780 * the timeout allows us to continue
1781 * to do minor compactions to make
1782 * more memory available
1783 */
1784 continue;
1785 }
1786
1787 /*
1788 * Swap out segments?
1789 */
1790 if (flush_all == FALSE) {
1791 boolean_t needs_to_swap;
1792
1793 lck_mtx_unlock_always(c_list_lock);
1794
1795 needs_to_swap = compressor_needs_to_swap();
1796
1797 lck_mtx_lock_spin_always(c_list_lock);
1798
1799 if (needs_to_swap == FALSE)
1800 break;
1801 }
1802 if (queue_empty(&c_age_list_head))
1803 break;
1804 c_seg = (c_segment_t) queue_first(&c_age_list_head);
1805
1806 if (flush_all == TRUE && c_seg->c_generation_id > c_generation_id_flush_barrier)
1807 break;
1808
1809 if (c_seg->c_filling) {
1810 /*
1811 * we're at or near the head... no more work to do
1812 */
1813 break;
1814 }
1815 lck_mtx_lock_spin_always(&c_seg->c_lock);
1816
1817 if (c_seg->c_busy) {
1818
1819 lck_mtx_unlock_always(c_list_lock);
1820 c_seg_wait_on_busy(c_seg);
1821 lck_mtx_lock_spin_always(c_list_lock);
1822
1823 continue;
1824 }
1825 c_seg->c_busy = 1;
1826
1827 if (c_seg_do_minor_compaction_and_unlock(c_seg, FALSE, TRUE, TRUE)) {
1828 /*
1829 * found an empty c_segment and freed it
1830 * so go grab the next guy in the queue
1831 */
1832 continue;
1833 }
1834 /*
1835 * Major compaction
1836 */
1837 keep_compacting = TRUE;
1838
1839 while (keep_compacting == TRUE) {
1840
1841 assert(c_seg->c_busy);
1842
1843 /* look for another segment to consolidate */
1844
1845 c_seg_next = (c_segment_t) queue_next(&c_seg->c_age_list);
1846
1847 if (queue_end(&c_age_list_head, (queue_entry_t)c_seg_next))
1848 break;
1849
1850 if (c_seg_major_compact_ok(c_seg, c_seg_next) == FALSE)
1851 break;
1852
1853 lck_mtx_lock_spin_always(&c_seg_next->c_lock);
1854
1855 if (c_seg_next->c_busy) {
1856
1857 lck_mtx_unlock_always(c_list_lock);
1858 c_seg_wait_on_busy(c_seg_next);
1859 lck_mtx_lock_spin_always(c_list_lock);
1860
1861 continue;
1862 }
1863 /* grab that segment */
1864 c_seg_next->c_busy = 1;
1865
1866 if (c_seg_do_minor_compaction_and_unlock(c_seg_next, FALSE, TRUE, TRUE)) {
1867 /*
1868 * found an empty c_segment and freed it
1869 * so we can't continue to use c_seg_next
1870 */
1871 continue;
1872 }
1873
1874 /* unlock the list ... */
1875 lck_mtx_unlock_always(c_list_lock);
1876
1877 /* do the major compaction */
1878
1879 keep_compacting = c_seg_major_compact(c_seg, c_seg_next);
1880
1881 PAGE_REPLACEMENT_DISALLOWED(TRUE);
1882
1883 lck_mtx_lock_spin_always(&c_seg_next->c_lock);
1884 /*
1885 * run a minor compaction on the donor segment
1886 * since we pulled at least some of it's
1887 * data into our target... if we've emptied
1888 * it, now is a good time to free it which
1889 * c_seg_minor_compaction_and_unlock also takes care of
1890 *
1891 * by passing TRUE, we ask for c_busy to be cleared
1892 * and c_wanted to be taken care of
1893 */
1894 c_seg_minor_compaction_and_unlock(c_seg_next, TRUE);
1895
1896 PAGE_REPLACEMENT_DISALLOWED(FALSE);
1897
1898 /* relock the list */
1899 lck_mtx_lock_spin_always(c_list_lock);
1900
1901 } /* major compaction */
1902
1903 c_seg_major_compact_stats.wasted_space_in_swapouts += C_SEG_BUFSIZE - c_seg->c_bytes_used;
1904 c_seg_major_compact_stats.count_of_swapouts++;
1905
1906 lck_mtx_lock_spin_always(&c_seg->c_lock);
1907
1908 assert(c_seg->c_busy);
1909 assert(c_seg->c_on_age_q);
1910 assert(!c_seg->c_on_minorcompact_q);
1911
1912 queue_remove(&c_age_list_head, c_seg, c_segment_t, c_age_list);
1913 c_seg->c_on_age_q = 0;
1914 c_age_count--;
1915
1916 if (vm_swap_up == TRUE) {
1917 queue_enter(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
1918 c_seg->c_on_swapout_q = 1;
1919 c_swapout_count++;
1920 } else {
1921 queue_enter(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
1922 c_seg->c_on_swappedout_q = 1;
1923 c_swappedout_count++;
1924 }
1925 C_SEG_WAKEUP_DONE(c_seg);
1926
1927 lck_mtx_unlock_always(&c_seg->c_lock);
1928
1929 if (c_swapout_count) {
1930 lck_mtx_unlock_always(c_list_lock);
1931
1932 thread_wakeup((event_t)&c_swapout_list_head);
1933
1934 lck_mtx_lock_spin_always(c_list_lock);
1935 }
1936 }
1937 }
1938
1939
1940 static c_segment_t
1941 c_seg_allocate(c_segment_t *current_chead)
1942 {
1943 clock_sec_t sec;
1944 clock_nsec_t nsec;
1945 c_segment_t c_seg;
1946 int slotarray;
1947
1948 if ( (c_seg = *current_chead) == NULL ) {
1949 uint32_t c_segno;
1950
1951 KERNEL_DEBUG(0xe0400004 | DBG_FUNC_START, 0, 0, 0, 0, 0);
1952
1953 lck_mtx_lock_spin_always(c_list_lock);
1954
1955 while (c_segments_busy == TRUE) {
1956 assert_wait((event_t) (&c_segments_busy), THREAD_UNINT);
1957
1958 lck_mtx_unlock_always(c_list_lock);
1959
1960 thread_block(THREAD_CONTINUE_NULL);
1961
1962 lck_mtx_lock_spin_always(c_list_lock);
1963 }
1964 if (c_free_segno_head == (uint32_t)-1) {
1965
1966 if (c_segments_available >= c_segments_limit || c_segment_pages_compressed >= c_segment_pages_compressed_limit) {
1967 lck_mtx_unlock_always(c_list_lock);
1968
1969 KERNEL_DEBUG(0xe0400004 | DBG_FUNC_END, 0, 0, 0, 1, 0);
1970 return (NULL);
1971 }
1972 c_segments_busy = TRUE;
1973 lck_mtx_unlock_always(c_list_lock);
1974
1975 kernel_memory_populate(kernel_map, (vm_offset_t)c_segments_next_page, PAGE_SIZE, KMA_KOBJECT);
1976 c_segments_next_page += PAGE_SIZE;
1977
1978 for (c_segno = c_segments_available + 1; c_segno < (c_segments_available + C_SEGMENTS_PER_PAGE); c_segno++)
1979 c_segments[c_segno - 1].c_segno = c_segno;
1980
1981 lck_mtx_lock_spin_always(c_list_lock);
1982
1983 c_segments[c_segno - 1].c_segno = c_free_segno_head;
1984 c_free_segno_head = c_segments_available;
1985 c_segments_available += C_SEGMENTS_PER_PAGE;
1986
1987 c_segments_busy = FALSE;
1988 thread_wakeup((event_t) (&c_segments_busy));
1989 }
1990 c_segno = c_free_segno_head;
1991 c_free_segno_head = c_segments[c_segno].c_segno;
1992
1993 lck_mtx_unlock_always(c_list_lock);
1994
1995 c_seg = (c_segment_t)zalloc(compressor_segment_zone);
1996 bzero((char *)c_seg, sizeof(struct c_segment));
1997
1998 if (kernel_memory_allocate(kernel_map, (vm_offset_t *)(&c_seg->c_store.c_buffer), C_SEG_ALLOCSIZE, 0, KMA_COMPRESSOR | KMA_VAONLY) != KERN_SUCCESS) {
1999 zfree(compressor_segment_zone, c_seg);
2000
2001 lck_mtx_lock_spin_always(c_list_lock);
2002
2003 c_segments[c_segno].c_segno = c_free_segno_head;
2004 c_free_segno_head = c_segno;
2005
2006 lck_mtx_unlock_always(c_list_lock);
2007
2008 KERNEL_DEBUG(0xe0400004 | DBG_FUNC_END, 0, 0, 0, 2, 0);
2009
2010 return (NULL);
2011 }
2012
2013 #if __i386__ || __x86_64__
2014 lck_mtx_init(&c_seg->c_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
2015 #else /* __i386__ || __x86_64__ */
2016 lck_spin_init(&c_seg->c_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
2017 #endif /* __i386__ || __x86_64__ */
2018
2019 kernel_memory_populate(kernel_map, (vm_offset_t)(c_seg->c_store.c_buffer), 3 * PAGE_SIZE, KMA_COMPRESSOR);
2020
2021 c_seg->c_populated_offset = C_SEG_BYTES_TO_OFFSET(3 * PAGE_SIZE);
2022 c_seg->c_firstemptyslot = C_SLOT_MAX;
2023 c_seg->c_mysegno = c_segno;
2024 c_seg->c_filling = 1;
2025
2026 lck_mtx_lock_spin_always(c_list_lock);
2027
2028 c_segment_count++;
2029 c_segments[c_segno].c_seg = c_seg;
2030
2031 c_seg->c_generation_id = c_generation_id++;
2032
2033 queue_enter(&c_age_list_head, c_seg, c_segment_t, c_age_list);
2034 c_seg->c_on_age_q = 1;
2035 c_age_count++;
2036
2037 lck_mtx_unlock_always(c_list_lock);
2038
2039 clock_get_system_nanotime(&sec, &nsec);
2040 c_seg->c_creation_ts = (uint32_t)sec;
2041
2042 *current_chead = c_seg;
2043
2044 KERNEL_DEBUG(0xe0400004 | DBG_FUNC_END, c_seg, 0, 0, 3, 0);
2045 }
2046 slotarray = C_SEG_SLOTARRAY_FROM_INDEX(c_seg, c_seg->c_nextslot);
2047
2048 if (c_seg->c_slots[slotarray] == 0) {
2049 KERNEL_DEBUG(0xe0400008 | DBG_FUNC_START, 0, 0, 0, 0, 0);
2050
2051 c_seg->c_slots[slotarray] = (struct c_slot *)kalloc(sizeof(struct c_slot) * C_SEG_SLOT_ARRAY_SIZE);
2052
2053 KERNEL_DEBUG(0xe0400008 | DBG_FUNC_END, 0, 0, 0, 0, 0);
2054 }
2055
2056 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2057
2058 lck_mtx_lock_spin_always(&c_seg->c_lock);
2059
2060 return (c_seg);
2061 }
2062
2063
2064
2065 static void
2066 c_current_seg_filled(c_segment_t c_seg, c_segment_t *current_chead)
2067 {
2068 uint32_t unused_bytes;
2069 uint32_t offset_to_depopulate;
2070
2071 unused_bytes = trunc_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset - c_seg->c_nextoffset));
2072
2073 if (unused_bytes) {
2074
2075 offset_to_depopulate = C_SEG_BYTES_TO_OFFSET(round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_nextoffset)));
2076
2077 /*
2078 * release the extra physical page(s) at the end of the segment
2079 */
2080 lck_mtx_unlock_always(&c_seg->c_lock);
2081
2082 kernel_memory_depopulate(
2083 kernel_map,
2084 (vm_offset_t) &c_seg->c_store.c_buffer[offset_to_depopulate],
2085 unused_bytes,
2086 KMA_COMPRESSOR);
2087
2088 lck_mtx_lock_spin_always(&c_seg->c_lock);
2089
2090 c_seg->c_populated_offset = offset_to_depopulate;
2091 }
2092 c_seg->c_filling = 0;
2093
2094 if (C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE)
2095 c_seg_need_delayed_compaction(c_seg);
2096
2097 lck_mtx_unlock_always(&c_seg->c_lock);
2098
2099 *current_chead = NULL;
2100 }
2101
2102
2103 /*
2104 * returns with c_seg locked
2105 */
2106 void
2107 c_seg_swapin_requeue(c_segment_t c_seg)
2108 {
2109 clock_sec_t sec;
2110 clock_nsec_t nsec;
2111
2112 clock_get_system_nanotime(&sec, &nsec);
2113
2114 lck_mtx_lock_spin_always(c_list_lock);
2115 lck_mtx_lock_spin_always(&c_seg->c_lock);
2116
2117 if (c_seg->c_on_swappedout_q) {
2118 queue_remove(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
2119 c_seg->c_on_swappedout_q = 0;
2120 c_swappedout_count--;
2121 } else {
2122 assert(c_seg->c_on_swappedout_sparse_q);
2123
2124 queue_remove(&c_swappedout_sparse_list_head, c_seg, c_segment_t, c_age_list);
2125 c_seg->c_on_swappedout_sparse_q = 0;
2126 c_swappedout_sparse_count--;
2127 }
2128 if (c_seg->c_store.c_buffer) {
2129 queue_enter(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
2130 c_seg->c_on_swappedin_q = 1;
2131 c_swappedin_count++;
2132 }
2133 #if TRACK_BAD_C_SEGMENTS
2134 else {
2135 queue_enter(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
2136 c_seg->c_on_bad_q = 1;
2137 c_bad_count++;
2138 }
2139 #endif
2140 c_seg->c_swappedin_ts = (uint32_t)sec;
2141 c_seg->c_ondisk = 0;
2142 c_seg->c_was_swapped_in = 1;
2143
2144 lck_mtx_unlock_always(c_list_lock);
2145 }
2146
2147
2148
2149 /*
2150 * c_seg has to be locked and is returned locked.
2151 * PAGE_REPLACMENT_DISALLOWED has to be TRUE on entry and is returned TRUE
2152 */
2153
2154 void
2155 c_seg_swapin(c_segment_t c_seg, boolean_t force_minor_compaction)
2156 {
2157 vm_offset_t addr = 0;
2158 uint32_t io_size = 0;
2159 uint64_t f_offset;
2160
2161 #if !CHECKSUM_THE_SWAP
2162 if (c_seg->c_ondisk)
2163 c_seg_trim_tail(c_seg);
2164 #endif
2165 io_size = round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset));
2166 f_offset = c_seg->c_store.c_swap_handle;
2167
2168 c_seg->c_busy = 1;
2169 lck_mtx_unlock_always(&c_seg->c_lock);
2170
2171 if (c_seg->c_ondisk) {
2172
2173 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2174
2175 if (kernel_memory_allocate(kernel_map, &addr, C_SEG_ALLOCSIZE, 0, KMA_COMPRESSOR | KMA_VAONLY) != KERN_SUCCESS)
2176 panic("c_seg_swapin: kernel_memory_allocate failed\n");
2177
2178 kernel_memory_populate(kernel_map, addr, io_size, KMA_COMPRESSOR);
2179
2180 if (vm_swap_get(addr, f_offset, io_size) != KERN_SUCCESS) {
2181 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2182
2183 kernel_memory_depopulate(kernel_map, addr, io_size, KMA_COMPRESSOR);
2184 kmem_free(kernel_map, addr, C_SEG_ALLOCSIZE);
2185
2186 c_seg->c_store.c_buffer = (int32_t*) NULL;
2187 } else {
2188 c_seg->c_store.c_buffer = (int32_t*) addr;
2189 #if CRYPTO
2190 vm_swap_decrypt(c_seg);
2191 #endif /* CRYPTO */
2192
2193 #if CHECKSUM_THE_SWAP
2194 if (c_seg->cseg_swap_size != io_size)
2195 panic("swapin size doesn't match swapout size");
2196
2197 if (c_seg->cseg_hash != hash_string((char*) c_seg->c_store.c_buffer, (int)io_size)) {
2198 panic("c_seg_swapin - Swap hash mismatch\n");
2199 }
2200 #endif /* CHECKSUM_THE_SWAP */
2201
2202 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2203
2204 if (force_minor_compaction == TRUE) {
2205 lck_mtx_lock_spin_always(&c_seg->c_lock);
2206
2207 c_seg_minor_compaction_and_unlock(c_seg, FALSE);
2208 }
2209 OSAddAtomic64(c_seg->c_bytes_used, &compressor_bytes_used);
2210 }
2211 }
2212 c_seg_swapin_requeue(c_seg);
2213
2214 C_SEG_WAKEUP_DONE(c_seg);
2215 }
2216
2217
2218 static int
2219 c_compress_page(char *src, c_slot_mapping_t slot_ptr, c_segment_t *current_chead, char *scratch_buf)
2220 {
2221 int c_size;
2222 int c_rounded_size;
2223 int max_csize;
2224 c_slot_t cs;
2225 c_segment_t c_seg;
2226
2227 KERNEL_DEBUG(0xe0400000 | DBG_FUNC_START, *current_chead, 0, 0, 0, 0);
2228 retry:
2229 if ((c_seg = c_seg_allocate(current_chead)) == NULL)
2230 return (1);
2231 /*
2232 * returns with c_seg lock held
2233 * and PAGE_REPLACEMENT_DISALLOWED(TRUE)
2234 */
2235 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_seg->c_nextslot);
2236
2237 cs->c_packed_ptr = C_SLOT_PACK_PTR(slot_ptr);
2238 cs->c_offset = c_seg->c_nextoffset;
2239
2240 max_csize = C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES((int32_t)cs->c_offset);
2241
2242 if (max_csize > PAGE_SIZE)
2243 max_csize = PAGE_SIZE;
2244
2245 if (C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset -
2246 c_seg->c_nextoffset)
2247 < (unsigned) max_csize + PAGE_SIZE &&
2248 (C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset)
2249 < C_SEG_ALLOCSIZE)) {
2250 lck_mtx_unlock_always(&c_seg->c_lock);
2251
2252 kernel_memory_populate(kernel_map,
2253 (vm_offset_t) &c_seg->c_store.c_buffer[c_seg->c_populated_offset],
2254 PAGE_SIZE,
2255 KMA_COMPRESSOR);
2256
2257 lck_mtx_lock_spin_always(&c_seg->c_lock);
2258
2259 c_seg->c_populated_offset += C_SEG_BYTES_TO_OFFSET(PAGE_SIZE);
2260 }
2261
2262 #if CHECKSUM_THE_DATA
2263 cs->c_hash_data = hash_string(src, PAGE_SIZE);
2264 #endif
2265 c_size = WKdm_compress_new((WK_word *)(uintptr_t)src, (WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
2266 (WK_word *)(uintptr_t)scratch_buf, max_csize - 4);
2267
2268 assert(c_size <= (max_csize - 4) && c_size >= -1);
2269
2270 if (c_size == -1) {
2271
2272 if (max_csize < PAGE_SIZE) {
2273 c_current_seg_filled(c_seg, current_chead);
2274
2275 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2276
2277 goto retry;
2278 }
2279 c_size = PAGE_SIZE;
2280
2281 memcpy(&c_seg->c_store.c_buffer[cs->c_offset], src, c_size);
2282 }
2283 #if CHECKSUM_THE_COMPRESSED_DATA
2284 cs->c_hash_compressed_data = hash_string((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size);
2285 #endif
2286 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
2287
2288 PACK_C_SIZE(cs, c_size);
2289 c_seg->c_bytes_used += c_rounded_size;
2290 c_seg->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
2291
2292 slot_ptr->s_cindx = c_seg->c_nextslot++;
2293 /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
2294 slot_ptr->s_cseg = c_seg->c_mysegno + 1;
2295
2296 if (c_seg->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg->c_nextslot >= C_SLOT_MAX)
2297 c_current_seg_filled(c_seg, current_chead);
2298 else
2299 lck_mtx_unlock_always(&c_seg->c_lock);
2300
2301 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2302
2303 OSAddAtomic64(c_rounded_size, &compressor_bytes_used);
2304 OSAddAtomic64(PAGE_SIZE, &c_segment_input_bytes);
2305 OSAddAtomic64(c_size, &c_segment_compressed_bytes);
2306
2307 OSAddAtomic(1, &c_segment_pages_compressed);
2308 OSAddAtomic(1, &sample_period_compression_count);
2309
2310 KERNEL_DEBUG(0xe0400000 | DBG_FUNC_END, *current_chead, c_size, c_segment_input_bytes, c_segment_compressed_bytes, 0);
2311
2312 if (vm_compressor_low_on_space()) {
2313 ipc_port_t trigger = IP_NULL;
2314
2315 PSL_LOCK();
2316 if (IP_VALID(min_pages_trigger_port)) {
2317 trigger = min_pages_trigger_port;
2318 min_pages_trigger_port = IP_NULL;
2319 }
2320 PSL_UNLOCK();
2321
2322 if (IP_VALID(trigger)) {
2323 no_paging_space_action();
2324 default_pager_space_alert(trigger, HI_WAT_ALERT);
2325 ipc_port_release_send(trigger);
2326 }
2327 }
2328 return (0);
2329 }
2330
2331
2332 static int
2333 c_decompress_page(char *dst, volatile c_slot_mapping_t slot_ptr, int flags, int *zeroslot)
2334 {
2335 c_slot_t cs;
2336 c_segment_t c_seg;
2337 int c_indx;
2338 int c_rounded_size;
2339 uint32_t c_size;
2340 int retval = 0;
2341 boolean_t c_seg_has_data = TRUE;
2342 boolean_t c_seg_swappedin = FALSE;
2343 boolean_t need_unlock = TRUE;
2344 boolean_t consider_defragmenting = FALSE;
2345
2346 ReTry:
2347 #if HIBERNATION
2348 if (dst) {
2349 if (lck_rw_try_lock_shared(&c_decompressor_lock) == 0) {
2350 if (flags & C_DONT_BLOCK) {
2351 *zeroslot = 0;
2352 return (-2);
2353 }
2354 lck_rw_lock_shared(&c_decompressor_lock);
2355 }
2356 }
2357 #endif
2358 PAGE_REPLACEMENT_DISALLOWED(TRUE);
2359
2360 /* s_cseg is actually "segno+1" */
2361 c_seg = c_segments[slot_ptr->s_cseg - 1].c_seg;
2362
2363 lck_mtx_lock_spin_always(&c_seg->c_lock);
2364
2365 if (flags & C_DONT_BLOCK) {
2366 if (c_seg->c_busy || c_seg->c_ondisk) {
2367
2368 retval = -2;
2369 *zeroslot = 0;
2370
2371 goto done;
2372 }
2373 }
2374 if (c_seg->c_busy) {
2375
2376 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2377 #if HIBERNATION
2378 if (dst)
2379 lck_rw_done(&c_decompressor_lock);
2380 #endif
2381 c_seg_wait_on_busy(c_seg);
2382
2383 goto ReTry;
2384 }
2385 c_indx = slot_ptr->s_cindx;
2386
2387 cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
2388
2389 c_size = UNPACK_C_SIZE(cs);
2390
2391 c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
2392
2393 if (dst) {
2394 uint32_t age_of_cseg;
2395 clock_sec_t cur_ts_sec;
2396 clock_nsec_t cur_ts_nsec;
2397
2398 if (c_seg->c_on_swappedout_q || c_seg->c_on_swappedout_sparse_q) {
2399 if (c_seg->c_ondisk)
2400 c_seg_swappedin = TRUE;
2401 c_seg_swapin(c_seg, FALSE);
2402 }
2403 if (c_seg->c_store.c_buffer == NULL) {
2404 c_seg_has_data = FALSE;
2405 goto c_seg_invalid_data;
2406 }
2407 #if CHECKSUM_THE_COMPRESSED_DATA
2408 if (cs->c_hash_compressed_data != hash_string((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size))
2409 panic("compressed data doesn't match original");
2410 #endif
2411 if (c_rounded_size == PAGE_SIZE) {
2412 /*
2413 * page wasn't compressible... just copy it out
2414 */
2415 memcpy(dst, &c_seg->c_store.c_buffer[cs->c_offset], PAGE_SIZE);
2416 } else {
2417 uint32_t my_cpu_no;
2418 char *scratch_buf;
2419
2420 /*
2421 * we're behind the c_seg lock held in spin mode
2422 * which means pre-emption is disabled... therefore
2423 * the following sequence is atomic and safe
2424 */
2425 my_cpu_no = cpu_number();
2426
2427 assert(my_cpu_no < compressor_cpus);
2428
2429 scratch_buf = &compressor_scratch_bufs[my_cpu_no * WKdm_SCRATCH_BUF_SIZE];
2430
2431 WKdm_decompress_new((WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
2432 (WK_word *)(uintptr_t)dst, (WK_word *)(uintptr_t)scratch_buf, c_size);
2433 }
2434
2435 #if CHECKSUM_THE_DATA
2436 if (cs->c_hash_data != hash_string(dst, PAGE_SIZE))
2437 panic("decompressed data doesn't match original");
2438 #endif
2439 if (!c_seg->c_was_swapped_in) {
2440
2441 clock_get_system_nanotime(&cur_ts_sec, &cur_ts_nsec);
2442
2443 age_of_cseg = (uint32_t)cur_ts_sec - c_seg->c_creation_ts;
2444
2445 if (age_of_cseg < DECOMPRESSION_SAMPLE_MAX_AGE)
2446 OSAddAtomic(1, &age_of_decompressions_during_sample_period[age_of_cseg]);
2447 else
2448 OSAddAtomic(1, &overage_decompressions_during_sample_period);
2449
2450 OSAddAtomic(1, &sample_period_decompression_count);
2451 }
2452 } else {
2453 if (c_seg->c_store.c_buffer == NULL)
2454 c_seg_has_data = FALSE;
2455 }
2456 c_seg_invalid_data:
2457
2458 if (c_seg_has_data == TRUE) {
2459 if (c_seg_swappedin == TRUE)
2460 retval = 1;
2461 else
2462 retval = 0;
2463 } else
2464 retval = -1;
2465
2466 if (flags & C_KEEP) {
2467 *zeroslot = 0;
2468 goto done;
2469 }
2470 c_seg->c_bytes_unused += c_rounded_size;
2471 c_seg->c_bytes_used -= c_rounded_size;
2472 PACK_C_SIZE(cs, 0);
2473
2474 if (c_indx < c_seg->c_firstemptyslot)
2475 c_seg->c_firstemptyslot = c_indx;
2476
2477 OSAddAtomic(-1, &c_segment_pages_compressed);
2478
2479 if (c_seg_has_data == TRUE && !c_seg->c_ondisk) {
2480 /*
2481 * c_ondisk == TRUE can occur when we're doing a
2482 * free of a compressed page (i.e. dst == NULL)
2483 */
2484 OSAddAtomic64(-c_rounded_size, &compressor_bytes_used);
2485 }
2486 if (!c_seg->c_filling) {
2487 if (c_seg->c_bytes_used == 0) {
2488 if (c_seg->c_on_minorcompact_q || c_seg->c_on_swappedout_sparse_q) {
2489 if (c_seg_try_free(c_seg) == TRUE)
2490 need_unlock = FALSE;
2491 } else {
2492 c_seg_free(c_seg);
2493 need_unlock = FALSE;
2494 }
2495 } else if (c_seg->c_on_minorcompact_q) {
2496
2497 if (C_SEG_INCORE_IS_SPARSE(c_seg)) {
2498 c_seg_try_minor_compaction_and_unlock(c_seg);
2499 need_unlock = FALSE;
2500 }
2501 } else if (!c_seg->c_ondisk) {
2502
2503 if (c_seg_has_data == TRUE && !c_seg->c_on_swapout_q && C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE) {
2504 c_seg_need_delayed_compaction(c_seg);
2505 }
2506 } else if (!c_seg->c_on_swappedout_sparse_q && C_SEG_ONDISK_IS_SPARSE(c_seg)) {
2507
2508 c_seg_move_to_sparse_list(c_seg);
2509 consider_defragmenting = TRUE;
2510 }
2511 }
2512 done:
2513 if (need_unlock == TRUE)
2514 lck_mtx_unlock_always(&c_seg->c_lock);
2515
2516 PAGE_REPLACEMENT_DISALLOWED(FALSE);
2517
2518 if (consider_defragmenting == TRUE)
2519 vm_swap_consider_defragmenting();
2520 #if HIBERNATION
2521 if (dst)
2522 lck_rw_done(&c_decompressor_lock);
2523 #endif
2524 return (retval);
2525 }
2526
2527
2528 int
2529 vm_compressor_get(ppnum_t pn, int *slot, int flags)
2530 {
2531 char *dst;
2532 int zeroslot = 1;
2533 int retval;
2534
2535 #if __x86_64__
2536 dst = PHYSMAP_PTOV((uint64_t)pn << (uint64_t)PAGE_SHIFT);
2537 #else
2538 #error "unsupported architecture"
2539 #endif
2540
2541 retval = c_decompress_page(dst, (c_slot_mapping_t)slot, flags, &zeroslot);
2542
2543 /*
2544 * zeroslot will be set to 0 by c_decompress_page if (flags & C_KEEP)
2545 * or (flags & C_DONT_BLOCK) and we found 'c_busy' or 'c_ondisk' set
2546 */
2547 if (zeroslot) {
2548 /*
2549 * We've just decompressed a page, and are about to hand that back to VM for
2550 * re-entry into some pmap. This is a decompression operation which must have no
2551 * impact on the pmap's physical footprint. However, when VM goes to re-enter
2552 * this page into the pmap, it doesn't know that it came from the compressor,
2553 * which means the pmap's physical footprint will be incremented. To compensate
2554 * for that, we decrement the physical footprint here, so that the total net effect
2555 * on the physical footprint statistic is zero.
2556 */
2557 pmap_ledger_debit(current_task()->map->pmap, task_ledgers.phys_footprint, PAGE_SIZE);
2558
2559 *slot = 0;
2560 }
2561 /*
2562 * returns 0 if we successfully decompressed a page from a segment already in memory
2563 * returns 1 if we had to first swap in the segment, before successfully decompressing the page
2564 * returns -1 if we encountered an error swapping in the segment - decompression failed
2565 * returns -2 if (flags & C_DONT_BLOCK) and we found 'c_busy' or 'c_ondisk' set
2566 */
2567 return (retval);
2568 }
2569
2570
2571 void
2572 vm_compressor_free(int *slot)
2573 {
2574 int zeroslot = 1;
2575
2576 (void)c_decompress_page(NULL, (c_slot_mapping_t)slot, 0, &zeroslot);
2577
2578 *slot = 0;
2579 }
2580
2581
2582 int
2583 vm_compressor_put(ppnum_t pn, int *slot, void **current_chead, char *scratch_buf)
2584 {
2585 char *src;
2586 int retval;
2587
2588 if ((vm_offset_t)slot < VM_MIN_KERNEL_AND_KEXT_ADDRESS || (vm_offset_t)slot >= VM_MAX_KERNEL_ADDRESS)
2589 panic("vm_compressor_put: slot 0x%llx address out of range [0x%llx:0x%llx]",
2590 (uint64_t)(vm_offset_t) slot,
2591 (uint64_t) VM_MIN_KERNEL_AND_KEXT_ADDRESS,
2592 (uint64_t) VM_MAX_KERNEL_ADDRESS);
2593
2594 #if __x86_64__
2595 src = PHYSMAP_PTOV((uint64_t)pn << (uint64_t)PAGE_SHIFT);
2596 #else
2597 #error "unsupported architecture"
2598 #endif
2599 retval = c_compress_page(src, (c_slot_mapping_t)slot, (c_segment_t *)current_chead, scratch_buf);
2600
2601 return (retval);
2602 }
mirror of XNU