]> git.saurik.com Git - apple/xnu.git/blame - osfmk/vm/vm_compressor_backing_store.c
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-2422.115.4.tar.gz
[apple/xnu.git] / osfmk / vm / vm_compressor_backing_store.c
CommitLineData
39236c6e
A		 1/*
2 * Copyright (c) 2000-2013 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include "vm_compressor_backing_store.h"
30#include <vm/vm_protos.h>
31
32#include <IOKit/IOHibernatePrivate.h>
33
34
35boolean_t compressor_store_stop_compaction = FALSE;
36boolean_t vm_swap_up = FALSE;
37boolean_t vm_swapfile_mgmt_needed = FALSE;
38
39int swapper_throttle = -1;
40boolean_t swapper_throttle_inited = FALSE;
41uint64_t vm_swapout_thread_id;
42
43uint64_t vm_swap_put_failures = 0;
44uint64_t vm_swap_get_failures = 0;
45int vm_num_swap_files = 0;
46int vm_swapout_thread_processed_segments = 0;
47int vm_swapout_thread_awakened = 0;
48int vm_swapfile_mgmt_thread_awakened = 0;
49int vm_swapfile_mgmt_thread_running = 0;
50
51unsigned int vm_swapfile_total_segs_alloced = 0;
52unsigned int vm_swapfile_total_segs_used = 0;
53
54
55#define SWAP_READY 0x1 /* Swap file is ready to be used */
56#define SWAP_RECLAIM 0x2 /* Swap file is marked to be reclaimed */
57#define SWAP_WANTED 0x4 /* Swap file has waiters */
58#define SWAP_REUSE 0x8 /* Swap file is on the Q and has a name. Reuse after init-ing.*/
59
60struct swapfile{
61 queue_head_t swp_queue; /* list of swap files */
62 char *swp_path; /* saved pathname of swap file */
63 struct vnode *swp_vp; /* backing vnode */
64 uint64_t swp_size; /* size of this swap file */
65 uint8_t *swp_bitmap; /* bitmap showing the alloced/freed slots in the swap file */
66 unsigned int swp_pathlen; /* length of pathname */
67 unsigned int swp_nsegs; /* #segments we can use */
68 unsigned int swp_nseginuse; /* #segments in use */
69 unsigned int swp_index; /* index of this swap file */
70 unsigned int swp_flags; /* state of swap file */
71 unsigned int swp_free_hint; /* offset of 1st free chunk */
72 unsigned int swp_io_count; /* count of outstanding I/Os */
73 c_segment_t *swp_csegs; /* back pointers to the c_segments. Used during swap reclaim. */
74
75 struct trim_list *swp_delayed_trim_list_head;
76 unsigned int swp_delayed_trim_count;
77 boolean_t swp_trim_supported;
78};
79
80queue_head_t swf_global_queue;
81
82#define VM_SWAPFILE_DELAYED_TRIM_MAX 128
83
84extern clock_sec_t dont_trim_until_ts;
85clock_sec_t vm_swapfile_last_failed_to_create_ts = 0;
86
87static void vm_swapout_thread_throttle_adjust(void);
88static void vm_swap_free_now(struct swapfile *swf, uint64_t f_offset);
89static void vm_swapout_thread(void);
90static void vm_swapfile_mgmt_thread(void);
91static void vm_swap_defragment();
92static void vm_swap_handle_delayed_trims(boolean_t);
93static void vm_swap_do_delayed_trim();
94
95
96#define VM_SWAPFILE_DELAYED_CREATE 30
97#define VM_SWAP_SHOULD_DEFRAGMENT() (c_swappedout_sparse_count > (vm_swapfile_total_segs_used / 4) ? 1 : 0)
98#define VM_SWAP_SHOULD_RECLAIM() (((vm_swapfile_total_segs_alloced - vm_swapfile_total_segs_used) >= SWAPFILE_RECLAIM_THRESHOLD_SEGS) ? 1 : 0)
99#define VM_SWAP_SHOULD_CREATE(cur_ts) (((vm_swapfile_total_segs_alloced - vm_swapfile_total_segs_used) < (unsigned int)VM_SWAPFILE_HIWATER_SEGS) && \
100 ((cur_ts - vm_swapfile_last_failed_to_create_ts) > VM_SWAPFILE_DELAYED_CREATE) ? 1 : 0)
101#define VM_SWAP_SHOULD_TRIM(swf) ((swf->swp_delayed_trim_count >= VM_SWAPFILE_DELAYED_TRIM_MAX) ? 1 : 0)
102
103
104#define VM_SWAP_BUSY() ((c_swapout_count && (swapper_throttle == THROTTLE_LEVEL_COMPRESSOR_TIER1 || swapper_throttle == THROTTLE_LEVEL_COMPRESSOR_TIER0)) ? 1 : 0)
105
106
107#if CHECKSUM_THE_SWAP
108extern unsigned int hash_string(char *cp, int len);
109#endif
110
111#if CRYPTO
112extern boolean_t swap_crypt_ctx_initialized;
113extern void swap_crypt_ctx_initialize(void);
114extern const unsigned char swap_crypt_null_iv[AES_BLOCK_SIZE];
115extern aes_ctx swap_crypt_ctx;
116extern unsigned long vm_page_encrypt_counter;
117extern unsigned long vm_page_decrypt_counter;
118#endif /* CRYPTO */
119
120extern void vm_pageout_io_throttle(void);
121
122struct swapfile *vm_swapfile_for_handle(uint64_t);
123
124/*
125 * Called with the vm_swap_data_lock held.
126 */
127
128struct swapfile *
129vm_swapfile_for_handle(uint64_t f_offset)
130{
131
132 uint64_t file_offset = 0;
133 unsigned int swapfile_index = 0;
134 struct swapfile* swf = NULL;
135
136 file_offset = (f_offset & SWAP_SLOT_MASK);
137 swapfile_index = (f_offset >> SWAP_DEVICE_SHIFT);
138
139 swf = (struct swapfile*) queue_first(&swf_global_queue);
140
141 while(queue_end(&swf_global_queue, (queue_entry_t)swf) == FALSE) {
142
143 if (swapfile_index == swf->swp_index) {
144 break;
145 }
146
147 swf = (struct swapfile*) queue_next(&swf->swp_queue);
148 }
149
150 if (queue_end(&swf_global_queue, (queue_entry_t) swf)) {
151 swf = NULL;
152 }
153
154 return swf;
155}
156
157void
158vm_swap_init()
159{
160 static boolean_t vm_swap_try_init = FALSE;
161 thread_t thread = NULL;
162
163 if (vm_swap_try_init == TRUE) {
164 return;
165 }
166
167 vm_swap_try_init = TRUE;
168
169 lck_grp_attr_setdefault(&vm_swap_data_lock_grp_attr);
170 lck_grp_init(&vm_swap_data_lock_grp,
171 "vm_swap_data",
172 &vm_swap_data_lock_grp_attr);
173 lck_attr_setdefault(&vm_swap_data_lock_attr);
174 lck_mtx_init_ext(&vm_swap_data_lock,
175 &vm_swap_data_lock_ext,
176 &vm_swap_data_lock_grp,
177 &vm_swap_data_lock_attr);
178
179 queue_init(&swf_global_queue);
180
181 if (vm_swap_create_file()) {
182
183 if (kernel_thread_start_priority((thread_continue_t)vm_swapout_thread, NULL,
184 BASEPRI_PREEMPT - 1, &thread) != KERN_SUCCESS) {
185 panic("vm_swapout_thread: create failed");
186 }
187 thread->options |= TH_OPT_VMPRIV;
188 vm_swapout_thread_id = thread->thread_id;
189
190 thread_deallocate(thread);
191
192 if (kernel_thread_start_priority((thread_continue_t)vm_swapfile_mgmt_thread, NULL,
193 BASEPRI_PREEMPT - 1, &thread) != KERN_SUCCESS) {
194 panic("vm_swapfile_mgmt_thread: create failed");
195 }
196 thread->options |= TH_OPT_VMPRIV;
197
198 thread_deallocate(thread);
199
200#if CRYPTO
201 if (swap_crypt_ctx_initialized == FALSE) {
202 swap_crypt_ctx_initialize();
203 }
204#endif /* CRYPTO */
205
206 vm_swap_up = TRUE;
207
208#if SANITY_CHECK_SWAP_ROUTINES
209extern lck_attr_t *vm_compressor_lck_attr;
210extern lck_grp_t *vm_compressor_lck_grp;
211
212 /*
213 * Changes COMPRESSED_SWAP_CHUNK_SIZE to make it (4*KB).
214 * Changes MIN_SWAP_FILE_SIZE to (4*KB).
215 * Changes MAX_SWAP_FILE_SIZE to (4*KB).
216 * That will then cause the below allocations to create
217 * 4 new swap files and put/get/free from them.
218 */
219 {
220 c_segment_t c_seg = NULL, c_seg1 = NULL, c_seg2 = NULL, c_seg3 = NULL;
221 vm_offset_t addr = 0;
222 vm_offset_t dup_addr = 0;
223 kern_return_t kr = KERN_SUCCESS;
224 uint64_t f_offset = 0;
225 uint64_t f_offset1 = 0;
226 uint64_t f_offset2 = 0;
227 uint64_t f_offset3 = 0;
228
229 if ((kr = kernel_memory_allocate(kernel_map,
230 &addr,
231 4 * COMPRESSED_SWAP_CHUNK_SIZE,
232 0,
233 KMA_KOBJECT))) {
234 printf("kernel_memory_allocate failed with %d\n", kr);
235 goto done;
236 }
237
238 if ((kr = kernel_memory_allocate(kernel_map,
239 &dup_addr,
240 4 * COMPRESSED_SWAP_CHUNK_SIZE,
241 0,
242 KMA_KOBJECT))) {
243 printf("kernel_memory_allocate failed with %d\n", kr);
244 goto done;
245 }
246
247 c_seg = (c_segment_t) kalloc(sizeof(*c_seg));
248 memset(c_seg, 0, sizeof(*c_seg));
249#if __i386__ || __x86_64__
250 lck_mtx_init(&c_seg->c_lock, vm_compressor_lck_grp, vm_compressor_lck_attr);
251#else /* __i386__ || __x86_64__ */
252 lck_spin_init(&c_seg->c_lock, vm_compressor_lck_grp, vm_compressor_lck_attr);
253#endif /* __i386__ || __x86_64__ */
254
255
256 c_seg1 = (c_segment_t) kalloc(sizeof(*c_seg));
257 memset(c_seg1, 0, sizeof(*c_seg));
258#if __i386__ || __x86_64__
259 lck_mtx_init(&c_seg1->c_lock, vm_compressor_lck_grp, vm_compressor_lck_attr);
260#else /* __i386__ || __x86_64__ */
261 lck_spin_init(&c_seg1->c_lock, vm_compressor_lck_grp, vm_compressor_lck_attr);
262#endif /* __i386__ || __x86_64__ */
263
264
265 c_seg2 = (c_segment_t) kalloc(sizeof(*c_seg));
266 memset(c_seg2, 0, sizeof(*c_seg));
267#if __i386__ || __x86_64__
268 lck_mtx_init(&c_seg2->c_lock, vm_compressor_lck_grp, vm_compressor_lck_attr);
269#else /* __i386__ || __x86_64__ */
270 lck_spin_init(&c_seg2->c_lock, vm_compressor_lck_grp, vm_compressor_lck_attr);
271#endif /* __i386__ || __x86_64__ */
272
273
274 c_seg3 = (c_segment_t) kalloc(sizeof(*c_seg));
275 memset(c_seg3, 0, sizeof(*c_seg));
276#if __i386__ || __x86_64__
277 lck_mtx_init(&c_seg3->c_lock, vm_compressor_lck_grp, vm_compressor_lck_attr);
278#else /* __i386__ || __x86_64__ */
279 lck_spin_init(&c_seg3->c_lock, vm_compressor_lck_grp, vm_compressor_lck_attr);
280#endif /* __i386__ || __x86_64__ */
281
282
283 memset((void*)addr, (int) 'a', PAGE_SIZE_64);
284 memset((void*)(addr + PAGE_SIZE_64), (int) 'b', PAGE_SIZE_64);
285 memset((void*)(addr + (2 * PAGE_SIZE_64)), (int) 'c', PAGE_SIZE_64);
286 memset((void*)(addr + (3 * PAGE_SIZE_64)), (int) 'd', PAGE_SIZE_64);
287
288 vm_swap_put(addr, &f_offset, PAGE_SIZE_64, c_seg);
289 c_seg->c_store.c_swap_handle = f_offset;
290
291 vm_swap_put(addr + PAGE_SIZE_64, &f_offset1, PAGE_SIZE_64, c_seg1);
292 c_seg1->c_store.c_swap_handle = f_offset1;
293
294 vm_swap_put(addr + (2 * PAGE_SIZE_64), &f_offset2, PAGE_SIZE_64, c_seg2);
295 c_seg2->c_store.c_swap_handle = f_offset2;
296
297 vm_swap_put(addr + (3 * PAGE_SIZE_64), &f_offset3, PAGE_SIZE_64, c_seg3);
298 c_seg3->c_store.c_swap_handle = f_offset3;
299
300 //vm_swap_free(f_offset);
301 vm_swap_get(dup_addr, f_offset, PAGE_SIZE_64);
302
303 //vm_swap_free(f_offset1);
304 vm_swap_reclaim();
305 vm_swap_get(dup_addr + PAGE_SIZE_64, c_seg1->c_store.c_swap_handle, PAGE_SIZE_64);
306
307 //vm_swap_free(f_offset2);
308 vm_swap_reclaim();
309 vm_swap_get(dup_addr + (2 * PAGE_SIZE_64), c_seg2->c_store.c_swap_handle, PAGE_SIZE_64);
310
311 //vm_swap_free(f_offset3);
312 vm_swap_reclaim();
313 vm_swap_get(dup_addr + (3 * PAGE_SIZE_64), c_seg3->c_store.c_swap_handle, PAGE_SIZE_64);
314
315 if (memcmp((void*)addr, (void*)dup_addr, PAGE_SIZE_64)) {
316 panic("First page data mismatch\n");
317 kr = KERN_FAILURE;
318 goto done;
319 }
320
321 if (memcmp((void*)(addr + PAGE_SIZE_64), (void*)(dup_addr + PAGE_SIZE_64), PAGE_SIZE_64)) {
322 panic("Second page data mismatch 0x%lx, 0x%lxn", addr, dup_addr);
323 kr = KERN_FAILURE;
324 goto done;
325 }
326
327 if (memcmp((void*)(addr + (2 * PAGE_SIZE_64)), (void*)(dup_addr + (2 * PAGE_SIZE_64)), PAGE_SIZE_64)) {
328 panic("Third page data mismatch\n");
329 kr = KERN_FAILURE;
330 goto done;
331 }
332
333 if (memcmp((void*)(addr + (3 * PAGE_SIZE_64)), (void*)(dup_addr + (3 * PAGE_SIZE_64)), PAGE_SIZE_64)) {
334 panic("Fourth page data mismatch 0x%lx, 0x%lxn", addr, dup_addr);
335 kr = KERN_FAILURE;
336 goto done;
337 }
338
339done:
340 printf("Sanity check %s\n", ((kr != KERN_SUCCESS) ? "FAILED" : "SUCCEEDED"));
341 kfree((void*)addr, 4 * COMPRESSED_SWAP_CHUNK_SIZE);
342 addr = 0;
343 kfree((void*)dup_addr, 4 * COMPRESSED_SWAP_CHUNK_SIZE);
344 dup_addr = 0;
345 }
346#endif /* SANITY_CHECK_SWAP_ROUTINES */
347 }
348
349 printf("VM Swap Subsystem is %s\n", (vm_swap_up == TRUE) ? "ON" : "OFF");
350}
351
352#if CRYPTO
353void
354vm_swap_encrypt(c_segment_t c_seg)
355{
356 vm_offset_t kernel_vaddr = 0;
357 uint64_t size = 0;
358
359 union {
360 unsigned char aes_iv[AES_BLOCK_SIZE];
361 void *c_seg;
362 } encrypt_iv;
363
364 assert(swap_crypt_ctx_initialized);
365
366 bzero(&encrypt_iv.aes_iv[0], sizeof (encrypt_iv.aes_iv));
367
368 encrypt_iv.c_seg = (void*)c_seg;
369
370 /* encrypt the "initial vector" */
371 aes_encrypt_cbc((const unsigned char *) &encrypt_iv.aes_iv[0],
372 swap_crypt_null_iv,
373 1,
374 &encrypt_iv.aes_iv[0],
375 &swap_crypt_ctx.encrypt);
376
377 kernel_vaddr = (vm_offset_t) c_seg->c_store.c_buffer;
378 size = round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset));
379
380 /*
381 * Encrypt the c_segment.
382 */
383 aes_encrypt_cbc((const unsigned char *) kernel_vaddr,
384 &encrypt_iv.aes_iv[0],
385 (unsigned int)(size / AES_BLOCK_SIZE),
386 (unsigned char *) kernel_vaddr,
387 &swap_crypt_ctx.encrypt);
388
389 vm_page_encrypt_counter += (size/PAGE_SIZE_64);
390}
391
392void
393vm_swap_decrypt(c_segment_t c_seg)
394{
395
396 vm_offset_t kernel_vaddr = 0;
397 uint64_t size = 0;
398
399 union {
400 unsigned char aes_iv[AES_BLOCK_SIZE];
401 void *c_seg;
402 } decrypt_iv;
403
404
405 assert(swap_crypt_ctx_initialized);
406
407 /*
408 * Prepare an "initial vector" for the decryption.
409 * It has to be the same as the "initial vector" we
410 * used to encrypt that page.
411 */
412 bzero(&decrypt_iv.aes_iv[0], sizeof (decrypt_iv.aes_iv));
413
414 decrypt_iv.c_seg = (void*)c_seg;
415
416 /* encrypt the "initial vector" */
417 aes_encrypt_cbc((const unsigned char *) &decrypt_iv.aes_iv[0],
418 swap_crypt_null_iv,
419 1,
420 &decrypt_iv.aes_iv[0],
421 &swap_crypt_ctx.encrypt);
422
423 kernel_vaddr = (vm_offset_t) c_seg->c_store.c_buffer;
424 size = round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset));
425
426 /*
427 * Decrypt the c_segment.
428 */
429 aes_decrypt_cbc((const unsigned char *) kernel_vaddr,
430 &decrypt_iv.aes_iv[0],
431 (unsigned int) (size / AES_BLOCK_SIZE),
432 (unsigned char *) kernel_vaddr,
433 &swap_crypt_ctx.decrypt);
434
435 vm_page_decrypt_counter += (size/PAGE_SIZE_64);
436}
437#endif /* CRYPTO */
438
439
440void
441vm_swap_consider_defragmenting()
442{
443 if (compressor_store_stop_compaction == FALSE && !VM_SWAP_BUSY() && (VM_SWAP_SHOULD_DEFRAGMENT() || VM_SWAP_SHOULD_RECLAIM())) {
444
445 if (!vm_swapfile_mgmt_thread_running) {
446 lck_mtx_lock(&vm_swap_data_lock);
447
448 if (!vm_swapfile_mgmt_thread_running)
449 thread_wakeup((event_t) &vm_swapfile_mgmt_needed);
450
451 lck_mtx_unlock(&vm_swap_data_lock);
452 }
453 }
454}
455
456
457int vm_swap_defragment_yielded = 0;
458int vm_swap_defragment_swapin = 0;
459int vm_swap_defragment_free = 0;
460int vm_swap_defragment_busy = 0;
461
462
463static void
464vm_swap_defragment()
465{
466 c_segment_t c_seg;
467
468 /*
469 * have to grab the master lock w/o holding
470 * any locks in spin mode
471 */
472 PAGE_REPLACEMENT_DISALLOWED(TRUE);
473
474 lck_mtx_lock_spin_always(c_list_lock);
475
476 while (!queue_empty(&c_swappedout_sparse_list_head)) {
477
478 if (compressor_store_stop_compaction == TRUE || VM_SWAP_BUSY()) {
479 vm_swap_defragment_yielded++;
480 break;
481 }
482 c_seg = (c_segment_t)queue_first(&c_swappedout_sparse_list_head);
483
484 lck_mtx_lock_spin_always(&c_seg->c_lock);
485
486 assert(c_seg->c_on_swappedout_sparse_q);
487
488 if (c_seg->c_busy) {
489 lck_mtx_unlock_always(c_list_lock);
490
491 PAGE_REPLACEMENT_DISALLOWED(FALSE);
492 /*
493 * c_seg_wait_on_busy consumes c_seg->c_lock
494 */
495 c_seg_wait_on_busy(c_seg);
496
497 PAGE_REPLACEMENT_DISALLOWED(TRUE);
498
499 lck_mtx_lock_spin_always(c_list_lock);
500
501 vm_swap_defragment_busy++;
502 continue;
503 }
504 if (c_seg->c_bytes_used == 0) {
505 /*
506 * c_seg_free_locked consumes the c_list_lock
507 * and c_seg->c_lock
508 */
509 c_seg_free_locked(c_seg);
510
511 vm_swap_defragment_free++;
512 } else {
513 lck_mtx_unlock_always(c_list_lock);
514
515 c_seg_swapin(c_seg, TRUE);
516 lck_mtx_unlock_always(&c_seg->c_lock);
517
518 vm_swap_defragment_swapin++;
519 }
520 PAGE_REPLACEMENT_DISALLOWED(FALSE);
521
522 vm_pageout_io_throttle();
523
524 /*
525 * because write waiters have privilege over readers,
526 * dropping and immediately retaking the master lock will
527 * still allow any thread waiting to acquire the
528 * master lock exclusively an opportunity to take it
529 */
530 PAGE_REPLACEMENT_DISALLOWED(TRUE);
531
532 lck_mtx_lock_spin_always(c_list_lock);
533 }
534 lck_mtx_unlock_always(c_list_lock);
535
536 PAGE_REPLACEMENT_DISALLOWED(FALSE);
537}
538
539
540
541static void
542vm_swapfile_mgmt_thread(void)
543{
544
545 boolean_t did_work = FALSE;
546 clock_sec_t sec;
547 clock_nsec_t nsec;
548
549 vm_swapfile_mgmt_thread_awakened++;
550 vm_swapfile_mgmt_thread_running = 1;
551
552try_again:
553
554 do {
555 if (vm_swap_up == FALSE)
556 break;
557 did_work = FALSE;
558 clock_get_system_nanotime(&sec, &nsec);
559
560 /*
561 * walk through the list of swap files
562 * and do the delayed frees/trims for
563 * any swap file whose count of delayed
564 * frees is above the batch limit
565 */
566 vm_swap_handle_delayed_trims(FALSE);
567
568 if (VM_SWAP_SHOULD_CREATE(sec)) {
569 if (vm_swap_create_file() == TRUE)
570 did_work = TRUE;
571 else {
572 vm_swapfile_last_failed_to_create_ts = sec;
573 HIBLOG("vm_swap_create_file failed @ %lu secs\n", sec);
574 }
575 }
576 if (VM_SWAP_SHOULD_DEFRAGMENT()) {
577 proc_set_task_policy_thread(kernel_task, current_thread()->thread_id,
578 TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER2);
579
580 vm_swap_defragment();
581
582 if (!VM_SWAP_BUSY())
583 did_work = TRUE;
584
585 proc_set_task_policy_thread(kernel_task, current_thread()->thread_id,
586 TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER1);
587 }
588 if (VM_SWAP_SHOULD_RECLAIM()) {
589 proc_set_task_policy_thread(kernel_task, current_thread()->thread_id,
590 TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER2);
591
592 vm_swap_defragment();
593 vm_swap_reclaim();
594
595 if (!VM_SWAP_BUSY())
596 did_work = TRUE;
597
598 proc_set_task_policy_thread(kernel_task, current_thread()->thread_id,
599 TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER1);
600 }
601
602 } while (did_work == TRUE);
603
604 lck_mtx_lock(&vm_swap_data_lock);
605
606 clock_get_system_nanotime(&sec, &nsec);
607
608 if (vm_swap_up == TRUE && (VM_SWAP_SHOULD_CREATE(sec) || ((!VM_SWAP_BUSY() && compressor_store_stop_compaction == FALSE) &&
609 (VM_SWAP_SHOULD_DEFRAGMENT() || VM_SWAP_SHOULD_RECLAIM())))) {
610 lck_mtx_unlock(&vm_swap_data_lock);
611 goto try_again;
612 }
613
614 vm_swapfile_mgmt_thread_running = 0;
615
616 assert_wait((event_t)&vm_swapfile_mgmt_needed, THREAD_UNINT);
617
618 lck_mtx_unlock(&vm_swap_data_lock);
619
620 thread_block((thread_continue_t)vm_swapfile_mgmt_thread);
621
622 /* NOTREACHED */
623}
624
625
626
627int swapper_entered_T0 = 0;
628int swapper_entered_T1 = 0;
629int swapper_entered_T2 = 0;
630
631static void
632vm_swapout_thread_throttle_adjust(void)
633{
634 int swapper_throttle_new;
635
636 if (swapper_throttle_inited == FALSE) {
637 /*
638 * force this thread to be set to the correct
639 * throttling tier
640 */
641 swapper_throttle_new = THROTTLE_LEVEL_COMPRESSOR_TIER2;
642 swapper_throttle = THROTTLE_LEVEL_COMPRESSOR_TIER1;
643 swapper_throttle_inited = TRUE;
644 swapper_entered_T2++;
645 goto done;
646 }
647 swapper_throttle_new = swapper_throttle;
648
649
650 switch(swapper_throttle) {
651
652 case THROTTLE_LEVEL_COMPRESSOR_TIER2:
653
654 if (SWAPPER_NEEDS_TO_UNTHROTTLE() || swapout_target_age || hibernate_flushing == TRUE) {
655 swapper_throttle_new = THROTTLE_LEVEL_COMPRESSOR_TIER1;
656 swapper_entered_T1++;
657 break;
658 }
659 break;
660
661 case THROTTLE_LEVEL_COMPRESSOR_TIER1:
662
663 if (VM_PAGEOUT_SCAN_NEEDS_TO_THROTTLE()) {
664 swapper_throttle_new = THROTTLE_LEVEL_COMPRESSOR_TIER0;
665 swapper_entered_T0++;
666 break;
667 }
668 if (COMPRESSOR_NEEDS_TO_SWAP() == 0 && swapout_target_age == 0 && hibernate_flushing == FALSE) {
669 swapper_throttle_new = THROTTLE_LEVEL_COMPRESSOR_TIER2;
670 swapper_entered_T2++;
671 break;
672 }
673 break;
674
675 case THROTTLE_LEVEL_COMPRESSOR_TIER0:
676
677 if (COMPRESSOR_NEEDS_TO_SWAP() == 0) {
678 swapper_throttle_new = THROTTLE_LEVEL_COMPRESSOR_TIER2;
679 swapper_entered_T2++;
680 break;
681 }
682 if (SWAPPER_NEEDS_TO_UNTHROTTLE() == 0) {
683 swapper_throttle_new = THROTTLE_LEVEL_COMPRESSOR_TIER1;
684 swapper_entered_T1++;
685 break;
686 }
687 break;
688 }
689done:
690 if (swapper_throttle != swapper_throttle_new) {
691 proc_set_task_policy_thread(kernel_task, vm_swapout_thread_id,
692 TASK_POLICY_INTERNAL, TASK_POLICY_IO, swapper_throttle_new);
693 proc_set_task_policy_thread(kernel_task, vm_swapout_thread_id,
694 TASK_POLICY_INTERNAL, TASK_POLICY_PASSIVE_IO, TASK_POLICY_ENABLE);
695
696 swapper_throttle = swapper_throttle_new;
697 }
698}
699
700
701static void
702vm_swapout_thread(void)
703{
704 uint64_t f_offset = 0;
705 uint32_t size = 0;
706 c_segment_t c_seg = NULL;
707 kern_return_t kr = KERN_SUCCESS;
708 vm_offset_t addr = 0;
709
710 vm_swapout_thread_awakened++;
711
712 lck_mtx_lock_spin_always(c_list_lock);
713
714 while (!queue_empty(&c_swapout_list_head)) {
715
716 c_seg = (c_segment_t)queue_first(&c_swapout_list_head);
717
718 lck_mtx_lock_spin_always(&c_seg->c_lock);
719
720 assert(c_seg->c_on_swapout_q);
721
722 if (c_seg->c_busy) {
39236c6e
A		 723 lck_mtx_unlock_always(c_list_lock);
724
8a3053a0 725 c_seg_wait_on_busy(c_seg);
39236c6e
A		 726
727 lck_mtx_lock_spin_always(c_list_lock);
728
729 continue;
730 }
731 queue_remove(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
732 c_seg->c_on_swapout_q = 0;
733 c_swapout_count--;
734
39236c6e
A		 735 vm_swapout_thread_processed_segments++;
736
737 thread_wakeup((event_t)&compaction_swapper_running);
738
8a3053a0
A		 739 size = round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset));
740
741 if (size == 0) {
742 c_seg_free_locked(c_seg);
743 goto c_seg_was_freed;
744 }
745 c_seg->c_busy = 1;
746 c_seg->c_busy_swapping = 1;
747
39236c6e
A		 748 lck_mtx_unlock_always(c_list_lock);
749
750 addr = (vm_offset_t) c_seg->c_store.c_buffer;
751
39236c6e
A		 752 lck_mtx_unlock_always(&c_seg->c_lock);
753
754#if CHECKSUM_THE_SWAP
755 c_seg->cseg_hash = hash_string((char*)addr, (int)size);
756 c_seg->cseg_swap_size = size;
757#endif /* CHECKSUM_THE_SWAP */
758
759#if CRYPTO
760 vm_swap_encrypt(c_seg);
761#endif /* CRYPTO */
762
763 vm_swapout_thread_throttle_adjust();
764
765 kr = vm_swap_put((vm_offset_t) addr, &f_offset, size, c_seg);
766
767 PAGE_REPLACEMENT_DISALLOWED(TRUE);
768
769 lck_mtx_lock_spin_always(c_list_lock);
770 lck_mtx_lock_spin_always(&c_seg->c_lock);
771
772 if (kr == KERN_SUCCESS) {
773
774 if (C_SEG_ONDISK_IS_SPARSE(c_seg) && hibernate_flushing == FALSE) {
775
776 c_seg_insert_into_q(&c_swappedout_sparse_list_head, c_seg);
777 c_seg->c_on_swappedout_sparse_q = 1;
778 c_swappedout_sparse_count++;
779
780 } else {
781 if (hibernate_flushing == TRUE && (c_seg->c_generation_id >= first_c_segment_to_warm_generation_id &&
782 c_seg->c_generation_id <= last_c_segment_to_warm_generation_id))
783 queue_enter_first(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
784 else
785 queue_enter(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
786 c_seg->c_on_swappedout_q = 1;
787 c_swappedout_count++;
788 }
789 c_seg->c_store.c_swap_handle = f_offset;
790 c_seg->c_ondisk = 1;
791
792 VM_STAT_INCR_BY(swapouts, size >> PAGE_SHIFT);
793
794 if (c_seg->c_bytes_used)
795 OSAddAtomic64(-c_seg->c_bytes_used, &compressor_bytes_used);
796 } else {
797#if CRYPTO
798 vm_swap_decrypt(c_seg);
799#endif /* CRYPTO */
800 c_seg_insert_into_q(&c_age_list_head, c_seg);
801 c_seg->c_on_age_q = 1;
802 c_age_count++;
803
804 vm_swap_put_failures++;
805 }
806 lck_mtx_unlock_always(c_list_lock);
807
808 c_seg->c_busy_swapping = 0;
809
810 C_SEG_WAKEUP_DONE(c_seg);
811
812 if (c_seg->c_must_free)
813 c_seg_free(c_seg);
814 else
815 lck_mtx_unlock_always(&c_seg->c_lock);
816
817 if (kr == KERN_SUCCESS)
818 kernel_memory_depopulate(kernel_map, (vm_offset_t) addr, size, KMA_COMPRESSOR);
819
820 PAGE_REPLACEMENT_DISALLOWED(FALSE);
821
822 if (kr == KERN_SUCCESS)
823 kmem_free(kernel_map, (vm_offset_t) addr, C_SEG_ALLOCSIZE);
824
825 vm_pageout_io_throttle();
8a3053a0 826c_seg_was_freed:
39236c6e
A		 827 if (c_swapout_count == 0)
828 vm_swap_consider_defragmenting();
829
830 lck_mtx_lock_spin_always(c_list_lock);
831 }
832
833 assert_wait((event_t)&c_swapout_list_head, THREAD_UNINT);
834
835 lck_mtx_unlock_always(c_list_lock);
836
837 thread_block((thread_continue_t)vm_swapout_thread);
838
839 /* NOTREACHED */
840}
841
842boolean_t
843vm_swap_create_file()
844{
845 uint64_t size = 0;
846 int namelen = 0;
847 boolean_t swap_file_created = FALSE;
848 boolean_t swap_file_reuse = FALSE;
849 struct swapfile *swf = NULL;
850
851
852 if (DEFAULT_PAGER_IS_ACTIVE || DEFAULT_FREEZER_IS_ACTIVE) {
853 }
854
855 /*
856 * Any swapfile structure ready for re-use?
857 */
858
859 lck_mtx_lock(&vm_swap_data_lock);
860
861 swf = (struct swapfile*) queue_first(&swf_global_queue);
862
863 while (queue_end(&swf_global_queue, (queue_entry_t)swf) == FALSE) {
864 if (swf->swp_flags == SWAP_REUSE) {
865 swap_file_reuse = TRUE;
866 break;
867 }
868 swf = (struct swapfile*) queue_next(&swf->swp_queue);
869 }
870
871 lck_mtx_unlock(&vm_swap_data_lock);
872
873 if (swap_file_reuse == FALSE) {
874
875 namelen = SWAPFILENAME_LEN + SWAPFILENAME_INDEX_LEN + 1;
876
877 swf = (struct swapfile*) kalloc(sizeof *swf);
878 memset(swf, 0, sizeof(*swf));
879
880 swf->swp_index = vm_num_swap_files + 1;
881 swf->swp_pathlen = namelen;
882 swf->swp_path = (char*)kalloc(swf->swp_pathlen);
883
884 memset(swf->swp_path, 0, namelen);
885
886 snprintf(swf->swp_path, namelen, "%s%d", SWAP_FILE_NAME, vm_num_swap_files + 1);
887 }
888
889 vm_swapfile_open(swf->swp_path, &swf->swp_vp);
890
891 if (swf->swp_vp == NULL) {
892 if (swap_file_reuse == FALSE) {
893 kfree(swf->swp_path, swf->swp_pathlen);
894 kfree(swf, sizeof *swf);
895 }
896 return FALSE;
897 }
898 size = MAX_SWAP_FILE_SIZE;
899
900 while (size >= MIN_SWAP_FILE_SIZE) {
901
902 if (vm_swapfile_preallocate(swf->swp_vp, &size) == 0) {
903
904 int num_bytes_for_bitmap = 0;
905
906 swap_file_created = TRUE;
907
908 swf->swp_size = size;
909 swf->swp_nsegs = (unsigned int) (size / COMPRESSED_SWAP_CHUNK_SIZE);
910 swf->swp_nseginuse = 0;
911 swf->swp_free_hint = 0;
912
913 num_bytes_for_bitmap = MAX((swf->swp_nsegs >> 3) , 1);
914 /*
915 * Allocate a bitmap that describes the
916 * number of segments held by this swapfile.
917 */
918 swf->swp_bitmap = (uint8_t*)kalloc(num_bytes_for_bitmap);
919 memset(swf->swp_bitmap, 0, num_bytes_for_bitmap);
920
921 swf->swp_csegs = (c_segment_t *) kalloc(swf->swp_nsegs * sizeof(c_segment_t));
922 memset(swf->swp_csegs, 0, (swf->swp_nsegs * sizeof(c_segment_t)));
923
924 /*
925 * passing a NULL trim_list into vnode_trim_list
926 * will return ENOTSUP if trim isn't supported
927 * and 0 if it is
928 */
929 if (vnode_trim_list(swf->swp_vp, NULL))
930 swf->swp_trim_supported = FALSE;
931 else
932 swf->swp_trim_supported = TRUE;
933
934 lck_mtx_lock(&vm_swap_data_lock);
935
936 swf->swp_flags = SWAP_READY;
937
938 if (swap_file_reuse == FALSE) {
939 queue_enter(&swf_global_queue, swf, struct swapfile*, swp_queue);
940 }
941
942 vm_num_swap_files++;
943
944 vm_swapfile_total_segs_alloced += swf->swp_nsegs;
945
946 lck_mtx_unlock(&vm_swap_data_lock);
947
948 thread_wakeup((event_t) &vm_num_swap_files);
949
950 break;
951 } else {
952
953 size = size / 2;
954 }
955 }
956 if (swap_file_created == FALSE) {
957
958 vm_swapfile_close((uint64_t)(swf->swp_path), swf->swp_vp);
959
960 swf->swp_vp = NULL;
961
962 if (swap_file_reuse == FALSE) {
963 kfree(swf->swp_path, swf->swp_pathlen);
964 kfree(swf, sizeof *swf);
965 }
966 }
967 return swap_file_created;
968}
969
970
971kern_return_t
972vm_swap_get(vm_offset_t addr, uint64_t f_offset, uint64_t size)
973{
974 struct swapfile *swf = NULL;
975 uint64_t file_offset = 0;
976 int retval;
977
978 if (addr == 0) {
979 return KERN_FAILURE;
980 }
981
982 lck_mtx_lock(&vm_swap_data_lock);
983
984 swf = vm_swapfile_for_handle(f_offset);
985
986 if (swf) {
987 if ((swf->swp_flags & SWAP_READY) || (swf->swp_flags & SWAP_RECLAIM)) {
988
989 swf->swp_io_count++;
990 file_offset = (f_offset & SWAP_SLOT_MASK);
991
992 lck_mtx_unlock(&vm_swap_data_lock);
993
994 } else {
995
996 lck_mtx_unlock(&vm_swap_data_lock);
997 return KERN_FAILURE;
998 }
999 } else {
1000
1001 lck_mtx_unlock(&vm_swap_data_lock);
1002 return KERN_FAILURE;
1003 }
1004
1005 retval = vm_swapfile_io(swf->swp_vp, file_offset, addr, (int)(size / PAGE_SIZE_64), SWAP_READ);
1006
1007 /*
1008 * Free this slot in the swap structure.
1009 */
1010 vm_swap_free(f_offset);
1011
1012 lck_mtx_lock(&vm_swap_data_lock);
1013 swf->swp_io_count--;
1014
1015 if ((swf->swp_flags & SWAP_WANTED) && swf->swp_io_count == 0) {
1016
1017 swf->swp_flags &= ~SWAP_WANTED;
1018 thread_wakeup((event_t) &swf->swp_flags);
1019 }
1020 if (retval == 0)
1021 VM_STAT_INCR_BY(swapins, size >> PAGE_SHIFT);
1022 lck_mtx_unlock(&vm_swap_data_lock);
1023
1024 if (retval == 0)
1025 return KERN_SUCCESS;
1026 else {
1027 vm_swap_get_failures++;
1028 return KERN_FAILURE;
1029 }
1030}
1031
1032kern_return_t
1033vm_swap_put(vm_offset_t addr, uint64_t *f_offset, uint64_t size, c_segment_t c_seg)
1034{
1035 unsigned int segidx = 0;
1036 struct swapfile *swf = NULL;
1037 uint64_t file_offset = 0;
1038 uint64_t swapfile_index = 0;
1039 unsigned int byte_for_segidx = 0;
1040 unsigned int offset_within_byte = 0;
1041 boolean_t swf_eligible = FALSE;
1042 boolean_t waiting = FALSE;
1043 int error = 0;
1044 clock_sec_t sec;
1045 clock_nsec_t nsec;
1046
1047 if (addr == 0 || f_offset == NULL) {
1048 return KERN_FAILURE;
1049 }
1050
1051 lck_mtx_lock(&vm_swap_data_lock);
1052
1053 swf = (struct swapfile*) queue_first(&swf_global_queue);
1054
1055 while(queue_end(&swf_global_queue, (queue_entry_t)swf) == FALSE) {
1056
1057 segidx = swf->swp_free_hint;
1058
1059 swf_eligible = (swf->swp_flags & SWAP_READY) && (swf->swp_nseginuse < swf->swp_nsegs);
1060
1061 if (swf_eligible) {
1062
1063 while(segidx < swf->swp_nsegs) {
1064
1065 byte_for_segidx = segidx >> 3;
1066 offset_within_byte = segidx % 8;
1067
1068 if ((swf->swp_bitmap)[byte_for_segidx] & (1 << offset_within_byte)) {
1069 segidx++;
1070 continue;
1071 }
1072
1073 (swf->swp_bitmap)[byte_for_segidx] |= (1 << offset_within_byte);
1074
1075 file_offset = segidx * COMPRESSED_SWAP_CHUNK_SIZE;
1076 swf->swp_nseginuse++;
1077 swf->swp_io_count++;
1078 swapfile_index = swf->swp_index;
1079
1080 vm_swapfile_total_segs_used++;
1081
1082 clock_get_system_nanotime(&sec, &nsec);
1083
1084 if (VM_SWAP_SHOULD_CREATE(sec) && !vm_swapfile_mgmt_thread_running)
1085 thread_wakeup((event_t) &vm_swapfile_mgmt_needed);
1086
1087 lck_mtx_unlock(&vm_swap_data_lock);
1088
1089 goto done;
1090 }
1091 }
1092 swf = (struct swapfile*) queue_next(&swf->swp_queue);
1093 }
1094 assert(queue_end(&swf_global_queue, (queue_entry_t) swf));
1095
1096 /*
1097 * we've run out of swap segments, but may not
1098 * be in a position to immediately create a new swap
1099 * file if we've recently failed to create due to a lack
1100 * of free space in the root filesystem... we'll try
1101 * to kick that create off, but in any event we're going
1102 * to take a breather (up to 1 second) so that we're not caught in a tight
1103 * loop back in "vm_compressor_compact_and_swap" trying to stuff
1104 * segments into swap files only to have them immediately put back
1105 * on the c_age queue due to vm_swap_put failing.
1106 *
1107 * if we're doing these puts due to a hibernation flush,
1108 * no need to block... setting hibernate_no_swapspace to TRUE,
1109 * will cause "vm_compressor_compact_and_swap" to immediately abort
1110 */
1111 clock_get_system_nanotime(&sec, &nsec);
1112
1113 if (VM_SWAP_SHOULD_CREATE(sec) && !vm_swapfile_mgmt_thread_running)
1114 thread_wakeup((event_t) &vm_swapfile_mgmt_needed);
1115
1116 if (hibernate_flushing == FALSE || VM_SWAP_SHOULD_CREATE(sec)) {
1117 waiting = TRUE;
1118 assert_wait_timeout((event_t) &vm_num_swap_files, THREAD_INTERRUPTIBLE, 1000, 1000*NSEC_PER_USEC);
1119 } else
1120 hibernate_no_swapspace = TRUE;
1121
1122 lck_mtx_unlock(&vm_swap_data_lock);
1123
1124 if (waiting == TRUE)
1125 thread_block(THREAD_CONTINUE_NULL);
1126
1127 return KERN_FAILURE;
1128
1129done:
1130 error = vm_swapfile_io(swf->swp_vp, file_offset, addr, (int) (size / PAGE_SIZE_64), SWAP_WRITE);
1131
1132 lck_mtx_lock(&vm_swap_data_lock);
1133
1134 swf->swp_csegs[segidx] = c_seg;
1135
1136 swf->swp_io_count--;
1137
1138 *f_offset = (swapfile_index << SWAP_DEVICE_SHIFT) | file_offset;
1139
1140 if ((swf->swp_flags & SWAP_WANTED) && swf->swp_io_count == 0) {
1141
1142 swf->swp_flags &= ~SWAP_WANTED;
1143 thread_wakeup((event_t) &swf->swp_flags);
1144 }
1145
1146 lck_mtx_unlock(&vm_swap_data_lock);
1147
1148#if SANITY_CHECK_SWAP_ROUTINES
1149 printf("Returned 0x%llx as offset\n", *f_offset);
1150#endif /* SANITY_CHECK_SWAP_ROUTINES */
1151
1152 if (error) {
1153 vm_swap_free(*f_offset);
1154
1155 return KERN_FAILURE;
1156 }
1157 return KERN_SUCCESS;
1158}
1159
1160
1161
1162static void
1163vm_swap_free_now(struct swapfile *swf, uint64_t f_offset)
1164{
1165 uint64_t file_offset = 0;
1166 unsigned int segidx = 0;
1167
1168
1169 if ((swf->swp_flags & SWAP_READY) || (swf->swp_flags & SWAP_RECLAIM)) {
1170
1171 unsigned int byte_for_segidx = 0;
1172 unsigned int offset_within_byte = 0;
1173
1174 file_offset = (f_offset & SWAP_SLOT_MASK);
1175 segidx = (unsigned int) (file_offset / COMPRESSED_SWAP_CHUNK_SIZE);
1176
1177 byte_for_segidx = segidx >> 3;
1178 offset_within_byte = segidx % 8;
1179
1180 if ((swf->swp_bitmap)[byte_for_segidx] & (1 << offset_within_byte)) {
1181
1182 (swf->swp_bitmap)[byte_for_segidx] &= ~(1 << offset_within_byte);
1183
1184 swf->swp_csegs[segidx] = NULL;
1185
1186 swf->swp_nseginuse--;
1187 vm_swapfile_total_segs_used--;
1188
1189 if (segidx < swf->swp_free_hint) {
1190 swf->swp_free_hint = segidx;
1191 }
1192 }
1193 if (VM_SWAP_SHOULD_RECLAIM() && !vm_swapfile_mgmt_thread_running)
1194 thread_wakeup((event_t) &vm_swapfile_mgmt_needed);
1195 }
1196 lck_mtx_unlock(&vm_swap_data_lock);
1197}
1198
1199
1200uint32_t vm_swap_free_now_count = 0;
1201uint32_t vm_swap_free_delayed_count = 0;
1202
1203
1204void
1205vm_swap_free(uint64_t f_offset)
1206{
1207 struct swapfile *swf = NULL;
1208 struct trim_list *tl;
1209 clock_sec_t sec;
1210 clock_nsec_t nsec;
1211
1212 lck_mtx_lock(&vm_swap_data_lock);
1213
1214 swf = vm_swapfile_for_handle(f_offset);
1215
1216 if (swf && (swf->swp_flags & (SWAP_READY | SWAP_RECLAIM))) {
1217
1218 if (swf->swp_trim_supported == FALSE || (swf->swp_flags & SWAP_RECLAIM)) {
1219 /*
1220 * don't delay the free if the underlying disk doesn't support
1221 * trim, or we're in the midst of reclaiming this swap file since
1222 * we don't want to move segments that are technically free
1223 * but not yet handled by the delayed free mechanism
1224 */
1225 vm_swap_free_now(swf, f_offset);
1226
1227 vm_swap_free_now_count++;
1228 return;
1229 }
1230 tl = kalloc(sizeof(struct trim_list));
1231
1232 tl->tl_offset = f_offset & SWAP_SLOT_MASK;
1233 tl->tl_length = COMPRESSED_SWAP_CHUNK_SIZE;
1234
1235 tl->tl_next = swf->swp_delayed_trim_list_head;
1236 swf->swp_delayed_trim_list_head = tl;
1237 swf->swp_delayed_trim_count++;
1238
1239 if (VM_SWAP_SHOULD_TRIM(swf) && !vm_swapfile_mgmt_thread_running) {
1240 clock_get_system_nanotime(&sec, &nsec);
1241
1242 if (sec > dont_trim_until_ts)
1243 thread_wakeup((event_t) &vm_swapfile_mgmt_needed);
1244 }
1245 vm_swap_free_delayed_count++;
1246 }
1247 lck_mtx_unlock(&vm_swap_data_lock);
1248}
1249
1250
1251static void
1252vm_swap_handle_delayed_trims(boolean_t force_now)
1253{
1254 struct swapfile *swf = NULL;
1255
1256 /*
1257 * because swap files are created or reclaimed on the
1258 * same thread that calls this function, it's safe
1259 * to iterate "swf_global_queue" w/o holding
1260 * the lock since those are the only 2 cases that can
1261 * change the items on the "swf_global_queue"
1262 */
1263 swf = (struct swapfile*) queue_first(&swf_global_queue);
1264
1265 while (queue_end(&swf_global_queue, (queue_entry_t)swf) == FALSE) {
1266
1267 assert(!(swf->swp_flags & SWAP_RECLAIM));
1268
1269 if ((swf->swp_flags & SWAP_READY) && (force_now == TRUE || VM_SWAP_SHOULD_TRIM(swf)))
1270 vm_swap_do_delayed_trim(swf);
1271
1272 swf = (struct swapfile*) queue_next(&swf->swp_queue);
1273 }
1274}
1275
1276
1277static void
1278vm_swap_do_delayed_trim(struct swapfile *swf)
1279{
1280 struct trim_list *tl, *tl_head;
1281
1282 lck_mtx_lock(&vm_swap_data_lock);
1283
1284 tl_head = swf->swp_delayed_trim_list_head;
1285 swf->swp_delayed_trim_list_head = NULL;
1286 swf->swp_delayed_trim_count = 0;
1287
1288 lck_mtx_unlock(&vm_swap_data_lock);
1289
1290 vnode_trim_list(swf->swp_vp, tl_head);
1291
1292 while ((tl = tl_head) != NULL) {
1293 unsigned int segidx = 0;
1294 unsigned int byte_for_segidx = 0;
1295 unsigned int offset_within_byte = 0;
1296
1297 lck_mtx_lock(&vm_swap_data_lock);
1298
1299 segidx = (unsigned int) (tl->tl_offset / COMPRESSED_SWAP_CHUNK_SIZE);
1300
1301 byte_for_segidx = segidx >> 3;
1302 offset_within_byte = segidx % 8;
1303
1304 if ((swf->swp_bitmap)[byte_for_segidx] & (1 << offset_within_byte)) {
1305
1306 (swf->swp_bitmap)[byte_for_segidx] &= ~(1 << offset_within_byte);
1307
1308 swf->swp_csegs[segidx] = NULL;
1309
1310 swf->swp_nseginuse--;
1311 vm_swapfile_total_segs_used--;
1312
1313 if (segidx < swf->swp_free_hint) {
1314 swf->swp_free_hint = segidx;
1315 }
1316 }
1317 lck_mtx_unlock(&vm_swap_data_lock);
1318
1319 tl_head = tl->tl_next;
1320
1321 kfree(tl, sizeof(struct trim_list));
1322 }
1323}
1324
1325
1326void
1327vm_swap_flush()
1328{
1329 return;
1330}
1331
1332int vm_swap_reclaim_yielded = 0;
1333
1334void
1335vm_swap_reclaim(void)
1336{
1337 vm_offset_t addr = 0;
1338 unsigned int segidx = 0;
1339 uint64_t f_offset = 0;
1340 struct swapfile *swf = NULL;
1341 struct swapfile *smallest_swf = NULL;
1342 unsigned int min_nsegs = 0;
1343 unsigned int byte_for_segidx = 0;
1344 unsigned int offset_within_byte = 0;
1345 uint32_t c_size = 0;
1346
1347 c_segment_t c_seg = NULL;
1348
1349 if (kernel_memory_allocate(kernel_map, (vm_offset_t *)(&addr), C_SEG_BUFSIZE, 0, KMA_KOBJECT) != KERN_SUCCESS) {
1350 panic("vm_swap_reclaim: kernel_memory_allocate failed\n");
1351 }
1352
1353 lck_mtx_lock(&vm_swap_data_lock);
1354
1355 swf = (struct swapfile*) queue_first(&swf_global_queue);
1356 min_nsegs = MAX_SWAP_FILE_SIZE / COMPRESSED_SWAP_CHUNK_SIZE;
1357 smallest_swf = NULL;
1358
1359 while (queue_end(&swf_global_queue, (queue_entry_t)swf) == FALSE) {
1360
1361 if ((swf->swp_flags & SWAP_READY) && (swf->swp_nseginuse <= min_nsegs)) {
1362
1363 smallest_swf = swf;
1364 min_nsegs = swf->swp_nseginuse;
1365 }
1366 swf = (struct swapfile*) queue_next(&swf->swp_queue);
1367 }
1368
1369 if (smallest_swf == NULL)
1370 goto done;
1371
1372 swf = smallest_swf;
1373
1374
1375 swf->swp_flags &= ~SWAP_READY;
1376 swf->swp_flags |= SWAP_RECLAIM;
1377
1378 if (swf->swp_delayed_trim_count) {
1379
1380 lck_mtx_unlock(&vm_swap_data_lock);
1381
1382 vm_swap_do_delayed_trim(swf);
1383
1384 lck_mtx_lock(&vm_swap_data_lock);
1385 }
1386 segidx = 0;
1387
1388 while (segidx < swf->swp_nsegs) {
1389
1390ReTry_for_cseg:
1391 if (compressor_store_stop_compaction == TRUE || (swf->swp_trim_supported == FALSE && VM_SWAP_BUSY())) {
1392 vm_swap_reclaim_yielded++;
1393 break;
1394 }
1395 /*
1396 * Wait for outgoing I/Os.
1397 */
1398 while (swf->swp_io_count) {
1399
1400 swf->swp_flags |= SWAP_WANTED;
1401
1402 assert_wait((event_t) &swf->swp_flags, THREAD_UNINT);
1403 lck_mtx_unlock(&vm_swap_data_lock);
1404
1405 thread_block(THREAD_CONTINUE_NULL);
1406
1407 lck_mtx_lock(&vm_swap_data_lock);
1408 }
1409
1410 byte_for_segidx = segidx >> 3;
1411 offset_within_byte = segidx % 8;
1412
1413 if (((swf->swp_bitmap)[byte_for_segidx] & (1 << offset_within_byte)) == 0) {
1414
1415 segidx++;
1416 continue;
1417 }
1418
1419 c_seg = swf->swp_csegs[segidx];
1420
1421 lck_mtx_lock_spin_always(&c_seg->c_lock);
1422
1423 assert(c_seg->c_ondisk);
1424
1425 if (c_seg->c_busy) {
1426
1427 c_seg->c_wanted = 1;
1428
1429 assert_wait((event_t) (c_seg), THREAD_UNINT);
1430 lck_mtx_unlock_always(&c_seg->c_lock);
1431
1432 lck_mtx_unlock(&vm_swap_data_lock);
1433
1434 thread_block(THREAD_CONTINUE_NULL);
1435
1436 lck_mtx_lock(&vm_swap_data_lock);
1437
1438 goto ReTry_for_cseg;
1439 }
1440 (swf->swp_bitmap)[byte_for_segidx] &= ~(1 << offset_within_byte);
1441
1442 f_offset = segidx * COMPRESSED_SWAP_CHUNK_SIZE;
1443
1444 swf->swp_csegs[segidx] = NULL;
1445 swf->swp_nseginuse--;
1446
1447 vm_swapfile_total_segs_used--;
1448
1449 lck_mtx_unlock(&vm_swap_data_lock);
1450
1451 if (c_seg->c_must_free) {
1452
1453 c_seg_free(c_seg);
1454 } else {
1455
1456 c_seg->c_busy = 1;
1457 c_seg->c_busy_swapping = 1;
1458#if !CHECKSUM_THE_SWAP
1459 c_seg_trim_tail(c_seg);
1460#endif
1461
1462#if SANITY_CHECK_SWAP_ROUTINES
1463
1464 c_size = COMPRESSED_SWAP_CHUNK_SIZE;
1465
1466#else /* SANITY_CHECK_SWAP_ROUTINES */
1467
1468 c_size = round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset));
1469
1470 assert(c_size <= C_SEG_BUFSIZE);
1471
1472#endif /* SANITY_CHECK_SWAP_ROUTINES */
1473
1474 lck_mtx_unlock_always(&c_seg->c_lock);
1475
1476 if (vm_swapfile_io(swf->swp_vp, f_offset, addr, (int)(c_size / PAGE_SIZE_64), SWAP_READ)) {
1477
1478 /*
1479 * reading the data back in failed, so convert c_seg
1480 * to a swapped in c_segment that contains no data
1481 */
1482 c_seg->c_store.c_buffer = (int32_t *)NULL;
1483 c_seg_swapin_requeue(c_seg);
1484
1485 goto swap_io_failed;
1486 }
1487 VM_STAT_INCR_BY(swapins, c_size >> PAGE_SHIFT);
1488
1489 if (vm_swap_put(addr, &f_offset, c_size, c_seg)) {
1490 vm_offset_t c_buffer;
1491
1492 /*
1493 * the put failed, so convert c_seg to a fully swapped in c_segment
1494 * with valid data
1495 */
1496 if (kernel_memory_allocate(kernel_map, &c_buffer, C_SEG_ALLOCSIZE, 0, KMA_COMPRESSOR | KMA_VAONLY) != KERN_SUCCESS)
1497 panic("vm_swap_reclaim: kernel_memory_allocate failed\n");
1498 kernel_memory_populate(kernel_map, c_buffer, c_size, KMA_COMPRESSOR);
1499
1500 memcpy((char *)c_buffer, (char *)addr, c_size);
1501
1502 c_seg->c_store.c_buffer = (int32_t *)c_buffer;
1503#if CRYPTO
1504 vm_swap_decrypt(c_seg);
1505#endif /* CRYPTO */
1506 c_seg_swapin_requeue(c_seg);
1507
1508 OSAddAtomic64(c_seg->c_bytes_used, &compressor_bytes_used);
1509
1510 goto swap_io_failed;
1511 }
1512 VM_STAT_INCR_BY(swapouts, c_size >> PAGE_SHIFT);
1513
1514 lck_mtx_lock_spin_always(&c_seg->c_lock);
1515
1516 assert(c_seg->c_ondisk);
1517 /*
1518 * The c_seg will now know about the new location on disk.
1519 */
1520 c_seg->c_store.c_swap_handle = f_offset;
1521swap_io_failed:
1522 c_seg->c_busy_swapping = 0;
1523
1524 if (c_seg->c_must_free)
1525 c_seg_free(c_seg);
1526 else {
1527 C_SEG_WAKEUP_DONE(c_seg);
1528
1529 lck_mtx_unlock_always(&c_seg->c_lock);
1530 }
1531 }
1532 lck_mtx_lock(&vm_swap_data_lock);
1533 }
1534
1535 if (swf->swp_nseginuse) {
1536
1537 swf->swp_flags &= ~SWAP_RECLAIM;
1538 swf->swp_flags |= SWAP_READY;
1539
1540 goto done;
1541 }
1542 /*
1543 * We don't remove this inactive swf from the queue.
1544 * That way, we can re-use it when needed again and
1545 * preserve the namespace.
1546 */
1547 //queue_remove(&swf_global_queue, swf, struct swapfile*, swp_queue);
1548
1549 vm_num_swap_files--;
1550
1551 vm_swapfile_total_segs_alloced -= swf->swp_nsegs;
1552
1553 lck_mtx_unlock(&vm_swap_data_lock);
1554
1555 vm_swapfile_close((uint64_t)(swf->swp_path), swf->swp_vp);
1556
1557 kfree(swf->swp_csegs, swf->swp_nsegs * sizeof(c_segment_t));
1558 kfree(swf->swp_bitmap, MAX((swf->swp_nsegs >> 3), 1));
1559
1560 lck_mtx_lock(&vm_swap_data_lock);
1561
1562 swf->swp_vp = NULL;
1563 swf->swp_size = 0;
1564 swf->swp_free_hint = 0;
1565 swf->swp_nsegs = 0;
1566 swf->swp_flags = SWAP_REUSE;
1567
1568 thread_wakeup((event_t) &swf->swp_flags);
1569done:
1570 lck_mtx_unlock(&vm_swap_data_lock);
1571
1572 kmem_free(kernel_map, (vm_offset_t) addr, C_SEG_BUFSIZE);
1573}
1574
1575
1576uint64_t
1577vm_swap_get_total_space(void)
1578{
1579 uint64_t total_space = 0;
1580
1581 total_space = (uint64_t)vm_swapfile_total_segs_alloced * COMPRESSED_SWAP_CHUNK_SIZE;
1582
1583 return total_space;
1584}
1585
1586uint64_t
1587vm_swap_get_used_space(void)
1588{
1589 uint64_t used_space = 0;
1590
1591 used_space = (uint64_t)vm_swapfile_total_segs_used * COMPRESSED_SWAP_CHUNK_SIZE;
1592
1593 return used_space;
1594}
1595
1596uint64_t
1597vm_swap_get_free_space(void)
1598{
1599 return (vm_swap_get_total_space() - vm_swap_get_used_space());
1600}
mirror of XNU