]> git.saurik.com Git - apple/xnu.git/blob - osfmk/vm/vm_compressor_backing_store.c
projects / apple / xnu.git / blob
? search:


f4ec70ce5747307fd048635d1bcc9971d300644a
[apple/xnu.git] / osfmk / vm / vm_compressor_backing_store.c
1 /*
2 * Copyright (c) 2000-2013 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include "vm_compressor_backing_store.h"
30 #include <vm/vm_protos.h>
31
32 #include <IOKit/IOHibernatePrivate.h>
33
34 #include <kern/policy_internal.h>
35
36 boolean_t compressor_store_stop_compaction = FALSE;
37 boolean_t vm_swapfile_create_needed = FALSE;
38 boolean_t vm_swapfile_gc_needed = FALSE;
39
40 int swapper_throttle = -1;
41 boolean_t swapper_throttle_inited = FALSE;
42 uint64_t vm_swapout_thread_id;
43
44 uint64_t vm_swap_put_failures = 0;
45 uint64_t vm_swap_get_failures = 0;
46 int vm_num_swap_files = 0;
47 int vm_num_pinned_swap_files = 0;
48 int vm_swapout_thread_processed_segments = 0;
49 int vm_swapout_thread_awakened = 0;
50 int vm_swapfile_create_thread_awakened = 0;
51 int vm_swapfile_create_thread_running = 0;
52 int vm_swapfile_gc_thread_awakened = 0;
53 int vm_swapfile_gc_thread_running = 0;
54
55 int64_t vm_swappin_avail = 0;
56 boolean_t vm_swappin_enabled = FALSE;
57 unsigned int vm_swapfile_total_segs_alloced = 0;
58 unsigned int vm_swapfile_total_segs_used = 0;
59
60 char swapfilename[MAX_SWAPFILENAME_LEN + 1] = SWAP_FILE_NAME;
61
62 extern vm_map_t compressor_map;
63
64
65 #define SWAP_READY 0x1 /* Swap file is ready to be used */
66 #define SWAP_RECLAIM 0x2 /* Swap file is marked to be reclaimed */
67 #define SWAP_WANTED 0x4 /* Swap file has waiters */
68 #define SWAP_REUSE 0x8 /* Swap file is on the Q and has a name. Reuse after init-ing.*/
69 #define SWAP_PINNED 0x10 /* Swap file is pinned (FusionDrive) */
70
71
72 struct swapfile{
73 queue_head_t swp_queue; /* list of swap files */
74 char *swp_path; /* saved pathname of swap file */
75 struct vnode *swp_vp; /* backing vnode */
76 uint64_t swp_size; /* size of this swap file */
77 uint8_t *swp_bitmap; /* bitmap showing the alloced/freed slots in the swap file */
78 unsigned int swp_pathlen; /* length of pathname */
79 unsigned int swp_nsegs; /* #segments we can use */
80 unsigned int swp_nseginuse; /* #segments in use */
81 unsigned int swp_index; /* index of this swap file */
82 unsigned int swp_flags; /* state of swap file */
83 unsigned int swp_free_hint; /* offset of 1st free chunk */
84 unsigned int swp_io_count; /* count of outstanding I/Os */
85 c_segment_t *swp_csegs; /* back pointers to the c_segments. Used during swap reclaim. */
86
87 struct trim_list *swp_delayed_trim_list_head;
88 unsigned int swp_delayed_trim_count;
89 };
90
91 queue_head_t swf_global_queue;
92 boolean_t swp_trim_supported = FALSE;
93
94 extern clock_sec_t dont_trim_until_ts;
95 clock_sec_t vm_swapfile_last_failed_to_create_ts = 0;
96 clock_sec_t vm_swapfile_last_successful_create_ts = 0;
97 int vm_swapfile_can_be_created = FALSE;
98 boolean_t delayed_trim_handling_in_progress = FALSE;
99
100 boolean_t hibernate_in_progress_with_pinned_swap = FALSE;
101
102 static void vm_swapout_thread_throttle_adjust(void);
103 static void vm_swap_free_now(struct swapfile *swf, uint64_t f_offset);
104 static void vm_swapout_thread(void);
105 static void vm_swapfile_create_thread(void);
106 static void vm_swapfile_gc_thread(void);
107 static void vm_swap_defragment(void);
108 static void vm_swap_handle_delayed_trims(boolean_t);
109 static void vm_swap_do_delayed_trim(struct swapfile *);
110 static void vm_swap_wait_on_trim_handling_in_progress(void);
111
112
113 #if CONFIG_EMBEDDED
114 /*
115 * Only 1 swap file currently allowed.
116 */
117 #define VM_MAX_SWAP_FILE_NUM 1
118 #define VM_SWAPFILE_DELAYED_TRIM_MAX 4
119
120 #define VM_SWAP_SHOULD_DEFRAGMENT() (c_swappedout_sparse_count > (vm_swapfile_total_segs_used / 16) ? 1 : 0)
121 #define VM_SWAP_SHOULD_RECLAIM() FALSE
122 #define VM_SWAP_SHOULD_ABORT_RECLAIM() FALSE
123 #define VM_SWAP_SHOULD_PIN(_size) FALSE
124 #define VM_SWAP_SHOULD_CREATE(cur_ts) ((vm_num_swap_files < VM_MAX_SWAP_FILE_NUM) && ((vm_swapfile_total_segs_alloced - vm_swapfile_total_segs_used) < (unsigned int)VM_SWAPFILE_HIWATER_SEGS) && \
125 ((cur_ts - vm_swapfile_last_failed_to_create_ts) > VM_SWAPFILE_DELAYED_CREATE) ? 1 : 0)
126 #define VM_SWAP_SHOULD_TRIM(swf) ((swf->swp_delayed_trim_count >= VM_SWAPFILE_DELAYED_TRIM_MAX) ? 1 : 0)
127
128 #else /* CONFIG_EMBEDDED */
129
130 #define VM_MAX_SWAP_FILE_NUM 100
131 #define VM_SWAPFILE_DELAYED_TRIM_MAX 128
132
133 #define VM_SWAP_SHOULD_DEFRAGMENT() (c_swappedout_sparse_count > (vm_swapfile_total_segs_used / 4) ? 1 : 0)
134 #define VM_SWAP_SHOULD_RECLAIM() (((vm_swapfile_total_segs_alloced - vm_swapfile_total_segs_used) >= SWAPFILE_RECLAIM_THRESHOLD_SEGS) ? 1 : 0)
135 #define VM_SWAP_SHOULD_ABORT_RECLAIM() (((vm_swapfile_total_segs_alloced - vm_swapfile_total_segs_used) <= SWAPFILE_RECLAIM_MINIMUM_SEGS) ? 1 : 0)
136 #define VM_SWAP_SHOULD_PIN(_size) (vm_swappin_avail > 0 && vm_swappin_avail >= (int64_t)(_size))
137 #define VM_SWAP_SHOULD_CREATE(cur_ts) ((vm_num_swap_files < VM_MAX_SWAP_FILE_NUM) && ((vm_swapfile_total_segs_alloced - vm_swapfile_total_segs_used) < (unsigned int)VM_SWAPFILE_HIWATER_SEGS) && \
138 ((cur_ts - vm_swapfile_last_failed_to_create_ts) > VM_SWAPFILE_DELAYED_CREATE) ? 1 : 0)
139 #define VM_SWAP_SHOULD_TRIM(swf) ((swf->swp_delayed_trim_count >= VM_SWAPFILE_DELAYED_TRIM_MAX) ? 1 : 0)
140
141 #endif /* CONFIG_EMBEDDED */
142
143 #define VM_SWAPFILE_DELAYED_CREATE 15
144
145 #define VM_SWAP_BUSY() ((c_swapout_count && (swapper_throttle == THROTTLE_LEVEL_COMPRESSOR_TIER1 || swapper_throttle == THROTTLE_LEVEL_COMPRESSOR_TIER0)) ? 1 : 0)
146
147
148 #if CHECKSUM_THE_SWAP
149 extern unsigned int hash_string(char *cp, int len);
150 #endif
151
152 #if RECORD_THE_COMPRESSED_DATA
153 boolean_t c_compressed_record_init_done = FALSE;
154 int c_compressed_record_write_error = 0;
155 struct vnode *c_compressed_record_vp = NULL;
156 uint64_t c_compressed_record_file_offset = 0;
157 void c_compressed_record_init(void);
158 void c_compressed_record_write(char *, int);
159 #endif
160
161 extern void vm_pageout_io_throttle(void);
162
163 static struct swapfile *vm_swapfile_for_handle(uint64_t);
164
165 /*
166 * Called with the vm_swap_data_lock held.
167 */
168
169 static struct swapfile *
170 vm_swapfile_for_handle(uint64_t f_offset)
171 {
172
173 uint64_t file_offset = 0;
174 unsigned int swapfile_index = 0;
175 struct swapfile* swf = NULL;
176
177 file_offset = (f_offset & SWAP_SLOT_MASK);
178 swapfile_index = (f_offset >> SWAP_DEVICE_SHIFT);
179
180 swf = (struct swapfile*) queue_first(&swf_global_queue);
181
182 while(queue_end(&swf_global_queue, (queue_entry_t)swf) == FALSE) {
183
184 if (swapfile_index == swf->swp_index) {
185 break;
186 }
187
188 swf = (struct swapfile*) queue_next(&swf->swp_queue);
189 }
190
191 if (queue_end(&swf_global_queue, (queue_entry_t) swf)) {
192 swf = NULL;
193 }
194
195 return swf;
196 }
197
198 #if ENCRYPTED_SWAP
199
200 #include <libkern/crypto/aes.h>
201 extern u_int32_t random(void); /* from <libkern/libkern.h> */
202
203 #define SWAP_CRYPT_AES_KEY_SIZE 128 /* XXX 192 and 256 don't work ! */
204
205 boolean_t swap_crypt_ctx_initialized;
206 void swap_crypt_ctx_initialize(void);
207
208 aes_ctx swap_crypt_ctx;
209 const unsigned char swap_crypt_null_iv[AES_BLOCK_SIZE] = {0xa, };
210 uint32_t swap_crypt_key[8]; /* big enough for a 256 key */
211
212 unsigned long vm_page_encrypt_counter;
213 unsigned long vm_page_decrypt_counter;
214
215
216 #if DEBUG
217 boolean_t swap_crypt_ctx_tested = FALSE;
218 unsigned char swap_crypt_test_page_ref[4096] __attribute__((aligned(4096)));
219 unsigned char swap_crypt_test_page_encrypt[4096] __attribute__((aligned(4096)));
220 unsigned char swap_crypt_test_page_decrypt[4096] __attribute__((aligned(4096)));
221 #endif /* DEBUG */
222
223 /*
224 * Initialize the encryption context: key and key size.
225 */
226 void swap_crypt_ctx_initialize(void); /* forward */
227 void
228 swap_crypt_ctx_initialize(void)
229 {
230 unsigned int i;
231
232 /*
233 * No need for locking to protect swap_crypt_ctx_initialized
234 * because the first use of encryption will come from the
235 * pageout thread (we won't pagein before there's been a pageout)
236 * and there's only one pageout thread.
237 */
238 if (swap_crypt_ctx_initialized == FALSE) {
239 for (i = 0;
240 i < (sizeof (swap_crypt_key) /
241 sizeof (swap_crypt_key[0]));
242 i++) {
243 swap_crypt_key[i] = random();
244 }
245 aes_encrypt_key((const unsigned char *) swap_crypt_key,
246 SWAP_CRYPT_AES_KEY_SIZE,
247 &swap_crypt_ctx.encrypt);
248 aes_decrypt_key((const unsigned char *) swap_crypt_key,
249 SWAP_CRYPT_AES_KEY_SIZE,
250 &swap_crypt_ctx.decrypt);
251 swap_crypt_ctx_initialized = TRUE;
252 }
253
254 #if DEBUG
255 /*
256 * Validate the encryption algorithms.
257 */
258 if (swap_crypt_ctx_tested == FALSE) {
259 /* initialize */
260 for (i = 0; i < 4096; i++) {
261 swap_crypt_test_page_ref[i] = (char) i;
262 }
263 /* encrypt */
264 aes_encrypt_cbc(swap_crypt_test_page_ref,
265 swap_crypt_null_iv,
266 PAGE_SIZE / AES_BLOCK_SIZE,
267 swap_crypt_test_page_encrypt,
268 &swap_crypt_ctx.encrypt);
269 /* decrypt */
270 aes_decrypt_cbc(swap_crypt_test_page_encrypt,
271 swap_crypt_null_iv,
272 PAGE_SIZE / AES_BLOCK_SIZE,
273 swap_crypt_test_page_decrypt,
274 &swap_crypt_ctx.decrypt);
275 /* compare result with original */
276 for (i = 0; i < 4096; i ++) {
277 if (swap_crypt_test_page_decrypt[i] !=
278 swap_crypt_test_page_ref[i]) {
279 panic("encryption test failed");
280 }
281 }
282
283 /* encrypt again */
284 aes_encrypt_cbc(swap_crypt_test_page_decrypt,
285 swap_crypt_null_iv,
286 PAGE_SIZE / AES_BLOCK_SIZE,
287 swap_crypt_test_page_decrypt,
288 &swap_crypt_ctx.encrypt);
289 /* decrypt in place */
290 aes_decrypt_cbc(swap_crypt_test_page_decrypt,
291 swap_crypt_null_iv,
292 PAGE_SIZE / AES_BLOCK_SIZE,
293 swap_crypt_test_page_decrypt,
294 &swap_crypt_ctx.decrypt);
295 for (i = 0; i < 4096; i ++) {
296 if (swap_crypt_test_page_decrypt[i] !=
297 swap_crypt_test_page_ref[i]) {
298 panic("in place encryption test failed");
299 }
300 }
301
302 swap_crypt_ctx_tested = TRUE;
303 }
304 #endif /* DEBUG */
305 }
306
307
308 void
309 vm_swap_encrypt(c_segment_t c_seg)
310 {
311 vm_offset_t kernel_vaddr = 0;
312 uint64_t size = 0;
313
314 union {
315 unsigned char aes_iv[AES_BLOCK_SIZE];
316 void *c_seg;
317 } encrypt_iv;
318
319 assert(swap_crypt_ctx_initialized);
320
321 #if DEVELOPMENT || DEBUG
322 C_SEG_MAKE_WRITEABLE(c_seg);
323 #endif
324 bzero(&encrypt_iv.aes_iv[0], sizeof (encrypt_iv.aes_iv));
325
326 encrypt_iv.c_seg = (void*)c_seg;
327
328 /* encrypt the "initial vector" */
329 aes_encrypt_cbc((const unsigned char *) &encrypt_iv.aes_iv[0],
330 swap_crypt_null_iv,
331 1,
332 &encrypt_iv.aes_iv[0],
333 &swap_crypt_ctx.encrypt);
334
335 kernel_vaddr = (vm_offset_t) c_seg->c_store.c_buffer;
336 size = round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset));
337
338 /*
339 * Encrypt the c_segment.
340 */
341 aes_encrypt_cbc((const unsigned char *) kernel_vaddr,
342 &encrypt_iv.aes_iv[0],
343 (unsigned int)(size / AES_BLOCK_SIZE),
344 (unsigned char *) kernel_vaddr,
345 &swap_crypt_ctx.encrypt);
346
347 vm_page_encrypt_counter += (size/PAGE_SIZE_64);
348
349 #if DEVELOPMENT || DEBUG
350 C_SEG_WRITE_PROTECT(c_seg);
351 #endif
352 }
353
354 void
355 vm_swap_decrypt(c_segment_t c_seg)
356 {
357
358 vm_offset_t kernel_vaddr = 0;
359 uint64_t size = 0;
360
361 union {
362 unsigned char aes_iv[AES_BLOCK_SIZE];
363 void *c_seg;
364 } decrypt_iv;
365
366
367 assert(swap_crypt_ctx_initialized);
368
369 #if DEVELOPMENT || DEBUG
370 C_SEG_MAKE_WRITEABLE(c_seg);
371 #endif
372 /*
373 * Prepare an "initial vector" for the decryption.
374 * It has to be the same as the "initial vector" we
375 * used to encrypt that page.
376 */
377 bzero(&decrypt_iv.aes_iv[0], sizeof (decrypt_iv.aes_iv));
378
379 decrypt_iv.c_seg = (void*)c_seg;
380
381 /* encrypt the "initial vector" */
382 aes_encrypt_cbc((const unsigned char *) &decrypt_iv.aes_iv[0],
383 swap_crypt_null_iv,
384 1,
385 &decrypt_iv.aes_iv[0],
386 &swap_crypt_ctx.encrypt);
387
388 kernel_vaddr = (vm_offset_t) c_seg->c_store.c_buffer;
389 size = round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset));
390
391 /*
392 * Decrypt the c_segment.
393 */
394 aes_decrypt_cbc((const unsigned char *) kernel_vaddr,
395 &decrypt_iv.aes_iv[0],
396 (unsigned int) (size / AES_BLOCK_SIZE),
397 (unsigned char *) kernel_vaddr,
398 &swap_crypt_ctx.decrypt);
399
400 vm_page_decrypt_counter += (size/PAGE_SIZE_64);
401
402 #if DEVELOPMENT || DEBUG
403 C_SEG_WRITE_PROTECT(c_seg);
404 #endif
405 }
406 #endif /* ENCRYPTED_SWAP */
407
408
409 void
410 vm_compressor_swap_init()
411 {
412 thread_t thread = NULL;
413
414 lck_grp_attr_setdefault(&vm_swap_data_lock_grp_attr);
415 lck_grp_init(&vm_swap_data_lock_grp,
416 "vm_swap_data",
417 &vm_swap_data_lock_grp_attr);
418 lck_attr_setdefault(&vm_swap_data_lock_attr);
419 lck_mtx_init_ext(&vm_swap_data_lock,
420 &vm_swap_data_lock_ext,
421 &vm_swap_data_lock_grp,
422 &vm_swap_data_lock_attr);
423
424 queue_init(&swf_global_queue);
425
426
427 if (kernel_thread_start_priority((thread_continue_t)vm_swapout_thread, NULL,
428 BASEPRI_VM, &thread) != KERN_SUCCESS) {
429 panic("vm_swapout_thread: create failed");
430 }
431 vm_swapout_thread_id = thread->thread_id;
432
433 thread_deallocate(thread);
434
435 if (kernel_thread_start_priority((thread_continue_t)vm_swapfile_create_thread, NULL,
436 BASEPRI_VM, &thread) != KERN_SUCCESS) {
437 panic("vm_swapfile_create_thread: create failed");
438 }
439
440 thread_deallocate(thread);
441
442 if (kernel_thread_start_priority((thread_continue_t)vm_swapfile_gc_thread, NULL,
443 BASEPRI_VM, &thread) != KERN_SUCCESS) {
444 panic("vm_swapfile_gc_thread: create failed");
445 }
446 thread_deallocate(thread);
447
448 proc_set_thread_policy_with_tid(kernel_task, thread->thread_id,
449 TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER2);
450 proc_set_thread_policy_with_tid(kernel_task, thread->thread_id,
451 TASK_POLICY_INTERNAL, TASK_POLICY_PASSIVE_IO, TASK_POLICY_ENABLE);
452
453 #if ENCRYPTED_SWAP
454 if (swap_crypt_ctx_initialized == FALSE) {
455 swap_crypt_ctx_initialize();
456 }
457 #endif /* ENCRYPTED_SWAP */
458
459 #if CONFIG_EMBEDDED
460 /*
461 * dummy value until the swap file gets created
462 * when we drive the first c_segment_t to the
463 * swapout queue... at that time we will
464 * know the true size we have to work with
465 */
466 c_overage_swapped_limit = 16;
467 #endif
468 printf("VM Swap Subsystem is ON\n");
469 }
470
471
472 #if RECORD_THE_COMPRESSED_DATA
473
474 void
475 c_compressed_record_init()
476 {
477 if (c_compressed_record_init_done == FALSE) {
478 vm_swapfile_open("/tmp/compressed_data", &c_compressed_record_vp);
479 c_compressed_record_init_done = TRUE;
480 }
481 }
482
483 void
484 c_compressed_record_write(char *buf, int size)
485 {
486 if (c_compressed_record_write_error == 0) {
487 c_compressed_record_write_error = vm_record_file_write(c_compressed_record_vp, c_compressed_record_file_offset, buf, size);
488 c_compressed_record_file_offset += size;
489 }
490 }
491 #endif
492
493
494 int compaction_swapper_inited = 0;
495
496 void
497 vm_compaction_swapper_do_init(void)
498 {
499 struct vnode *vp;
500 char *pathname;
501 int namelen;
502
503 if (compaction_swapper_inited)
504 return;
505
506 if (vm_compressor_mode != VM_PAGER_COMPRESSOR_WITH_SWAP) {
507 compaction_swapper_inited = 1;
508 return;
509 }
510 lck_mtx_lock(&vm_swap_data_lock);
511
512 if ( !compaction_swapper_inited) {
513
514 namelen = (int)strlen(swapfilename) + SWAPFILENAME_INDEX_LEN + 1;
515 pathname = (char*)kalloc(namelen);
516 memset(pathname, 0, namelen);
517 snprintf(pathname, namelen, "%s%d", swapfilename, 0);
518
519 vm_swapfile_open(pathname, &vp);
520
521 if (vp) {
522
523 if (vnode_pager_isSSD(vp) == FALSE) {
524 vm_compressor_minorcompact_threshold_divisor = 18;
525 vm_compressor_majorcompact_threshold_divisor = 22;
526 vm_compressor_unthrottle_threshold_divisor = 32;
527 }
528 #if !CONFIG_EMBEDDED
529 vnode_setswapmount(vp);
530 vm_swappin_avail = vnode_getswappin_avail(vp);
531
532 if (vm_swappin_avail)
533 vm_swappin_enabled = TRUE;
534 #endif
535 vm_swapfile_close((uint64_t)pathname, vp);
536 }
537 kfree(pathname, namelen);
538
539 compaction_swapper_inited = 1;
540 }
541 lck_mtx_unlock(&vm_swap_data_lock);
542 }
543
544
545
546 void
547 vm_swap_consider_defragmenting()
548 {
549 if (compressor_store_stop_compaction == FALSE && !VM_SWAP_BUSY() &&
550 (VM_SWAP_SHOULD_DEFRAGMENT() || VM_SWAP_SHOULD_RECLAIM())) {
551
552 if (!vm_swapfile_gc_thread_running) {
553 lck_mtx_lock(&vm_swap_data_lock);
554
555 if (!vm_swapfile_gc_thread_running)
556 thread_wakeup((event_t) &vm_swapfile_gc_needed);
557
558 lck_mtx_unlock(&vm_swap_data_lock);
559 }
560 }
561 }
562
563
564 int vm_swap_defragment_yielded = 0;
565 int vm_swap_defragment_swapin = 0;
566 int vm_swap_defragment_free = 0;
567 int vm_swap_defragment_busy = 0;
568
569
570 static void
571 vm_swap_defragment()
572 {
573 c_segment_t c_seg;
574
575 /*
576 * have to grab the master lock w/o holding
577 * any locks in spin mode
578 */
579 PAGE_REPLACEMENT_DISALLOWED(TRUE);
580
581 lck_mtx_lock_spin_always(c_list_lock);
582
583 while (!queue_empty(&c_swappedout_sparse_list_head)) {
584
585 if (compressor_store_stop_compaction == TRUE || VM_SWAP_BUSY()) {
586 vm_swap_defragment_yielded++;
587 break;
588 }
589 c_seg = (c_segment_t)queue_first(&c_swappedout_sparse_list_head);
590
591 lck_mtx_lock_spin_always(&c_seg->c_lock);
592
593 assert(c_seg->c_state == C_ON_SWAPPEDOUTSPARSE_Q);
594
595 if (c_seg->c_busy) {
596 lck_mtx_unlock_always(c_list_lock);
597
598 PAGE_REPLACEMENT_DISALLOWED(FALSE);
599 /*
600 * c_seg_wait_on_busy consumes c_seg->c_lock
601 */
602 c_seg_wait_on_busy(c_seg);
603
604 PAGE_REPLACEMENT_DISALLOWED(TRUE);
605
606 lck_mtx_lock_spin_always(c_list_lock);
607
608 vm_swap_defragment_busy++;
609 continue;
610 }
611 if (c_seg->c_bytes_used == 0) {
612 /*
613 * c_seg_free_locked consumes the c_list_lock
614 * and c_seg->c_lock
615 */
616 C_SEG_BUSY(c_seg);
617 c_seg_free_locked(c_seg);
618
619 vm_swap_defragment_free++;
620 } else {
621 lck_mtx_unlock_always(c_list_lock);
622
623 if (c_seg_swapin(c_seg, TRUE, FALSE) == 0)
624 lck_mtx_unlock_always(&c_seg->c_lock);
625
626 vm_swap_defragment_swapin++;
627 }
628 PAGE_REPLACEMENT_DISALLOWED(FALSE);
629
630 vm_pageout_io_throttle();
631
632 /*
633 * because write waiters have privilege over readers,
634 * dropping and immediately retaking the master lock will
635 * still allow any thread waiting to acquire the
636 * master lock exclusively an opportunity to take it
637 */
638 PAGE_REPLACEMENT_DISALLOWED(TRUE);
639
640 lck_mtx_lock_spin_always(c_list_lock);
641 }
642 lck_mtx_unlock_always(c_list_lock);
643
644 PAGE_REPLACEMENT_DISALLOWED(FALSE);
645 }
646
647
648
649 static void
650 vm_swapfile_create_thread(void)
651 {
652 clock_sec_t sec;
653 clock_nsec_t nsec;
654
655 current_thread()->options |= TH_OPT_VMPRIV;
656
657 vm_swapfile_create_thread_awakened++;
658 vm_swapfile_create_thread_running = 1;
659
660 while (TRUE) {
661 /*
662 * walk through the list of swap files
663 * and do the delayed frees/trims for
664 * any swap file whose count of delayed
665 * frees is above the batch limit
666 */
667 vm_swap_handle_delayed_trims(FALSE);
668
669 lck_mtx_lock(&vm_swap_data_lock);
670
671 if (hibernate_in_progress_with_pinned_swap == TRUE)
672 break;
673
674 clock_get_system_nanotime(&sec, &nsec);
675
676 if (VM_SWAP_SHOULD_CREATE(sec) == 0)
677 break;
678
679 lck_mtx_unlock(&vm_swap_data_lock);
680
681 if (vm_swap_create_file() == FALSE) {
682 vm_swapfile_last_failed_to_create_ts = sec;
683 HIBLOG("vm_swap_create_file failed @ %lu secs\n", (unsigned long)sec);
684
685 } else
686 vm_swapfile_last_successful_create_ts = sec;
687 }
688 vm_swapfile_create_thread_running = 0;
689
690 if (hibernate_in_progress_with_pinned_swap == TRUE)
691 thread_wakeup((event_t)&hibernate_in_progress_with_pinned_swap);
692
693 assert_wait((event_t)&vm_swapfile_create_needed, THREAD_UNINT);
694
695 lck_mtx_unlock(&vm_swap_data_lock);
696
697 thread_block((thread_continue_t)vm_swapfile_create_thread);
698
699 /* NOTREACHED */
700 }
701
702
703 #if HIBERNATION
704
705 kern_return_t
706 hibernate_pin_swap(boolean_t start)
707 {
708 vm_compaction_swapper_do_init();
709
710 if (start == FALSE) {
711
712 lck_mtx_lock(&vm_swap_data_lock);
713 hibernate_in_progress_with_pinned_swap = FALSE;
714 lck_mtx_unlock(&vm_swap_data_lock);
715
716 return (KERN_SUCCESS);
717 }
718 if (vm_swappin_enabled == FALSE)
719 return (KERN_SUCCESS);
720
721 lck_mtx_lock(&vm_swap_data_lock);
722
723 hibernate_in_progress_with_pinned_swap = TRUE;
724
725 while (vm_swapfile_create_thread_running || vm_swapfile_gc_thread_running) {
726
727 assert_wait((event_t)&hibernate_in_progress_with_pinned_swap, THREAD_UNINT);
728
729 lck_mtx_unlock(&vm_swap_data_lock);
730
731 thread_block(THREAD_CONTINUE_NULL);
732
733 lck_mtx_lock(&vm_swap_data_lock);
734 }
735 if (vm_num_swap_files > vm_num_pinned_swap_files) {
736 hibernate_in_progress_with_pinned_swap = FALSE;
737 lck_mtx_unlock(&vm_swap_data_lock);
738
739 HIBLOG("hibernate_pin_swap failed - vm_num_swap_files = %d, vm_num_pinned_swap_files = %d\n",
740 vm_num_swap_files, vm_num_pinned_swap_files);
741 return (KERN_FAILURE);
742 }
743 lck_mtx_unlock(&vm_swap_data_lock);
744
745 while (VM_SWAP_SHOULD_PIN(MAX_SWAP_FILE_SIZE)) {
746 if (vm_swap_create_file() == FALSE)
747 break;
748 }
749 return (KERN_SUCCESS);
750 }
751 #endif
752
753 static void
754 vm_swapfile_gc_thread(void)
755
756 {
757 boolean_t need_defragment;
758 boolean_t need_reclaim;
759
760 vm_swapfile_gc_thread_awakened++;
761 vm_swapfile_gc_thread_running = 1;
762
763 while (TRUE) {
764
765 lck_mtx_lock(&vm_swap_data_lock);
766
767 if (hibernate_in_progress_with_pinned_swap == TRUE)
768 break;
769
770 if (VM_SWAP_BUSY() || compressor_store_stop_compaction == TRUE)
771 break;
772
773 need_defragment = FALSE;
774 need_reclaim = FALSE;
775
776 if (VM_SWAP_SHOULD_DEFRAGMENT())
777 need_defragment = TRUE;
778
779 if (VM_SWAP_SHOULD_RECLAIM()) {
780 need_defragment = TRUE;
781 need_reclaim = TRUE;
782 }
783 if (need_defragment == FALSE && need_reclaim == FALSE)
784 break;
785
786 lck_mtx_unlock(&vm_swap_data_lock);
787
788 if (need_defragment == TRUE)
789 vm_swap_defragment();
790 if (need_reclaim == TRUE)
791 vm_swap_reclaim();
792 }
793 vm_swapfile_gc_thread_running = 0;
794
795 if (hibernate_in_progress_with_pinned_swap == TRUE)
796 thread_wakeup((event_t)&hibernate_in_progress_with_pinned_swap);
797
798 assert_wait((event_t)&vm_swapfile_gc_needed, THREAD_UNINT);
799
800 lck_mtx_unlock(&vm_swap_data_lock);
801
802 thread_block((thread_continue_t)vm_swapfile_gc_thread);
803
804 /* NOTREACHED */
805 }
806
807
808
809 int swapper_entered_T0 = 0;
810 int swapper_entered_T1 = 0;
811 int swapper_entered_T2 = 0;
812
813 static void
814 vm_swapout_thread_throttle_adjust(void)
815 {
816 int swapper_throttle_new;
817
818 if (swapper_throttle_inited == FALSE) {
819 /*
820 * force this thread to be set to the correct
821 * throttling tier
822 */
823 swapper_throttle_new = THROTTLE_LEVEL_COMPRESSOR_TIER2;
824 swapper_throttle = THROTTLE_LEVEL_COMPRESSOR_TIER1;
825 swapper_throttle_inited = TRUE;
826 swapper_entered_T2++;
827 goto done;
828 }
829 swapper_throttle_new = swapper_throttle;
830
831
832 switch(swapper_throttle) {
833
834 case THROTTLE_LEVEL_COMPRESSOR_TIER2:
835
836 if (SWAPPER_NEEDS_TO_UNTHROTTLE() || swapout_target_age || hibernate_flushing == TRUE) {
837 swapper_throttle_new = THROTTLE_LEVEL_COMPRESSOR_TIER1;
838 swapper_entered_T1++;
839 break;
840 }
841 break;
842
843 case THROTTLE_LEVEL_COMPRESSOR_TIER1:
844
845 if (VM_PAGEOUT_SCAN_NEEDS_TO_THROTTLE()) {
846 swapper_throttle_new = THROTTLE_LEVEL_COMPRESSOR_TIER0;
847 swapper_entered_T0++;
848 break;
849 }
850 if (COMPRESSOR_NEEDS_TO_SWAP() == 0 && swapout_target_age == 0 && hibernate_flushing == FALSE) {
851 swapper_throttle_new = THROTTLE_LEVEL_COMPRESSOR_TIER2;
852 swapper_entered_T2++;
853 break;
854 }
855 break;
856
857 case THROTTLE_LEVEL_COMPRESSOR_TIER0:
858
859 if (COMPRESSOR_NEEDS_TO_SWAP() == 0) {
860 swapper_throttle_new = THROTTLE_LEVEL_COMPRESSOR_TIER2;
861 swapper_entered_T2++;
862 break;
863 }
864 if (SWAPPER_NEEDS_TO_UNTHROTTLE() == 0) {
865 swapper_throttle_new = THROTTLE_LEVEL_COMPRESSOR_TIER1;
866 swapper_entered_T1++;
867 break;
868 }
869 break;
870 }
871 done:
872 if (swapper_throttle != swapper_throttle_new) {
873 proc_set_thread_policy_with_tid(kernel_task, vm_swapout_thread_id,
874 TASK_POLICY_INTERNAL, TASK_POLICY_IO, swapper_throttle_new);
875 proc_set_thread_policy_with_tid(kernel_task, vm_swapout_thread_id,
876 TASK_POLICY_INTERNAL, TASK_POLICY_PASSIVE_IO, TASK_POLICY_ENABLE);
877
878 swapper_throttle = swapper_throttle_new;
879 }
880 }
881
882
883 int vm_swapout_found_empty = 0;
884
885 static void
886 vm_swapout_thread(void)
887 {
888 uint64_t f_offset = 0;
889 uint32_t size = 0;
890 c_segment_t c_seg = NULL;
891 kern_return_t kr = KERN_SUCCESS;
892 vm_offset_t addr = 0;
893
894 current_thread()->options |= TH_OPT_VMPRIV;
895
896 vm_swapout_thread_awakened++;
897
898 lck_mtx_lock_spin_always(c_list_lock);
899
900 while (!queue_empty(&c_swapout_list_head)) {
901
902 c_seg = (c_segment_t)queue_first(&c_swapout_list_head);
903
904 lck_mtx_lock_spin_always(&c_seg->c_lock);
905
906 assert(c_seg->c_state == C_ON_SWAPOUT_Q);
907
908 if (c_seg->c_busy) {
909 lck_mtx_unlock_always(c_list_lock);
910
911 c_seg_wait_on_busy(c_seg);
912
913 lck_mtx_lock_spin_always(c_list_lock);
914
915 continue;
916 }
917 vm_swapout_thread_processed_segments++;
918
919 size = round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset));
920
921 if (size == 0) {
922 assert(c_seg->c_bytes_used == 0);
923
924 if (!c_seg->c_on_minorcompact_q)
925 c_seg_need_delayed_compaction(c_seg, TRUE);
926
927 c_seg_switch_state(c_seg, C_IS_EMPTY, FALSE);
928 lck_mtx_unlock_always(&c_seg->c_lock);
929 lck_mtx_unlock_always(c_list_lock);
930
931 vm_swapout_found_empty++;
932 goto c_seg_is_empty;
933 }
934 C_SEG_BUSY(c_seg);
935 c_seg->c_busy_swapping = 1;
936
937 lck_mtx_unlock_always(c_list_lock);
938
939 addr = (vm_offset_t) c_seg->c_store.c_buffer;
940
941 lck_mtx_unlock_always(&c_seg->c_lock);
942
943 #if CHECKSUM_THE_SWAP
944 c_seg->cseg_hash = hash_string((char*)addr, (int)size);
945 c_seg->cseg_swap_size = size;
946 #endif /* CHECKSUM_THE_SWAP */
947
948 #if ENCRYPTED_SWAP
949 vm_swap_encrypt(c_seg);
950 #endif /* ENCRYPTED_SWAP */
951
952 vm_swapout_thread_throttle_adjust();
953
954 kr = vm_swap_put((vm_offset_t) addr, &f_offset, size, c_seg);
955
956 PAGE_REPLACEMENT_DISALLOWED(TRUE);
957
958 if (kr == KERN_SUCCESS) {
959 kernel_memory_depopulate(compressor_map, (vm_offset_t) addr, size, KMA_COMPRESSOR);
960 }
961 #if ENCRYPTED_SWAP
962 else {
963 vm_swap_decrypt(c_seg);
964 }
965 #endif /* ENCRYPTED_SWAP */
966 lck_mtx_lock_spin_always(c_list_lock);
967 lck_mtx_lock_spin_always(&c_seg->c_lock);
968
969 if (kr == KERN_SUCCESS) {
970 int new_state = C_ON_SWAPPEDOUT_Q;
971 boolean_t insert_head = FALSE;
972
973 if (hibernate_flushing == TRUE) {
974 if (c_seg->c_generation_id >= first_c_segment_to_warm_generation_id &&
975 c_seg->c_generation_id <= last_c_segment_to_warm_generation_id)
976 insert_head = TRUE;
977 } else if (C_SEG_ONDISK_IS_SPARSE(c_seg))
978 new_state = C_ON_SWAPPEDOUTSPARSE_Q;
979
980 c_seg_switch_state(c_seg, new_state, insert_head);
981
982 c_seg->c_store.c_swap_handle = f_offset;
983
984 VM_STAT_INCR_BY(swapouts, size >> PAGE_SHIFT);
985
986 if (c_seg->c_bytes_used)
987 OSAddAtomic64(-c_seg->c_bytes_used, &compressor_bytes_used);
988 } else {
989 if (c_seg->c_overage_swap == TRUE) {
990 c_seg->c_overage_swap = FALSE;
991 c_overage_swapped_count--;
992 }
993 c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
994
995 if (!c_seg->c_on_minorcompact_q && C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE)
996 c_seg_need_delayed_compaction(c_seg, TRUE);
997 }
998 assert(c_seg->c_busy_swapping);
999 assert(c_seg->c_busy);
1000
1001 c_seg->c_busy_swapping = 0;
1002 lck_mtx_unlock_always(c_list_lock);
1003
1004 C_SEG_WAKEUP_DONE(c_seg);
1005 lck_mtx_unlock_always(&c_seg->c_lock);
1006
1007 PAGE_REPLACEMENT_DISALLOWED(FALSE);
1008
1009 vm_pageout_io_throttle();
1010 c_seg_is_empty:
1011 if (c_swapout_count == 0)
1012 vm_swap_consider_defragmenting();
1013
1014 lck_mtx_lock_spin_always(c_list_lock);
1015 }
1016
1017 assert_wait((event_t)&c_swapout_list_head, THREAD_UNINT);
1018
1019 lck_mtx_unlock_always(c_list_lock);
1020
1021 thread_block((thread_continue_t)vm_swapout_thread);
1022
1023 /* NOTREACHED */
1024 }
1025
1026 boolean_t
1027 vm_swap_create_file()
1028 {
1029 uint64_t size = 0;
1030 int namelen = 0;
1031 boolean_t swap_file_created = FALSE;
1032 boolean_t swap_file_reuse = FALSE;
1033 boolean_t swap_file_pin = FALSE;
1034 struct swapfile *swf = NULL;
1035
1036 /*
1037 * make sure we've got all the info we need
1038 * to potentially pin a swap file... we could
1039 * be swapping out due to hibernation w/o ever
1040 * having run vm_pageout_scan, which is normally
1041 * the trigger to do the init
1042 */
1043 vm_compaction_swapper_do_init();
1044
1045 /*
1046 * Any swapfile structure ready for re-use?
1047 */
1048
1049 lck_mtx_lock(&vm_swap_data_lock);
1050
1051 swf = (struct swapfile*) queue_first(&swf_global_queue);
1052
1053 while (queue_end(&swf_global_queue, (queue_entry_t)swf) == FALSE) {
1054 if (swf->swp_flags == SWAP_REUSE) {
1055 swap_file_reuse = TRUE;
1056 break;
1057 }
1058 swf = (struct swapfile*) queue_next(&swf->swp_queue);
1059 }
1060
1061 lck_mtx_unlock(&vm_swap_data_lock);
1062
1063 if (swap_file_reuse == FALSE) {
1064
1065 namelen = (int)strlen(swapfilename) + SWAPFILENAME_INDEX_LEN + 1;
1066
1067 swf = (struct swapfile*) kalloc(sizeof *swf);
1068 memset(swf, 0, sizeof(*swf));
1069
1070 swf->swp_index = vm_num_swap_files + 1;
1071 swf->swp_pathlen = namelen;
1072 swf->swp_path = (char*)kalloc(swf->swp_pathlen);
1073
1074 memset(swf->swp_path, 0, namelen);
1075
1076 snprintf(swf->swp_path, namelen, "%s%d", swapfilename, vm_num_swap_files);
1077 }
1078
1079 vm_swapfile_open(swf->swp_path, &swf->swp_vp);
1080
1081 if (swf->swp_vp == NULL) {
1082 if (swap_file_reuse == FALSE) {
1083 kfree(swf->swp_path, swf->swp_pathlen);
1084 kfree(swf, sizeof *swf);
1085 }
1086 return FALSE;
1087 }
1088 vm_swapfile_can_be_created = TRUE;
1089
1090 size = MAX_SWAP_FILE_SIZE;
1091
1092 while (size >= MIN_SWAP_FILE_SIZE) {
1093
1094 swap_file_pin = VM_SWAP_SHOULD_PIN(size);
1095
1096 if (vm_swapfile_preallocate(swf->swp_vp, &size, &swap_file_pin) == 0) {
1097
1098 int num_bytes_for_bitmap = 0;
1099
1100 swap_file_created = TRUE;
1101
1102 swf->swp_size = size;
1103 swf->swp_nsegs = (unsigned int) (size / COMPRESSED_SWAP_CHUNK_SIZE);
1104 swf->swp_nseginuse = 0;
1105 swf->swp_free_hint = 0;
1106
1107 num_bytes_for_bitmap = MAX((swf->swp_nsegs >> 3) , 1);
1108 /*
1109 * Allocate a bitmap that describes the
1110 * number of segments held by this swapfile.
1111 */
1112 swf->swp_bitmap = (uint8_t*)kalloc(num_bytes_for_bitmap);
1113 memset(swf->swp_bitmap, 0, num_bytes_for_bitmap);
1114
1115 swf->swp_csegs = (c_segment_t *) kalloc(swf->swp_nsegs * sizeof(c_segment_t));
1116 memset(swf->swp_csegs, 0, (swf->swp_nsegs * sizeof(c_segment_t)));
1117
1118 /*
1119 * passing a NULL trim_list into vnode_trim_list
1120 * will return ENOTSUP if trim isn't supported
1121 * and 0 if it is
1122 */
1123 if (vnode_trim_list(swf->swp_vp, NULL, FALSE) == 0)
1124 swp_trim_supported = TRUE;
1125
1126 lck_mtx_lock(&vm_swap_data_lock);
1127
1128 swf->swp_flags = SWAP_READY;
1129
1130 if (swap_file_reuse == FALSE) {
1131 queue_enter(&swf_global_queue, swf, struct swapfile*, swp_queue);
1132 }
1133
1134 vm_num_swap_files++;
1135
1136 vm_swapfile_total_segs_alloced += swf->swp_nsegs;
1137
1138 if (swap_file_pin == TRUE) {
1139 vm_num_pinned_swap_files++;
1140 swf->swp_flags |= SWAP_PINNED;
1141 vm_swappin_avail -= swf->swp_size;
1142 }
1143
1144 lck_mtx_unlock(&vm_swap_data_lock);
1145
1146 thread_wakeup((event_t) &vm_num_swap_files);
1147 #if CONFIG_EMBEDDED
1148 if (vm_num_swap_files == 1) {
1149
1150 c_overage_swapped_limit = (uint32_t)size / C_SEG_BUFSIZE;
1151
1152 if (VM_CONFIG_FREEZER_SWAP_IS_ACTIVE)
1153 c_overage_swapped_limit /= 2;
1154 }
1155 #endif
1156 break;
1157 } else {
1158
1159 size = size / 2;
1160 }
1161 }
1162 if (swap_file_created == FALSE) {
1163
1164 vm_swapfile_close((uint64_t)(swf->swp_path), swf->swp_vp);
1165
1166 swf->swp_vp = NULL;
1167
1168 if (swap_file_reuse == FALSE) {
1169 kfree(swf->swp_path, swf->swp_pathlen);
1170 kfree(swf, sizeof *swf);
1171 }
1172 }
1173 return swap_file_created;
1174 }
1175
1176
1177 kern_return_t
1178 vm_swap_get(c_segment_t c_seg, uint64_t f_offset, uint64_t size)
1179 {
1180 struct swapfile *swf = NULL;
1181 uint64_t file_offset = 0;
1182 int retval = 0;
1183
1184 assert(c_seg->c_store.c_buffer);
1185
1186 lck_mtx_lock(&vm_swap_data_lock);
1187
1188 swf = vm_swapfile_for_handle(f_offset);
1189
1190 if (swf == NULL || ( !(swf->swp_flags & SWAP_READY) && !(swf->swp_flags & SWAP_RECLAIM))) {
1191 retval = 1;
1192 goto done;
1193 }
1194 swf->swp_io_count++;
1195
1196 lck_mtx_unlock(&vm_swap_data_lock);
1197
1198 #if DEVELOPMENT || DEBUG
1199 C_SEG_MAKE_WRITEABLE(c_seg);
1200 #endif
1201 file_offset = (f_offset & SWAP_SLOT_MASK);
1202 retval = vm_swapfile_io(swf->swp_vp, file_offset, (uint64_t)c_seg->c_store.c_buffer, (int)(size / PAGE_SIZE_64), SWAP_READ);
1203
1204 #if DEVELOPMENT || DEBUG
1205 C_SEG_WRITE_PROTECT(c_seg);
1206 #endif
1207 if (retval == 0)
1208 VM_STAT_INCR_BY(swapins, size >> PAGE_SHIFT);
1209 else
1210 vm_swap_get_failures++;
1211
1212 /*
1213 * Free this slot in the swap structure.
1214 */
1215 vm_swap_free(f_offset);
1216
1217 lck_mtx_lock(&vm_swap_data_lock);
1218 swf->swp_io_count--;
1219
1220 if ((swf->swp_flags & SWAP_WANTED) && swf->swp_io_count == 0) {
1221
1222 swf->swp_flags &= ~SWAP_WANTED;
1223 thread_wakeup((event_t) &swf->swp_flags);
1224 }
1225 done:
1226 lck_mtx_unlock(&vm_swap_data_lock);
1227
1228 if (retval == 0)
1229 return KERN_SUCCESS;
1230 else
1231 return KERN_FAILURE;
1232 }
1233
1234 kern_return_t
1235 vm_swap_put(vm_offset_t addr, uint64_t *f_offset, uint64_t size, c_segment_t c_seg)
1236 {
1237 unsigned int segidx = 0;
1238 struct swapfile *swf = NULL;
1239 uint64_t file_offset = 0;
1240 uint64_t swapfile_index = 0;
1241 unsigned int byte_for_segidx = 0;
1242 unsigned int offset_within_byte = 0;
1243 boolean_t swf_eligible = FALSE;
1244 boolean_t waiting = FALSE;
1245 boolean_t retried = FALSE;
1246 int error = 0;
1247 clock_sec_t sec;
1248 clock_nsec_t nsec;
1249
1250 if (addr == 0 || f_offset == NULL) {
1251 return KERN_FAILURE;
1252 }
1253 retry:
1254 lck_mtx_lock(&vm_swap_data_lock);
1255
1256 swf = (struct swapfile*) queue_first(&swf_global_queue);
1257
1258 while(queue_end(&swf_global_queue, (queue_entry_t)swf) == FALSE) {
1259
1260 segidx = swf->swp_free_hint;
1261
1262 swf_eligible = (swf->swp_flags & SWAP_READY) && (swf->swp_nseginuse < swf->swp_nsegs);
1263
1264 if (swf_eligible) {
1265
1266 while(segidx < swf->swp_nsegs) {
1267
1268 byte_for_segidx = segidx >> 3;
1269 offset_within_byte = segidx % 8;
1270
1271 if ((swf->swp_bitmap)[byte_for_segidx] & (1 << offset_within_byte)) {
1272 segidx++;
1273 continue;
1274 }
1275
1276 (swf->swp_bitmap)[byte_for_segidx] |= (1 << offset_within_byte);
1277
1278 file_offset = segidx * COMPRESSED_SWAP_CHUNK_SIZE;
1279 swf->swp_nseginuse++;
1280 swf->swp_io_count++;
1281 swapfile_index = swf->swp_index;
1282
1283 vm_swapfile_total_segs_used++;
1284
1285 clock_get_system_nanotime(&sec, &nsec);
1286
1287 if (VM_SWAP_SHOULD_CREATE(sec) && !vm_swapfile_create_thread_running)
1288 thread_wakeup((event_t) &vm_swapfile_create_needed);
1289
1290 lck_mtx_unlock(&vm_swap_data_lock);
1291
1292 goto done;
1293 }
1294 }
1295 swf = (struct swapfile*) queue_next(&swf->swp_queue);
1296 }
1297 assert(queue_end(&swf_global_queue, (queue_entry_t) swf));
1298
1299 /*
1300 * we've run out of swap segments, but may not
1301 * be in a position to immediately create a new swap
1302 * file if we've recently failed to create due to a lack
1303 * of free space in the root filesystem... we'll try
1304 * to kick that create off, but in any event we're going
1305 * to take a breather (up to 1 second) so that we're not caught in a tight
1306 * loop back in "vm_compressor_compact_and_swap" trying to stuff
1307 * segments into swap files only to have them immediately put back
1308 * on the c_age queue due to vm_swap_put failing.
1309 *
1310 * if we're doing these puts due to a hibernation flush,
1311 * no need to block... setting hibernate_no_swapspace to TRUE,
1312 * will cause "vm_compressor_compact_and_swap" to immediately abort
1313 */
1314 clock_get_system_nanotime(&sec, &nsec);
1315
1316 if (VM_SWAP_SHOULD_CREATE(sec) && !vm_swapfile_create_thread_running)
1317 thread_wakeup((event_t) &vm_swapfile_create_needed);
1318
1319 if (hibernate_flushing == FALSE || VM_SWAP_SHOULD_CREATE(sec)) {
1320 waiting = TRUE;
1321 assert_wait_timeout((event_t) &vm_num_swap_files, THREAD_INTERRUPTIBLE, 1000, 1000*NSEC_PER_USEC);
1322 } else
1323 hibernate_no_swapspace = TRUE;
1324
1325 lck_mtx_unlock(&vm_swap_data_lock);
1326
1327 if (waiting == TRUE) {
1328 thread_block(THREAD_CONTINUE_NULL);
1329
1330 if (retried == FALSE && hibernate_flushing == TRUE) {
1331 retried = TRUE;
1332 goto retry;
1333 }
1334 }
1335 vm_swap_put_failures++;
1336
1337 return KERN_FAILURE;
1338
1339 done:
1340 assert(c_seg->c_busy_swapping);
1341 assert(c_seg->c_busy);
1342 assert(!c_seg->c_on_minorcompact_q);
1343
1344 error = vm_swapfile_io(swf->swp_vp, file_offset, addr, (int) (size / PAGE_SIZE_64), SWAP_WRITE);
1345
1346 lck_mtx_lock(&vm_swap_data_lock);
1347
1348 swf->swp_csegs[segidx] = c_seg;
1349
1350 swf->swp_io_count--;
1351
1352 *f_offset = (swapfile_index << SWAP_DEVICE_SHIFT) | file_offset;
1353
1354 if ((swf->swp_flags & SWAP_WANTED) && swf->swp_io_count == 0) {
1355
1356 swf->swp_flags &= ~SWAP_WANTED;
1357 thread_wakeup((event_t) &swf->swp_flags);
1358 }
1359
1360 lck_mtx_unlock(&vm_swap_data_lock);
1361
1362 if (error) {
1363 vm_swap_free(*f_offset);
1364
1365 vm_swap_put_failures++;
1366
1367 return KERN_FAILURE;
1368 }
1369 return KERN_SUCCESS;
1370 }
1371
1372
1373
1374 static void
1375 vm_swap_free_now(struct swapfile *swf, uint64_t f_offset)
1376 {
1377 uint64_t file_offset = 0;
1378 unsigned int segidx = 0;
1379
1380
1381 if ((swf->swp_flags & SWAP_READY) || (swf->swp_flags & SWAP_RECLAIM)) {
1382
1383 unsigned int byte_for_segidx = 0;
1384 unsigned int offset_within_byte = 0;
1385
1386 file_offset = (f_offset & SWAP_SLOT_MASK);
1387 segidx = (unsigned int) (file_offset / COMPRESSED_SWAP_CHUNK_SIZE);
1388
1389 byte_for_segidx = segidx >> 3;
1390 offset_within_byte = segidx % 8;
1391
1392 if ((swf->swp_bitmap)[byte_for_segidx] & (1 << offset_within_byte)) {
1393
1394 (swf->swp_bitmap)[byte_for_segidx] &= ~(1 << offset_within_byte);
1395
1396 swf->swp_csegs[segidx] = NULL;
1397
1398 swf->swp_nseginuse--;
1399 vm_swapfile_total_segs_used--;
1400
1401 if (segidx < swf->swp_free_hint) {
1402 swf->swp_free_hint = segidx;
1403 }
1404 }
1405 if (VM_SWAP_SHOULD_RECLAIM() && !vm_swapfile_gc_thread_running)
1406 thread_wakeup((event_t) &vm_swapfile_gc_needed);
1407 }
1408 }
1409
1410
1411 uint32_t vm_swap_free_now_count = 0;
1412 uint32_t vm_swap_free_delayed_count = 0;
1413
1414
1415 void
1416 vm_swap_free(uint64_t f_offset)
1417 {
1418 struct swapfile *swf = NULL;
1419 struct trim_list *tl = NULL;
1420 clock_sec_t sec;
1421 clock_nsec_t nsec;
1422
1423 if (swp_trim_supported == TRUE)
1424 tl = kalloc(sizeof(struct trim_list));
1425
1426 lck_mtx_lock(&vm_swap_data_lock);
1427
1428 swf = vm_swapfile_for_handle(f_offset);
1429
1430 if (swf && (swf->swp_flags & (SWAP_READY | SWAP_RECLAIM))) {
1431
1432 if (swp_trim_supported == FALSE || (swf->swp_flags & SWAP_RECLAIM)) {
1433 /*
1434 * don't delay the free if the underlying disk doesn't support
1435 * trim, or we're in the midst of reclaiming this swap file since
1436 * we don't want to move segments that are technically free
1437 * but not yet handled by the delayed free mechanism
1438 */
1439 vm_swap_free_now(swf, f_offset);
1440
1441 vm_swap_free_now_count++;
1442 goto done;
1443 }
1444 tl->tl_offset = f_offset & SWAP_SLOT_MASK;
1445 tl->tl_length = COMPRESSED_SWAP_CHUNK_SIZE;
1446
1447 tl->tl_next = swf->swp_delayed_trim_list_head;
1448 swf->swp_delayed_trim_list_head = tl;
1449 swf->swp_delayed_trim_count++;
1450 tl = NULL;
1451
1452 if (VM_SWAP_SHOULD_TRIM(swf) && !vm_swapfile_create_thread_running) {
1453 clock_get_system_nanotime(&sec, &nsec);
1454
1455 if (sec > dont_trim_until_ts)
1456 thread_wakeup((event_t) &vm_swapfile_create_needed);
1457 }
1458 vm_swap_free_delayed_count++;
1459 }
1460 done:
1461 lck_mtx_unlock(&vm_swap_data_lock);
1462
1463 if (tl != NULL)
1464 kfree(tl, sizeof(struct trim_list));
1465 }
1466
1467
1468 static void
1469 vm_swap_wait_on_trim_handling_in_progress()
1470 {
1471 while (delayed_trim_handling_in_progress == TRUE) {
1472
1473 assert_wait((event_t) &delayed_trim_handling_in_progress, THREAD_UNINT);
1474 lck_mtx_unlock(&vm_swap_data_lock);
1475
1476 thread_block(THREAD_CONTINUE_NULL);
1477
1478 lck_mtx_lock(&vm_swap_data_lock);
1479 }
1480 }
1481
1482
1483 static void
1484 vm_swap_handle_delayed_trims(boolean_t force_now)
1485 {
1486 struct swapfile *swf = NULL;
1487
1488 /*
1489 * serialize the race between us and vm_swap_reclaim...
1490 * if vm_swap_reclaim wins it will turn off SWAP_READY
1491 * on the victim it has chosen... we can just skip over
1492 * that file since vm_swap_reclaim will first process
1493 * all of the delayed trims associated with it
1494 */
1495 lck_mtx_lock(&vm_swap_data_lock);
1496
1497 delayed_trim_handling_in_progress = TRUE;
1498
1499 lck_mtx_unlock(&vm_swap_data_lock);
1500
1501 /*
1502 * no need to hold the lock to walk the swf list since
1503 * vm_swap_create (the only place where we add to this list)
1504 * is run on the same thread as this function
1505 * and vm_swap_reclaim doesn't remove items from this list
1506 * instead marking them with SWAP_REUSE for future re-use
1507 */
1508 swf = (struct swapfile*) queue_first(&swf_global_queue);
1509
1510 while (queue_end(&swf_global_queue, (queue_entry_t)swf) == FALSE) {
1511
1512 if ((swf->swp_flags & SWAP_READY) && (force_now == TRUE || VM_SWAP_SHOULD_TRIM(swf))) {
1513
1514 assert(!(swf->swp_flags & SWAP_RECLAIM));
1515 vm_swap_do_delayed_trim(swf);
1516 }
1517 swf = (struct swapfile*) queue_next(&swf->swp_queue);
1518 }
1519 lck_mtx_lock(&vm_swap_data_lock);
1520
1521 delayed_trim_handling_in_progress = FALSE;
1522 thread_wakeup((event_t) &delayed_trim_handling_in_progress);
1523
1524 if (VM_SWAP_SHOULD_RECLAIM() && !vm_swapfile_gc_thread_running)
1525 thread_wakeup((event_t) &vm_swapfile_gc_needed);
1526
1527 lck_mtx_unlock(&vm_swap_data_lock);
1528
1529 }
1530
1531 static void
1532 vm_swap_do_delayed_trim(struct swapfile *swf)
1533 {
1534 struct trim_list *tl, *tl_head;
1535
1536 lck_mtx_lock(&vm_swap_data_lock);
1537
1538 tl_head = swf->swp_delayed_trim_list_head;
1539 swf->swp_delayed_trim_list_head = NULL;
1540 swf->swp_delayed_trim_count = 0;
1541
1542 lck_mtx_unlock(&vm_swap_data_lock);
1543
1544 vnode_trim_list(swf->swp_vp, tl_head, TRUE);
1545
1546 while ((tl = tl_head) != NULL) {
1547 unsigned int segidx = 0;
1548 unsigned int byte_for_segidx = 0;
1549 unsigned int offset_within_byte = 0;
1550
1551 lck_mtx_lock(&vm_swap_data_lock);
1552
1553 segidx = (unsigned int) (tl->tl_offset / COMPRESSED_SWAP_CHUNK_SIZE);
1554
1555 byte_for_segidx = segidx >> 3;
1556 offset_within_byte = segidx % 8;
1557
1558 if ((swf->swp_bitmap)[byte_for_segidx] & (1 << offset_within_byte)) {
1559
1560 (swf->swp_bitmap)[byte_for_segidx] &= ~(1 << offset_within_byte);
1561
1562 swf->swp_csegs[segidx] = NULL;
1563
1564 swf->swp_nseginuse--;
1565 vm_swapfile_total_segs_used--;
1566
1567 if (segidx < swf->swp_free_hint) {
1568 swf->swp_free_hint = segidx;
1569 }
1570 }
1571 lck_mtx_unlock(&vm_swap_data_lock);
1572
1573 tl_head = tl->tl_next;
1574
1575 kfree(tl, sizeof(struct trim_list));
1576 }
1577 }
1578
1579
1580 void
1581 vm_swap_flush()
1582 {
1583 return;
1584 }
1585
1586 int vm_swap_reclaim_yielded = 0;
1587
1588 void
1589 vm_swap_reclaim(void)
1590 {
1591 vm_offset_t addr = 0;
1592 unsigned int segidx = 0;
1593 uint64_t f_offset = 0;
1594 struct swapfile *swf = NULL;
1595 struct swapfile *smallest_swf = NULL;
1596 unsigned int min_nsegs = 0;
1597 unsigned int byte_for_segidx = 0;
1598 unsigned int offset_within_byte = 0;
1599 uint32_t c_size = 0;
1600
1601 c_segment_t c_seg = NULL;
1602
1603 if (kernel_memory_allocate(compressor_map, (vm_offset_t *)(&addr), C_SEG_BUFSIZE, 0, KMA_KOBJECT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS) {
1604 panic("vm_swap_reclaim: kernel_memory_allocate failed\n");
1605 }
1606
1607 lck_mtx_lock(&vm_swap_data_lock);
1608
1609 /*
1610 * if we're running the swapfile list looking for
1611 * candidates with delayed trims, we need to
1612 * wait before making our decision concerning
1613 * the swapfile we want to reclaim
1614 */
1615 vm_swap_wait_on_trim_handling_in_progress();
1616
1617 /*
1618 * from here until we knock down the SWAP_READY bit,
1619 * we need to remain behind the vm_swap_data_lock...
1620 * once that bit has been turned off, "vm_swap_handle_delayed_trims"
1621 * will not consider this swapfile for processing
1622 */
1623 swf = (struct swapfile*) queue_first(&swf_global_queue);
1624 min_nsegs = MAX_SWAP_FILE_SIZE / COMPRESSED_SWAP_CHUNK_SIZE;
1625 smallest_swf = NULL;
1626
1627 while (queue_end(&swf_global_queue, (queue_entry_t)swf) == FALSE) {
1628
1629 if ((swf->swp_flags & SWAP_READY) && (swf->swp_nseginuse <= min_nsegs)) {
1630
1631 smallest_swf = swf;
1632 min_nsegs = swf->swp_nseginuse;
1633 }
1634 swf = (struct swapfile*) queue_next(&swf->swp_queue);
1635 }
1636
1637 if (smallest_swf == NULL)
1638 goto done;
1639
1640 swf = smallest_swf;
1641
1642
1643 swf->swp_flags &= ~SWAP_READY;
1644 swf->swp_flags |= SWAP_RECLAIM;
1645
1646 if (swf->swp_delayed_trim_count) {
1647
1648 lck_mtx_unlock(&vm_swap_data_lock);
1649
1650 vm_swap_do_delayed_trim(swf);
1651
1652 lck_mtx_lock(&vm_swap_data_lock);
1653 }
1654 segidx = 0;
1655
1656 while (segidx < swf->swp_nsegs) {
1657
1658 ReTry_for_cseg:
1659 /*
1660 * Wait for outgoing I/Os.
1661 */
1662 while (swf->swp_io_count) {
1663
1664 swf->swp_flags |= SWAP_WANTED;
1665
1666 assert_wait((event_t) &swf->swp_flags, THREAD_UNINT);
1667 lck_mtx_unlock(&vm_swap_data_lock);
1668
1669 thread_block(THREAD_CONTINUE_NULL);
1670
1671 lck_mtx_lock(&vm_swap_data_lock);
1672 }
1673 if (compressor_store_stop_compaction == TRUE || VM_SWAP_SHOULD_ABORT_RECLAIM() || VM_SWAP_BUSY()) {
1674 vm_swap_reclaim_yielded++;
1675 break;
1676 }
1677
1678 byte_for_segidx = segidx >> 3;
1679 offset_within_byte = segidx % 8;
1680
1681 if (((swf->swp_bitmap)[byte_for_segidx] & (1 << offset_within_byte)) == 0) {
1682
1683 segidx++;
1684 continue;
1685 }
1686
1687 c_seg = swf->swp_csegs[segidx];
1688 assert(c_seg);
1689
1690 lck_mtx_lock_spin_always(&c_seg->c_lock);
1691
1692 if (c_seg->c_busy) {
1693 /*
1694 * a swapped out c_segment in the process of being freed will remain in the
1695 * busy state until after the vm_swap_free is called on it... vm_swap_free
1696 * takes the vm_swap_data_lock, so can't change the swap state until after
1697 * we drop the vm_swap_data_lock... once we do, vm_swap_free will complete
1698 * which will allow c_seg_free_locked to clear busy and wake up this thread...
1699 * at that point, we re-look up the swap state which will now indicate that
1700 * this c_segment no longer exists.
1701 */
1702 c_seg->c_wanted = 1;
1703
1704 assert_wait((event_t) (c_seg), THREAD_UNINT);
1705 lck_mtx_unlock_always(&c_seg->c_lock);
1706
1707 lck_mtx_unlock(&vm_swap_data_lock);
1708
1709 thread_block(THREAD_CONTINUE_NULL);
1710
1711 lck_mtx_lock(&vm_swap_data_lock);
1712
1713 goto ReTry_for_cseg;
1714 }
1715 (swf->swp_bitmap)[byte_for_segidx] &= ~(1 << offset_within_byte);
1716
1717 f_offset = segidx * COMPRESSED_SWAP_CHUNK_SIZE;
1718
1719 assert(c_seg == swf->swp_csegs[segidx]);
1720 swf->swp_csegs[segidx] = NULL;
1721 swf->swp_nseginuse--;
1722
1723 vm_swapfile_total_segs_used--;
1724
1725 lck_mtx_unlock(&vm_swap_data_lock);
1726
1727 assert(C_SEG_IS_ONDISK(c_seg));
1728
1729 C_SEG_BUSY(c_seg);
1730 c_seg->c_busy_swapping = 1;
1731 #if !CHECKSUM_THE_SWAP
1732 c_seg_trim_tail(c_seg);
1733 #endif
1734 c_size = round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset));
1735
1736 assert(c_size <= C_SEG_BUFSIZE && c_size);
1737
1738 lck_mtx_unlock_always(&c_seg->c_lock);
1739
1740 if (vm_swapfile_io(swf->swp_vp, f_offset, addr, (int)(c_size / PAGE_SIZE_64), SWAP_READ)) {
1741
1742 /*
1743 * reading the data back in failed, so convert c_seg
1744 * to a swapped in c_segment that contains no data
1745 */
1746 c_seg_swapin_requeue(c_seg, FALSE, TRUE, FALSE);
1747 /*
1748 * returns with c_busy_swapping cleared
1749 */
1750
1751 vm_swap_get_failures++;
1752 goto swap_io_failed;
1753 }
1754 VM_STAT_INCR_BY(swapins, c_size >> PAGE_SHIFT);
1755
1756 if (vm_swap_put(addr, &f_offset, c_size, c_seg)) {
1757 vm_offset_t c_buffer;
1758
1759 /*
1760 * the put failed, so convert c_seg to a fully swapped in c_segment
1761 * with valid data
1762 */
1763 c_buffer = (vm_offset_t)C_SEG_BUFFER_ADDRESS(c_seg->c_mysegno);
1764
1765 kernel_memory_populate(compressor_map, c_buffer, c_size, KMA_COMPRESSOR, VM_KERN_MEMORY_COMPRESSOR);
1766
1767 memcpy((char *)c_buffer, (char *)addr, c_size);
1768
1769 c_seg->c_store.c_buffer = (int32_t *)c_buffer;
1770 #if ENCRYPTED_SWAP
1771 vm_swap_decrypt(c_seg);
1772 #endif /* ENCRYPTED_SWAP */
1773 c_seg_swapin_requeue(c_seg, TRUE, TRUE, FALSE);
1774 /*
1775 * returns with c_busy_swapping cleared
1776 */
1777 OSAddAtomic64(c_seg->c_bytes_used, &compressor_bytes_used);
1778
1779 goto swap_io_failed;
1780 }
1781 VM_STAT_INCR_BY(swapouts, c_size >> PAGE_SHIFT);
1782
1783 lck_mtx_lock_spin_always(&c_seg->c_lock);
1784
1785 assert(C_SEG_IS_ONDISK(c_seg));
1786 /*
1787 * The c_seg will now know about the new location on disk.
1788 */
1789 c_seg->c_store.c_swap_handle = f_offset;
1790
1791 assert(c_seg->c_busy_swapping);
1792 c_seg->c_busy_swapping = 0;
1793 swap_io_failed:
1794 assert(c_seg->c_busy);
1795 C_SEG_WAKEUP_DONE(c_seg);
1796
1797 lck_mtx_unlock_always(&c_seg->c_lock);
1798 lck_mtx_lock(&vm_swap_data_lock);
1799 }
1800
1801 if (swf->swp_nseginuse) {
1802
1803 swf->swp_flags &= ~SWAP_RECLAIM;
1804 swf->swp_flags |= SWAP_READY;
1805
1806 goto done;
1807 }
1808 /*
1809 * We don't remove this inactive swf from the queue.
1810 * That way, we can re-use it when needed again and
1811 * preserve the namespace. The delayed_trim processing
1812 * is also dependent on us not removing swfs from the queue.
1813 */
1814 //queue_remove(&swf_global_queue, swf, struct swapfile*, swp_queue);
1815
1816 vm_num_swap_files--;
1817
1818 vm_swapfile_total_segs_alloced -= swf->swp_nsegs;
1819
1820 lck_mtx_unlock(&vm_swap_data_lock);
1821
1822 vm_swapfile_close((uint64_t)(swf->swp_path), swf->swp_vp);
1823
1824 kfree(swf->swp_csegs, swf->swp_nsegs * sizeof(c_segment_t));
1825 kfree(swf->swp_bitmap, MAX((swf->swp_nsegs >> 3), 1));
1826
1827 lck_mtx_lock(&vm_swap_data_lock);
1828
1829 if (swf->swp_flags & SWAP_PINNED) {
1830 vm_num_pinned_swap_files--;
1831 vm_swappin_avail += swf->swp_size;
1832 }
1833
1834 swf->swp_vp = NULL;
1835 swf->swp_size = 0;
1836 swf->swp_free_hint = 0;
1837 swf->swp_nsegs = 0;
1838 swf->swp_flags = SWAP_REUSE;
1839
1840 done:
1841 thread_wakeup((event_t) &swf->swp_flags);
1842 lck_mtx_unlock(&vm_swap_data_lock);
1843
1844 kmem_free(compressor_map, (vm_offset_t) addr, C_SEG_BUFSIZE);
1845 }
1846
1847
1848 uint64_t
1849 vm_swap_get_total_space(void)
1850 {
1851 uint64_t total_space = 0;
1852
1853 total_space = (uint64_t)vm_swapfile_total_segs_alloced * COMPRESSED_SWAP_CHUNK_SIZE;
1854
1855 return total_space;
1856 }
1857
1858 uint64_t
1859 vm_swap_get_used_space(void)
1860 {
1861 uint64_t used_space = 0;
1862
1863 used_space = (uint64_t)vm_swapfile_total_segs_used * COMPRESSED_SWAP_CHUNK_SIZE;
1864
1865 return used_space;
1866 }
1867
1868 uint64_t
1869 vm_swap_get_free_space(void)
1870 {
1871 return (vm_swap_get_total_space() - vm_swap_get_used_space());
1872 }
1873
1874
1875 int
1876 vm_swap_low_on_space(void)
1877 {
1878
1879 if (vm_num_swap_files == 0 && vm_swapfile_can_be_created == FALSE)
1880 return (0);
1881
1882 if (((vm_swapfile_total_segs_alloced - vm_swapfile_total_segs_used) < ((unsigned int)VM_SWAPFILE_HIWATER_SEGS) / 8)) {
1883
1884 if (vm_num_swap_files == 0 && !SWAPPER_NEEDS_TO_UNTHROTTLE())
1885 return (0);
1886
1887 if (vm_swapfile_last_failed_to_create_ts >= vm_swapfile_last_successful_create_ts)
1888 return (1);
1889 }
1890 return (0);
1891 }
1892
1893 boolean_t
1894 vm_swap_files_pinned(void)
1895 {
1896 boolean_t result;
1897
1898 if (vm_swappin_enabled == FALSE)
1899 return(TRUE);
1900
1901 result = (vm_num_pinned_swap_files == vm_num_swap_files);
1902
1903 return (result);
1904 }
mirror of XNU