From 8a3053a07cee346dca737a5670e546fd26a7c9d6 Mon Sep 17 00:00:00 2001 From: Apple Date: Wed, 13 Aug 2014 17:59:03 +0000 Subject: [PATCH] xnu-2422.110.17.tar.gz --- bsd/hfs/hfs_fsctl.h | 1 + bsd/kern/kern_event.c | 1 + bsd/kern/kern_kpc.c | 19 ++++ bsd/netinet/tcp_input.c | 15 ++- bsd/netinet/tcp_output.c | 16 ++-- bsd/netinet/tcp_timer.c | 75 ++++++++++----- bsd/netinet/tcp_timer.h | 3 +- config/MasterVersion | 2 +- iokit/IOKit/IOHibernatePrivate.h | 3 + iokit/Kernel/IOHibernateIO.cpp | 22 ++++- iokit/Kernel/IOPMrootDomain.cpp | 3 + osfmk/kern/hibernate.c | 2 +- osfmk/vm/vm_compressor.c | 126 +++++++++++++++++++------ osfmk/vm/vm_compressor.h | 1 + osfmk/vm/vm_compressor_backing_store.c | 19 ++-- osfmk/vm/vm_resident.c | 86 ++++++++++------- 16 files changed, 284 insertions(+), 110 deletions(-) diff --git a/bsd/hfs/hfs_fsctl.h b/bsd/hfs/hfs_fsctl.h index aad94ddab..9739a8555 100644 --- a/bsd/hfs/hfs_fsctl.h +++ b/bsd/hfs/hfs_fsctl.h @@ -148,6 +148,7 @@ struct hfs_journal_info { #define HFSIOC_GET_WRITE_GEN_COUNTER _IOR('h', 30, u_int32_t) #define HFS_GET_WRITE_GEN_COUNTER IOCBASECMD(HFSIOC_GET_WRITE_GEN_COUNTER) +/* revisiond uses this to allocate a doc-id for files from Cab and earlier systems that are marked tracked but don't have a doc-id */ #define HFS_DOCUMENT_ID_ALLOCATE 0x1 #define HFSIOC_GET_DOCUMENT_ID _IOR('h', 31, u_int32_t) diff --git a/bsd/kern/kern_event.c b/bsd/kern/kern_event.c index 52d271a02..e74047e2c 100644 --- a/bsd/kern/kern_event.c +++ b/bsd/kern/kern_event.c @@ -1285,6 +1285,7 @@ kqueue_body(struct proc *p, fp_allocfn_t fp_zalloc, void *cra, int32_t *retval) fp->f_data = kq; proc_fdlock(p); + *fdflags(p, fd) |= UF_EXCLOSE; procfdtbl_releasefd(p, fd, NULL); fp_drop(p, fd, fp, 1); proc_fdunlock(p); diff --git a/bsd/kern/kern_kpc.c b/bsd/kern/kern_kpc.c index 321fa1b5a..9cf0ab817 100644 --- a/bsd/kern/kern_kpc.c +++ b/bsd/kern/kern_kpc.c @@ -37,6 +37,7 @@ #include #include +#include /* Various sysctl requests */ #define REQ_CLASSES (1) @@ -387,6 +388,24 @@ kpc_sysctl SYSCTL_HANDLER_ARGS if( !kpc_initted ) panic("kpc_init not called"); + // Most sysctls require an access check, but a few are public. + switch( (uintptr_t) arg1 ) { + case REQ_CLASSES: + case REQ_CONFIG_COUNT: + case REQ_COUNTER_COUNT: + // These read-only sysctls are public. + break; + + default: + // Require kperf access to read or write anything else. + // This is either root or the blessed pid. + ret = kperf_access_check(); + if (ret) { + return ret; + } + break; + } + lck_mtx_lock(&sysctl_buffer_lock); /* which request */ diff --git a/bsd/netinet/tcp_input.c b/bsd/netinet/tcp_input.c index 005d449c2..95b5b703e 100644 --- a/bsd/netinet/tcp_input.c +++ b/bsd/netinet/tcp_input.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2013 Apple Inc. All rights reserved. + * Copyright (c) 2000-2014 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -3367,6 +3367,19 @@ trimthenstep6: */ if (SEQ_LEQ(th->th_ack, tp->snd_una)) { if (tlen == 0 && tiwin == tp->snd_wnd) { + /* + * If both ends send FIN at the same time, + * then the ack will be a duplicate ack + * but we have to process the FIN. Check + * for this condition and process the FIN + * instead of the dupack + */ + if ((thflags & TH_FIN) && + (tp->t_flags & TF_SENTFIN) && + !TCPS_HAVERCVDFIN(tp->t_state) && + (th->th_ack + 1) == tp->snd_max) { + break; + } process_dupack: #if MPTCP /* diff --git a/bsd/netinet/tcp_output.c b/bsd/netinet/tcp_output.c index 8c58a9bfc..fa6e5348c 100644 --- a/bsd/netinet/tcp_output.c +++ b/bsd/netinet/tcp_output.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2013 Apple Inc. All rights reserved. + * Copyright (c) 2000-2014 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -854,6 +854,7 @@ after_sack_rexmit: recwin = tcp_sbspace(tp); + /* * If the socket is capable of doing unordered send, * pull the amount of data that can be sent from the @@ -992,8 +993,8 @@ after_sack_rexmit: * If our state indicates that FIN should be sent * and we have not yet done so, then we need to send. */ - if (flags & TH_FIN && - ((tp->t_flags & TF_SENTFIN) == 0 || tp->snd_nxt == tp->snd_una)) + if ((flags & TH_FIN) && + (!(tp->t_flags & TF_SENTFIN) || tp->snd_nxt == tp->snd_una)) goto send; /* * In SACK, it is possible for tcp_output to fail to send a segment @@ -1800,7 +1801,8 @@ send: if (flags & (TH_SYN|TH_FIN)) { if (flags & TH_SYN) tp->snd_nxt++; - if (flags & TH_FIN) { + if ((flags & TH_FIN) && + !(tp->t_flags & TF_SENTFIN)) { tp->snd_nxt++; tp->t_flags |= TF_SENTFIN; } @@ -1832,7 +1834,8 @@ send: timer: if (tp->t_timer[TCPT_REXMT] == 0 && ((sack_rxmit && tp->snd_nxt != tp->snd_max) || - tp->snd_nxt != tp->snd_una)) { + tp->snd_nxt != tp->snd_una || + (flags & TH_FIN))) { if (tp->t_timer[TCPT_PERSIST]) { tp->t_timer[TCPT_PERSIST] = 0; tp->t_rxtshift = 0; @@ -1849,7 +1852,8 @@ timer: int xlen = len; if (flags & TH_SYN) ++xlen; - if (flags & TH_FIN) { + if ((flags & TH_FIN) && + !(tp->t_flags & TF_SENTFIN)) { ++xlen; tp->t_flags |= TF_SENTFIN; } diff --git a/bsd/netinet/tcp_timer.c b/bsd/netinet/tcp_timer.c index a855e6fa9..b1ac3138b 100644 --- a/bsd/netinet/tcp_timer.c +++ b/bsd/netinet/tcp_timer.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2013 Apple Inc. All rights reserved. + * Copyright (c) 2000-2014 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -282,6 +282,8 @@ timer_diff(uint32_t t1, uint32_t toff1, uint32_t t2, uint32_t toff2) { /* Returns true if the timer is on the timer list */ #define TIMER_IS_ON_LIST(tp) ((tp)->t_flags & TF_TIMER_ONLIST) +/* Run the TCP timerlist atleast once every hour */ +#define TCP_TIMERLIST_MAX_OFFSET (60 * 60 * TCP_RETRANSHZ) static void add_to_time_wait_locked(struct tcpcb *tp, uint32_t delay); void add_to_time_wait(struct tcpcb *tp, uint32_t delay) ; @@ -1107,16 +1109,18 @@ need_to_resched_timerlist(uint32_t runtime, uint16_t index) { int32_t diff; boolean_t is_fast; - if (runtime == 0 || index == TCPT_NONE) + if (index == TCPT_NONE) return FALSE; is_fast = !(IS_TIMER_SLOW(index)); /* If the list is being processed then the state of the list is in flux. * In this case always acquire the lock and set the state correctly. */ - if (listp->running) { + if (listp->running) return TRUE; - } + + if (!listp->scheduled) + return (TRUE); diff = timer_diff(listp->runtime, 0, runtime, 0); if (diff <= 0) { @@ -1143,12 +1147,16 @@ tcp_sched_timerlist(uint32_t offset) lck_mtx_assert(listp->mtx, LCK_MTX_ASSERT_OWNED); + offset = min(offset, TCP_TIMERLIST_MAX_OFFSET); listp->runtime = tcp_now + offset; + if (listp->runtime == 0) + listp->runtime++; clock_interval_to_deadline(offset, NSEC_PER_SEC / TCP_RETRANSHZ, &deadline); thread_call_enter_delayed(listp->call, deadline); + listp->scheduled = TRUE; } /* Function to run the timers for a connection. @@ -1188,11 +1196,9 @@ tcp_run_conn_timer(struct tcpcb *tp, uint16_t *next_index) { * with another thread that can cancel or reschedule the timer that is * about to run. Check if we need to run anything. */ - index = tp->tentry.index; - timer_val = tp->t_timer[index]; - - if (index == TCPT_NONE || tp->tentry.runtime == 0) + if ((index = tp->tentry.index) == TCPT_NONE) goto done; + timer_val = tp->t_timer[index]; diff = timer_diff(tp->tentry.runtime, 0, tcp_now, 0); if (diff > 0) { @@ -1235,8 +1241,8 @@ tcp_run_conn_timer(struct tcpcb *tp, uint16_t *next_index) { tp->tentry.index = lo_index; if (lo_index != TCPT_NONE) { tp->tentry.runtime = tp->tentry.timer_start + tp->t_timer[lo_index]; - } else { - tp->tentry.runtime = 0; + if (tp->tentry.runtime == 0) + tp->tentry.runtime++; } if (count > 0) { @@ -1245,8 +1251,11 @@ tcp_run_conn_timer(struct tcpcb *tp, uint16_t *next_index) { if (needtorun[i]) { tp->t_timer[i] = 0; tp = tcp_timers(tp, i); - if (tp == NULL) + if (tp == NULL) { + offset = 0; + *(next_index) = TCPT_NONE; goto done; + } } } tcp_set_lotimer_index(tp); @@ -1260,6 +1269,7 @@ tcp_run_conn_timer(struct tcpcb *tp, uint16_t *next_index) { done: if (tp != NULL && tp->tentry.index == TCPT_NONE) { tcp_remove_timer(tp); + offset = 0; } tcp_unlock(so, 1, 0); return offset; @@ -1288,7 +1298,7 @@ tcp_run_timerlist(void * arg1, void * arg2) { LIST_FOREACH_SAFE(te, &listp->lhead, le, next_te) { uint32_t offset = 0; uint32_t runtime = te->runtime; - if (TSTMP_GT(runtime, tcp_now)) { + if (te->index < TCPT_NONE && TSTMP_GT(runtime, tcp_now)) { offset = timer_diff(runtime, 0, tcp_now, 0); if (next_timer == 0 || offset < next_timer) { next_timer = offset; @@ -1384,8 +1394,11 @@ tcp_run_timerlist(void * arg1, void * arg2) { tcp_sched_timerlist(next_timer); } else { - /* No need to reschedule this timer */ - listp->runtime = 0; + /* + * No need to reschedule this timer, but always run + * periodically at a much higher granularity. + */ + tcp_sched_timerlist(TCP_TIMERLIST_MAX_OFFSET); } listp->running = FALSE; @@ -1402,7 +1415,7 @@ tcp_sched_timers(struct tcpcb *tp) struct tcptimerentry *te = &tp->tentry; uint16_t index = te->index; struct tcptimerlist *listp = &tcp_timer_list; - uint32_t offset = 0; + int32_t offset = 0; boolean_t is_fast; int list_locked = 0; @@ -1420,8 +1433,8 @@ tcp_sched_timers(struct tcpcb *tp) } is_fast = !(IS_TIMER_SLOW(index)); - offset = te->runtime - tcp_now; - if (offset == 0) { + offset = timer_diff(te->runtime, 0, tcp_now, 0); + if (offset <= 0) { offset = 1; tcp_timer_advanced++; } @@ -1442,7 +1455,7 @@ tcp_sched_timers(struct tcpcb *tp) listp->maxentries = listp->entries; /* if the list is not scheduled, just schedule it */ - if (listp->runtime == 0) + if (!listp->scheduled) goto schedule; } @@ -1464,15 +1477,22 @@ tcp_sched_timers(struct tcpcb *tp) if (is_fast) { listp->pref_mode = TCP_TIMERLIST_FASTMODE; } else if (listp->pref_offset == 0 || - ((int)offset) < listp->pref_offset) { + offset < listp->pref_offset) { listp->pref_offset = offset; } } else { - int32_t diff; - diff = timer_diff(listp->runtime, 0, tcp_now, offset); - if (diff <= 0) { - /* The list is going to run before this timer */ - goto done; + /* + * The list could have got scheduled while this + * thread was waiting for the lock + */ + if (listp->scheduled) { + int32_t diff; + diff = timer_diff(listp->runtime, 0, + tcp_now, offset); + if (diff <= 0) + goto done; + else + goto schedule; } else { goto schedule; } @@ -1508,8 +1528,8 @@ tcp_set_lotimer_index(struct tcpcb *tp) { tp->tentry.index = lo_index; if (lo_index != TCPT_NONE) { tp->tentry.runtime = tp->tentry.timer_start + tp->t_timer[lo_index]; - } else { - tp->tentry.runtime = 0; + if (tp->tentry.runtime == 0) + tp->tentry.runtime++; } } @@ -1518,6 +1538,9 @@ tcp_check_timer_state(struct tcpcb *tp) { lck_mtx_assert(&tp->t_inpcb->inpcb_mtx, LCK_MTX_ASSERT_OWNED); + if (tp->t_inpcb->inp_flags2 & INP2_TIMEWAIT) + return; + tcp_set_lotimer_index(tp); tcp_sched_timers(tp); diff --git a/bsd/netinet/tcp_timer.h b/bsd/netinet/tcp_timer.h index 55f428707..f7b775db5 100644 --- a/bsd/netinet/tcp_timer.h +++ b/bsd/netinet/tcp_timer.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2010 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2000-2014 Apple Computer, Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -208,6 +208,7 @@ struct tcptimerlist { /* Set desired mode when timer list running */ boolean_t running; /* Set when timer list is being processed */ + boolean_t scheduled; /* Set when timer is scheduled */ #define TCP_TIMERLIST_FASTMODE 0x1 #define TCP_TIMERLIST_SLOWMODE 0x2 uint32_t mode; /* Current mode, fast or slow */ diff --git a/config/MasterVersion b/config/MasterVersion index 394f474c1..8fcb30c96 100644 --- a/config/MasterVersion +++ b/config/MasterVersion @@ -1,4 +1,4 @@ -13.2.0 +13.3.0 # The first line of this file contains the master version number for the kernel. # All other instances of the kernel version in xnu are derived from this file. diff --git a/iokit/IOKit/IOHibernatePrivate.h b/iokit/IOKit/IOHibernatePrivate.h index 525fb5d7a..eddd86c05 100644 --- a/iokit/IOKit/IOHibernatePrivate.h +++ b/iokit/IOKit/IOHibernatePrivate.h @@ -339,6 +339,9 @@ kern_close_file_for_direct_io(struct kern_direct_file_io_ref_t * ref, off_t discard_offset, off_t discard_end); #endif /* _SYS_CONF_H_ */ +void +vm_compressor_do_warmup(void); + hibernate_page_list_t * hibernate_page_list_allocate(boolean_t log); diff --git a/iokit/Kernel/IOHibernateIO.cpp b/iokit/Kernel/IOHibernateIO.cpp index 3bd133eba..851adac60 100644 --- a/iokit/Kernel/IOHibernateIO.cpp +++ b/iokit/Kernel/IOHibernateIO.cpp @@ -1262,11 +1262,13 @@ IOHibernateSystemSleep(void) bzero(&consoleInfo, sizeof(consoleInfo)); IOService::getPlatform()->getConsoleInfo(&consoleInfo); - // estimate: 5% increase in pages compressed + // estimate: 6% increase in pages compressed // screen preview 2 images compressed 50% - setFileSize = ((ptoa_64((105 * pageCount) / 100) * gIOHibernateCompression) >> 8) + setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8) + vars->page_list->list_size - + (consoleInfo.v_width * consoleInfo.v_height * 4); + + (consoleInfo.v_width * consoleInfo.v_height * 8); + enum { setFileRound = 1024*1024ULL }; + setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1)); HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n", pageCount, (100ULL * gIOHibernateCompression) >> 8, @@ -1284,6 +1286,7 @@ IOHibernateSystemSleep(void) err = IOPolledFileOpen(gIOHibernateFilename, setFileSize, vars->ioBuffer, &vars->fileVars, &vars->fileExtents, &data, &vars->volumeCryptKey[0]); + if (KERN_SUCCESS != err) { HIBLOG("IOPolledFileOpen(%x)\n", err); @@ -2658,7 +2661,14 @@ hibernate_write_image(void) } } if (kIOReturnSuccess != err) + { + if (kIOReturnOverrun == err) + { + // update actual compression ratio on not enough space + gIOHibernateCompression = (compressedSize << 8) / uncompressedSize; + } break; + } // Header: @@ -3113,6 +3123,12 @@ void IOHibernateSetWakeCapabilities(uint32_t capability) if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) { gIOHibernateStats->wakeCapability = capability; + + if (kIOPMSystemCapabilityGraphics & capability) + { + vm_compressor_do_warmup(); + } + } } diff --git a/iokit/Kernel/IOPMrootDomain.cpp b/iokit/Kernel/IOPMrootDomain.cpp index 60f715588..7221295fa 100644 --- a/iokit/Kernel/IOPMrootDomain.cpp +++ b/iokit/Kernel/IOPMrootDomain.cpp @@ -6880,6 +6880,9 @@ void IOPMrootDomain::willEnterFullWake( void ) (kFullWakeReasonLocalUser == fullWakeReason) ? kOSBooleanTrue : kOSBooleanFalse); } +#if HIBERNATION + IOHibernateSetWakeCapabilities(_pendingCapability); +#endif IOService::setAdvisoryTickleEnable( true ); tellClients(kIOMessageSystemWillPowerOn); diff --git a/osfmk/kern/hibernate.c b/osfmk/kern/hibernate.c index 7b3d38563..8d0806589 100644 --- a/osfmk/kern/hibernate.c +++ b/osfmk/kern/hibernate.c @@ -160,7 +160,7 @@ hibernate_teardown(hibernate_page_list_t * page_list, need_to_unlock_decompressor = FALSE; vm_decompressor_unlock(); } - vm_compressor_do_warmup(); + vm_compressor_delay_trim(); } return (KERN_SUCCESS); } diff --git a/osfmk/vm/vm_compressor.c b/osfmk/vm/vm_compressor.c index ca9ff6c6d..bc565e3f1 100644 --- a/osfmk/vm/vm_compressor.c +++ b/osfmk/vm/vm_compressor.c @@ -775,7 +775,8 @@ c_seg_free_locked(c_segment_t c_seg) lck_mtx_unlock_always(&c_seg->c_lock); if (c_buffer) { - kernel_memory_depopulate(kernel_map, (vm_offset_t) c_buffer, pages_populated * PAGE_SIZE, KMA_COMPRESSOR); + if (pages_populated) + kernel_memory_depopulate(kernel_map, (vm_offset_t) c_buffer, pages_populated * PAGE_SIZE, KMA_COMPRESSOR); kmem_free(kernel_map, (vm_offset_t) c_buffer, C_SEG_ALLOCSIZE); } else if (c_swap_handle) @@ -1450,6 +1451,15 @@ vm_compressor_do_delayed_compactions(boolean_t flush_all) c_seg = (c_segment_t)queue_first(&c_minor_list_head); lck_mtx_lock_spin_always(&c_seg->c_lock); + + if (c_seg->c_busy) { + + lck_mtx_unlock_always(c_list_lock); + c_seg_wait_on_busy(c_seg); + lck_mtx_lock_spin_always(c_list_lock); + + continue; + } c_seg->c_busy = 1; c_seg_do_minor_compaction_and_unlock(c_seg, TRUE, FALSE, TRUE); @@ -1604,16 +1614,17 @@ vm_compressor_record_warmup_start(void) lck_mtx_lock_spin_always(c_list_lock); - if (!queue_empty(&c_age_list_head)) { - - c_seg = (c_segment_t)queue_last(&c_age_list_head); + if (first_c_segment_to_warm_generation_id == 0) { + if (!queue_empty(&c_age_list_head)) { - first_c_segment_to_warm_generation_id = c_seg->c_generation_id; - } else - first_c_segment_to_warm_generation_id = 0; + c_seg = (c_segment_t)queue_last(&c_age_list_head); - fastwake_recording_in_progress = TRUE; + first_c_segment_to_warm_generation_id = c_seg->c_generation_id; + } else + first_c_segment_to_warm_generation_id = 0; + fastwake_recording_in_progress = TRUE; + } lck_mtx_unlock_always(c_list_lock); } @@ -1625,16 +1636,20 @@ vm_compressor_record_warmup_end(void) lck_mtx_lock_spin_always(c_list_lock); - if (!queue_empty(&c_age_list_head)) { + if (fastwake_recording_in_progress == TRUE) { - c_seg = (c_segment_t)queue_last(&c_age_list_head); + if (!queue_empty(&c_age_list_head)) { - last_c_segment_to_warm_generation_id = c_seg->c_generation_id; - } else - last_c_segment_to_warm_generation_id = first_c_segment_to_warm_generation_id; + c_seg = (c_segment_t)queue_last(&c_age_list_head); + + last_c_segment_to_warm_generation_id = c_seg->c_generation_id; + } else + last_c_segment_to_warm_generation_id = first_c_segment_to_warm_generation_id; - fastwake_recording_in_progress = FALSE; + fastwake_recording_in_progress = FALSE; + HIBLOG("vm_compressor_record_warmup (%qd - %qd)\n", first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id); + } lck_mtx_unlock_always(c_list_lock); } @@ -1642,19 +1657,28 @@ vm_compressor_record_warmup_end(void) #define DELAY_TRIM_ON_WAKE_SECS 4 void -vm_compressor_do_warmup(void) +vm_compressor_delay_trim(void) { - clock_sec_t sec; + clock_sec_t sec; clock_nsec_t nsec; clock_get_system_nanotime(&sec, &nsec); dont_trim_until_ts = sec + DELAY_TRIM_ON_WAKE_SECS; +} - if (first_c_segment_to_warm_generation_id == last_c_segment_to_warm_generation_id) - return; +void +vm_compressor_do_warmup(void) +{ lck_mtx_lock_spin_always(c_list_lock); + if (first_c_segment_to_warm_generation_id == last_c_segment_to_warm_generation_id) { + first_c_segment_to_warm_generation_id = last_c_segment_to_warm_generation_id = 0; + + lck_mtx_unlock_always(c_list_lock); + return; + } + if (compaction_swapper_running == 0) { fastwake_warmup = TRUE; @@ -1670,6 +1694,13 @@ do_fastwake_warmup(void) { uint64_t my_thread_id; c_segment_t c_seg = NULL; + AbsoluteTime startTime, endTime; + uint64_t nsec; + + + HIBLOG("vm_compressor_fastwake_warmup (%qd - %qd) - starting\n", first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id); + + clock_get_uptime(&startTime); lck_mtx_unlock_always(c_list_lock); @@ -1692,15 +1723,19 @@ do_fastwake_warmup(void) lck_mtx_lock_spin_always(&c_seg->c_lock); lck_mtx_unlock_always(c_list_lock); - if (c_seg->c_busy) + if (c_seg->c_busy) { + PAGE_REPLACEMENT_DISALLOWED(FALSE); c_seg_wait_on_busy(c_seg); - else { + PAGE_REPLACEMENT_DISALLOWED(TRUE); + } else { c_seg_swapin(c_seg, TRUE); lck_mtx_unlock_always(&c_seg->c_lock); - c_segment_warmup_count++; + + PAGE_REPLACEMENT_DISALLOWED(FALSE); vm_pageout_io_throttle(); + PAGE_REPLACEMENT_DISALLOWED(TRUE); } lck_mtx_lock_spin_always(c_list_lock); } @@ -1711,7 +1746,15 @@ do_fastwake_warmup(void) proc_set_task_policy_thread(kernel_task, my_thread_id, TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER0); + clock_get_uptime(&endTime); + SUB_ABSOLUTETIME(&endTime, &startTime); + absolutetime_to_nanoseconds(endTime, &nsec); + + HIBLOG("vm_compressor_fastwake_warmup completed - took %qd msecs\n", nsec / 1000000ULL); + lck_mtx_lock_spin_always(c_list_lock); + + first_c_segment_to_warm_generation_id = last_c_segment_to_warm_generation_id = 0; } @@ -1735,6 +1778,18 @@ vm_compressor_compact_and_swap(boolean_t flush_all) fastwake_warmup = FALSE; } + /* + * it's possible for the c_age_list_head to be empty if we + * hit our limits for growing the compressor pool and we subsequently + * hibernated... on the next hibernation we could see the queue as + * empty and not proceeed even though we have a bunch of segments on + * the swapped in queue that need to be dealt with. + */ + vm_compressor_do_delayed_compactions(flush_all); + + vm_compressor_age_swapped_in_segments(flush_all); + + while (!queue_empty(&c_age_list_head) && compaction_swapper_abort == 0) { if (hibernate_flushing == TRUE) { @@ -2485,13 +2540,28 @@ c_seg_invalid_data: } if (!c_seg->c_filling) { if (c_seg->c_bytes_used == 0) { - if (c_seg->c_on_minorcompact_q || c_seg->c_on_swappedout_sparse_q) { - if (c_seg_try_free(c_seg) == TRUE) - need_unlock = FALSE; - } else { - c_seg_free(c_seg); - need_unlock = FALSE; - } + if (!c_seg->c_ondisk) { + int pages_populated; + + pages_populated = (round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE; + c_seg->c_populated_offset = C_SEG_BYTES_TO_OFFSET(0); + + if (pages_populated) { + assert(c_seg->c_store.c_buffer != NULL); + + c_seg->c_busy = 1; + lck_mtx_unlock_always(&c_seg->c_lock); + + kernel_memory_depopulate(kernel_map, (vm_offset_t) c_seg->c_store.c_buffer, pages_populated * PAGE_SIZE, KMA_COMPRESSOR); + + lck_mtx_lock_spin_always(&c_seg->c_lock); + C_SEG_WAKEUP_DONE(c_seg); + } + if (!c_seg->c_on_minorcompact_q && !c_seg->c_on_swapout_q) + c_seg_need_delayed_compaction(c_seg); + } else + assert(c_seg->c_on_swappedout_sparse_q); + } else if (c_seg->c_on_minorcompact_q) { if (C_SEG_INCORE_IS_SPARSE(c_seg)) { diff --git a/osfmk/vm/vm_compressor.h b/osfmk/vm/vm_compressor.h index c2d85ab08..4721f3e32 100644 --- a/osfmk/vm/vm_compressor.h +++ b/osfmk/vm/vm_compressor.h @@ -155,6 +155,7 @@ void c_seg_insert_into_age_q(c_segment_t); void vm_decompressor_lock(void); void vm_decompressor_unlock(void); +void vm_compressor_delay_trim(void); void vm_compressor_do_warmup(void); void vm_compressor_record_warmup_start(void); void vm_compressor_record_warmup_end(void); diff --git a/osfmk/vm/vm_compressor_backing_store.c b/osfmk/vm/vm_compressor_backing_store.c index 80db056d4..bf57f06e5 100644 --- a/osfmk/vm/vm_compressor_backing_store.c +++ b/osfmk/vm/vm_compressor_backing_store.c @@ -720,10 +720,9 @@ vm_swapout_thread(void) assert(c_seg->c_on_swapout_q); if (c_seg->c_busy) { - lck_mtx_unlock_always(&c_seg->c_lock); lck_mtx_unlock_always(c_list_lock); - mutex_pause(2); + c_seg_wait_on_busy(c_seg); lck_mtx_lock_spin_always(c_list_lock); @@ -733,19 +732,23 @@ vm_swapout_thread(void) c_seg->c_on_swapout_q = 0; c_swapout_count--; - c_seg->c_busy = 1; - c_seg->c_busy_swapping = 1; - vm_swapout_thread_processed_segments++; thread_wakeup((event_t)&compaction_swapper_running); + size = round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset)); + + if (size == 0) { + c_seg_free_locked(c_seg); + goto c_seg_was_freed; + } + c_seg->c_busy = 1; + c_seg->c_busy_swapping = 1; + lck_mtx_unlock_always(c_list_lock); addr = (vm_offset_t) c_seg->c_store.c_buffer; - size = round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset)); - lck_mtx_unlock_always(&c_seg->c_lock); #if CHECKSUM_THE_SWAP @@ -820,7 +823,7 @@ vm_swapout_thread(void) kmem_free(kernel_map, (vm_offset_t) addr, C_SEG_ALLOCSIZE); vm_pageout_io_throttle(); - +c_seg_was_freed: if (c_swapout_count == 0) vm_swap_consider_defragmenting(); diff --git a/osfmk/vm/vm_resident.c b/osfmk/vm/vm_resident.c index a9fd4d0b2..32271953a 100644 --- a/osfmk/vm/vm_resident.c +++ b/osfmk/vm/vm_resident.c @@ -4809,6 +4809,7 @@ struct hibernate_statistics { int cd_found_laundry; int cd_found_dirty; int cd_found_xpmapped; + int cd_skipped_xpmapped; int cd_local_free; int cd_total_free; int cd_vm_page_wire_count; @@ -4819,6 +4820,13 @@ struct hibernate_statistics { } hibernate_stats; +/* + * clamp the number of 'xpmapped' pages we'll sweep into the hibernation image + * so that we don't overrun the estimated image size, which would + * result in a hibernation failure. + */ +#define HIBERNATE_XPMAPPED_LIMIT 40000 + static int hibernate_drain_pageout_queue(struct vm_pageout_queue *q) @@ -5360,10 +5368,15 @@ hibernate_consider_discard(vm_page_t m, boolean_t preflight) if (discard == FALSE) { if (!preflight) hibernate_stats.cd_found_dirty++; - } else if (m->xpmapped && m->reference) { - if (!preflight) - hibernate_stats.cd_found_xpmapped++; - discard = FALSE; + } else if (m->xpmapped && m->reference && !object->internal) { + if (hibernate_stats.cd_found_xpmapped < HIBERNATE_XPMAPPED_LIMIT) { + if (!preflight) + hibernate_stats.cd_found_xpmapped++; + discard = FALSE; + } else { + if (!preflight) + hibernate_stats.cd_skipped_xpmapped++; + } } } while (FALSE); @@ -5650,8 +5663,8 @@ hibernate_page_list_setall(hibernate_page_list_t * page_list, m = next; } - m = (vm_page_t) queue_first(&vm_page_queue_inactive); - while (m && !queue_end(&vm_page_queue_inactive, (queue_entry_t)m)) + m = (vm_page_t) queue_first(&vm_page_queue_cleaned); + while (m && !queue_end(&vm_page_queue_cleaned, (queue_entry_t)m)) { next = (vm_page_t) m->pageq.next; discard = FALSE; @@ -5662,19 +5675,42 @@ hibernate_page_list_setall(hibernate_page_list_t * page_list, if (m->dirty) count_discard_purgeable++; else - count_discard_inactive++; + count_discard_cleaned++; discard = discard_all; } else - count_inactive++; + count_cleaned++; count_wire--; if (!preflight) hibernate_page_bitset(page_list_wired, TRUE, m->phys_page); if (discard) hibernate_discard_page(m); m = next; } - m = (vm_page_t) queue_first(&vm_page_queue_cleaned); - while (m && !queue_end(&vm_page_queue_cleaned, (queue_entry_t)m)) + m = (vm_page_t) queue_first(&vm_page_queue_active); + while (m && !queue_end(&vm_page_queue_active, (queue_entry_t)m)) + { + next = (vm_page_t) m->pageq.next; + discard = FALSE; + if ((kIOHibernateModeDiscardCleanActive & gIOHibernateMode) + && hibernate_consider_discard(m, preflight)) + { + if (!preflight) hibernate_page_bitset(page_list, TRUE, m->phys_page); + if (m->dirty) + count_discard_purgeable++; + else + count_discard_active++; + discard = discard_all; + } + else + count_active++; + count_wire--; + if (!preflight) hibernate_page_bitset(page_list_wired, TRUE, m->phys_page); + if (discard) hibernate_discard_page(m); + m = next; + } + + m = (vm_page_t) queue_first(&vm_page_queue_inactive); + while (m && !queue_end(&vm_page_queue_inactive, (queue_entry_t)m)) { next = (vm_page_t) m->pageq.next; discard = FALSE; @@ -5685,11 +5721,11 @@ hibernate_page_list_setall(hibernate_page_list_t * page_list, if (m->dirty) count_discard_purgeable++; else - count_discard_cleaned++; + count_discard_inactive++; discard = discard_all; } else - count_cleaned++; + count_inactive++; count_wire--; if (!preflight) hibernate_page_bitset(page_list_wired, TRUE, m->phys_page); if (discard) hibernate_discard_page(m); @@ -5719,29 +5755,6 @@ hibernate_page_list_setall(hibernate_page_list_t * page_list, } } - m = (vm_page_t) queue_first(&vm_page_queue_active); - while (m && !queue_end(&vm_page_queue_active, (queue_entry_t)m)) - { - next = (vm_page_t) m->pageq.next; - discard = FALSE; - if ((kIOHibernateModeDiscardCleanActive & gIOHibernateMode) - && hibernate_consider_discard(m, preflight)) - { - if (!preflight) hibernate_page_bitset(page_list, TRUE, m->phys_page); - if (m->dirty) - count_discard_purgeable++; - else - count_discard_active++; - discard = discard_all; - } - else - count_active++; - count_wire--; - if (!preflight) hibernate_page_bitset(page_list_wired, TRUE, m->phys_page); - if (discard) hibernate_discard_page(m); - m = next; - } - queue_iterate(&compressor_object->memq, m, vm_page_t, listq) { count_compressor++; @@ -5795,6 +5808,9 @@ hibernate_page_list_setall(hibernate_page_list_t * page_list, discard_all ? "did" : "could", count_discard_active, count_discard_inactive, count_discard_purgeable, count_discard_speculative, count_discard_cleaned); + if (hibernate_stats.cd_skipped_xpmapped) + HIBLOG("WARNING: hibernate_page_list_setall skipped %d xpmapped pages\n", hibernate_stats.cd_skipped_xpmapped); + *pagesOut = pages - count_discard_active - count_discard_inactive - count_discard_purgeable - count_discard_speculative - count_discard_cleaned; if (preflight && will_discard) *pagesOut -= count_compressor + count_throttled + count_anonymous + count_inactive + count_cleaned + count_speculative + count_active; -- 2.47.2