xnu-2422.110.17.tar.gz

diff --git a/bsd/hfs/hfs_fsctl.h b/bsd/hfs/hfs_fsctl.h
index aad94ddabebb03f1062b9ccaad0939b709d41918..9739a855587458d23717f24958594b990501050f 100644 (file)
--- a/bsd/hfs/hfs_fsctl.h
+++ b/bsd/hfs/hfs_fsctl.h
@@ -148,6 +148,7 @@ struct hfs_journal_info {
 #define HFSIOC_GET_WRITE_GEN_COUNTER  _IOR('h', 30, u_int32_t)
 #define HFS_GET_WRITE_GEN_COUNTER  IOCBASECMD(HFSIOC_GET_WRITE_GEN_COUNTER)
 
 #define HFSIOC_GET_WRITE_GEN_COUNTER  _IOR('h', 30, u_int32_t)
 #define HFS_GET_WRITE_GEN_COUNTER  IOCBASECMD(HFSIOC_GET_WRITE_GEN_COUNTER)
 

+/* revisiond uses this to allocate a doc-id for files from Cab and earlier systems that are marked tracked but don't have a doc-id */

 #define HFS_DOCUMENT_ID_ALLOCATE       0x1
 
 #define HFSIOC_GET_DOCUMENT_ID  _IOR('h', 31, u_int32_t)
 #define HFS_DOCUMENT_ID_ALLOCATE       0x1
 
 #define HFSIOC_GET_DOCUMENT_ID  _IOR('h', 31, u_int32_t)

diff --git a/bsd/kern/kern_event.c b/bsd/kern/kern_event.c
index 52d271a02754ad961a3494f7d7209456db42535f..e74047e2c62402ae548c2a7e4e780e9540158124 100644 (file)
--- a/bsd/kern/kern_event.c
+++ b/bsd/kern/kern_event.c
@@ -1285,6 +1285,7 @@ kqueue_body(struct proc *p, fp_allocfn_t fp_zalloc, void *cra, int32_t *retval)
        fp->f_data = kq;
 
        proc_fdlock(p);
        fp->f_data = kq;
 
        proc_fdlock(p);

+       *fdflags(p, fd) |= UF_EXCLOSE;

        procfdtbl_releasefd(p, fd, NULL);
        fp_drop(p, fd, fp, 1);
        proc_fdunlock(p);
        procfdtbl_releasefd(p, fd, NULL);
        fp_drop(p, fd, fp, 1);
        proc_fdunlock(p);

diff --git a/bsd/kern/kern_kpc.c b/bsd/kern/kern_kpc.c
index 321fa1b5add70fc7bba1e5c97fd4636ff4c70314..9cf0ab817712c4c2ee150d01d9c38b5027e6d342 100644 (file)
--- a/bsd/kern/kern_kpc.c
+++ b/bsd/kern/kern_kpc.c
@@ -37,6 +37,7 @@
 #include <kern/kpc.h>
 
 #include <pexpert/pexpert.h>
 #include <kern/kpc.h>
 
 #include <pexpert/pexpert.h>

+#include <kperf/kperf.h>

 
 /* Various sysctl requests */
 #define REQ_CLASSES              (1)
 
 /* Various sysctl requests */
 #define REQ_CLASSES              (1)


@@ -387,6 +388,24 @@ kpc_sysctl SYSCTL_HANDLER_ARGS
        if( !kpc_initted )
                panic("kpc_init not called");
 
        if( !kpc_initted )
                panic("kpc_init not called");
 

+       // Most sysctls require an access check, but a few are public.
+       switch( (uintptr_t) arg1 ) {
+       case REQ_CLASSES:
+       case REQ_CONFIG_COUNT:
+       case REQ_COUNTER_COUNT:
+               // These read-only sysctls are public.
+               break;
+
+       default:
+               // Require kperf access to read or write anything else.
+               // This is either root or the blessed pid.
+               ret = kperf_access_check();
+               if (ret) {
+                       return ret;
+               }
+               break;
+       }
+

        lck_mtx_lock(&sysctl_buffer_lock);
 
        /* which request */
        lck_mtx_lock(&sysctl_buffer_lock);
 
        /* which request */

diff --git a/bsd/netinet/tcp_input.c b/bsd/netinet/tcp_input.c
index 005d449c24cf7a08716bfa64cdb5842e7ab0fbfb..95b5b703efb94995a2347a3ec23d6157a72dc168 100644 (file)
--- a/bsd/netinet/tcp_input.c
+++ b/bsd/netinet/tcp_input.c
@@ -1,5 +1,5 @@
 /*
 /*

- * Copyright (c) 2000-2013 Apple Inc. All rights reserved.
+ * Copyright (c) 2000-2014 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 


@@ -3367,6 +3367,19 @@ trimthenstep6:
                 */ 
                if (SEQ_LEQ(th->th_ack, tp->snd_una)) {
                        if (tlen == 0 && tiwin == tp->snd_wnd) {
                 */ 
                if (SEQ_LEQ(th->th_ack, tp->snd_una)) {
                        if (tlen == 0 && tiwin == tp->snd_wnd) {

+                               /*
+                                * If both ends send FIN at the same time,
+                                * then the ack will be a duplicate ack
+                                * but we have to process the FIN. Check
+                                * for this condition and process the FIN
+                                * instead of the dupack
+                                */ 
+                               if ((thflags & TH_FIN) &&
+                                       (tp->t_flags & TF_SENTFIN) &&
+                                       !TCPS_HAVERCVDFIN(tp->t_state) &&
+                                       (th->th_ack + 1) == tp->snd_max) {
+                                       break;
+                               }

 process_dupack:
 #if MPTCP
                                /*
 process_dupack:
 #if MPTCP
                                /*

diff --git a/bsd/netinet/tcp_output.c b/bsd/netinet/tcp_output.c
index 8c58a9bfc9d227423c7fefc694601e11a32175da..fa6e5348c0337aeb7847d37e17a534a230bba4c5 100644 (file)
--- a/bsd/netinet/tcp_output.c
+++ b/bsd/netinet/tcp_output.c
@@ -1,5 +1,5 @@
 /*
 /*

- * Copyright (c) 2000-2013 Apple Inc. All rights reserved.
+ * Copyright (c) 2000-2014 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 


@@ -854,6 +854,7 @@ after_sack_rexmit:
 
        recwin = tcp_sbspace(tp);
 
 
        recwin = tcp_sbspace(tp);
 

+

        /*
         * If the socket is capable of doing unordered send,
         * pull the amount of data that can be sent from the
        /*
         * If the socket is capable of doing unordered send,
         * pull the amount of data that can be sent from the


@@ -992,8 +993,8 @@ after_sack_rexmit:
         * If our state indicates that FIN should be sent
         * and we have not yet done so, then we need to send.
         */
         * If our state indicates that FIN should be sent
         * and we have not yet done so, then we need to send.
         */

-       if (flags & TH_FIN &&
-           ((tp->t_flags & TF_SENTFIN) == 0 || tp->snd_nxt == tp->snd_una))
+       if ((flags & TH_FIN) &&
+           (!(tp->t_flags & TF_SENTFIN) || tp->snd_nxt == tp->snd_una))

                goto send;
        /*
         * In SACK, it is possible for tcp_output to fail to send a segment
                goto send;
        /*
         * In SACK, it is possible for tcp_output to fail to send a segment


@@ -1800,7 +1801,8 @@ send:
                if (flags & (TH_SYN|TH_FIN)) {
                        if (flags & TH_SYN)
                                tp->snd_nxt++;
                if (flags & (TH_SYN|TH_FIN)) {
                        if (flags & TH_SYN)
                                tp->snd_nxt++;

-                       if (flags & TH_FIN) {
+                       if ((flags & TH_FIN) && 
+                               !(tp->t_flags & TF_SENTFIN)) {

                                tp->snd_nxt++;
                                tp->t_flags |= TF_SENTFIN;
                        }
                                tp->snd_nxt++;
                                tp->t_flags |= TF_SENTFIN;
                        }


@@ -1832,7 +1834,8 @@ send:
 timer:
                if (tp->t_timer[TCPT_REXMT] == 0 &&
                    ((sack_rxmit && tp->snd_nxt != tp->snd_max) ||
 timer:
                if (tp->t_timer[TCPT_REXMT] == 0 &&
                    ((sack_rxmit && tp->snd_nxt != tp->snd_max) ||

-                       tp->snd_nxt != tp->snd_una)) {
+                       tp->snd_nxt != tp->snd_una ||
+                       (flags & TH_FIN))) {

                        if (tp->t_timer[TCPT_PERSIST]) {
                                tp->t_timer[TCPT_PERSIST] = 0;
                                tp->t_rxtshift = 0;
                        if (tp->t_timer[TCPT_PERSIST]) {
                                tp->t_timer[TCPT_PERSIST] = 0;
                                tp->t_rxtshift = 0;


@@ -1849,7 +1852,8 @@ timer:
                int xlen = len;
                if (flags & TH_SYN)
                        ++xlen;
                int xlen = len;
                if (flags & TH_SYN)
                        ++xlen;

-               if (flags & TH_FIN) {
+               if ((flags & TH_FIN) && 
+                       !(tp->t_flags & TF_SENTFIN)) {

                        ++xlen;
                        tp->t_flags |= TF_SENTFIN;
                }
                        ++xlen;
                        tp->t_flags |= TF_SENTFIN;
                }

diff --git a/bsd/netinet/tcp_timer.c b/bsd/netinet/tcp_timer.c
index a855e6fa90a51a5e0518cf747eb70b4115862903..b1ac3138b186b3b102ab02419ae4442d0c95eff5 100644 (file)
--- a/bsd/netinet/tcp_timer.c
+++ b/bsd/netinet/tcp_timer.c
@@ -1,5 +1,5 @@
 /*
 /*

- * Copyright (c) 2000-2013 Apple Inc. All rights reserved.
+ * Copyright (c) 2000-2014 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 


@@ -282,6 +282,8 @@ timer_diff(uint32_t t1, uint32_t toff1, uint32_t t2, uint32_t toff2) {
 /* Returns true if the timer is on the timer list */
 #define TIMER_IS_ON_LIST(tp) ((tp)->t_flags & TF_TIMER_ONLIST)
 
 /* Returns true if the timer is on the timer list */
 #define TIMER_IS_ON_LIST(tp) ((tp)->t_flags & TF_TIMER_ONLIST)
 

+/* Run the TCP timerlist atleast once every hour */
+#define        TCP_TIMERLIST_MAX_OFFSET        (60 * 60 * TCP_RETRANSHZ)

 
 static void add_to_time_wait_locked(struct tcpcb *tp, uint32_t delay);
 void   add_to_time_wait(struct tcpcb *tp, uint32_t delay) ;
 
 static void add_to_time_wait_locked(struct tcpcb *tp, uint32_t delay);
 void   add_to_time_wait(struct tcpcb *tp, uint32_t delay) ;


@@ -1107,16 +1109,18 @@ need_to_resched_timerlist(uint32_t runtime, uint16_t index) {
        int32_t diff;
        boolean_t is_fast;
 
        int32_t diff;
        boolean_t is_fast;
 

-       if (runtime == 0 || index == TCPT_NONE)
+       if (index == TCPT_NONE)

                return FALSE;
        is_fast = !(IS_TIMER_SLOW(index));
 
        /* If the list is being processed then the state of the list is in flux.
         * In this case always acquire the lock and set the state correctly.
         */
                return FALSE;
        is_fast = !(IS_TIMER_SLOW(index));
 
        /* If the list is being processed then the state of the list is in flux.
         * In this case always acquire the lock and set the state correctly.
         */

-       if (listp->running) {
+       if (listp->running)

                return TRUE;
                return TRUE;

-       }
+
+       if (!listp->scheduled)
+               return (TRUE);

 
        diff = timer_diff(listp->runtime, 0, runtime, 0);
        if (diff <= 0) {
 
        diff = timer_diff(listp->runtime, 0, runtime, 0);
        if (diff <= 0) {


@@ -1143,12 +1147,16 @@ tcp_sched_timerlist(uint32_t offset)
 
        lck_mtx_assert(listp->mtx, LCK_MTX_ASSERT_OWNED);
 
 
        lck_mtx_assert(listp->mtx, LCK_MTX_ASSERT_OWNED);
 

+       offset = min(offset, TCP_TIMERLIST_MAX_OFFSET);

        listp->runtime = tcp_now + offset;
        listp->runtime = tcp_now + offset;

+       if (listp->runtime == 0)
+               listp->runtime++;

 
        clock_interval_to_deadline(offset, NSEC_PER_SEC / TCP_RETRANSHZ,
                &deadline);
 
        thread_call_enter_delayed(listp->call, deadline);
 
        clock_interval_to_deadline(offset, NSEC_PER_SEC / TCP_RETRANSHZ,
                &deadline);
 
        thread_call_enter_delayed(listp->call, deadline);

+       listp->scheduled = TRUE;

 }
 
 /* Function to run the timers for a connection.
 }
 
 /* Function to run the timers for a connection.


@@ -1188,11 +1196,9 @@ tcp_run_conn_timer(struct tcpcb *tp, uint16_t *next_index) {
          * with another thread that can cancel or reschedule the timer that is
          * about to run. Check if we need to run anything.
          */
          * with another thread that can cancel or reschedule the timer that is
          * about to run. Check if we need to run anything.
          */

-       index = tp->tentry.index;
-       timer_val = tp->t_timer[index];
-
-        if (index == TCPT_NONE || tp->tentry.runtime == 0) 
+       if ((index = tp->tentry.index) == TCPT_NONE)

                goto done;
                goto done;

+       timer_val = tp->t_timer[index];

 
        diff = timer_diff(tp->tentry.runtime, 0, tcp_now, 0);
        if (diff > 0) {
 
        diff = timer_diff(tp->tentry.runtime, 0, tcp_now, 0);
        if (diff > 0) {


@@ -1235,8 +1241,8 @@ tcp_run_conn_timer(struct tcpcb *tp, uint16_t *next_index) {
        tp->tentry.index = lo_index;
        if (lo_index != TCPT_NONE) {
                tp->tentry.runtime = tp->tentry.timer_start + tp->t_timer[lo_index];
        tp->tentry.index = lo_index;
        if (lo_index != TCPT_NONE) {
                tp->tentry.runtime = tp->tentry.timer_start + tp->t_timer[lo_index];

-       } else {
-               tp->tentry.runtime = 0;
+               if (tp->tentry.runtime == 0)
+                       tp->tentry.runtime++;

        }
 
        if (count > 0) {
        }
 
        if (count > 0) {


@@ -1245,8 +1251,11 @@ tcp_run_conn_timer(struct tcpcb *tp, uint16_t *next_index) {
                        if (needtorun[i]) {
                                tp->t_timer[i] = 0;
                                tp = tcp_timers(tp, i);
                        if (needtorun[i]) {
                                tp->t_timer[i] = 0;
                                tp = tcp_timers(tp, i);

-                               if (tp == NULL) 
+                               if (tp == NULL) {
+                                       offset = 0;
+                                       *(next_index) = TCPT_NONE;

                                        goto done;
                                        goto done;

+                               }

                        }
                }
                tcp_set_lotimer_index(tp);
                        }
                }
                tcp_set_lotimer_index(tp);


@@ -1260,6 +1269,7 @@ tcp_run_conn_timer(struct tcpcb *tp, uint16_t *next_index) {
 done:
        if (tp != NULL && tp->tentry.index == TCPT_NONE) {
                tcp_remove_timer(tp);
 done:
        if (tp != NULL && tp->tentry.index == TCPT_NONE) {
                tcp_remove_timer(tp);

+               offset = 0;

        }
         tcp_unlock(so, 1, 0);
         return offset;
        }
         tcp_unlock(so, 1, 0);
         return offset;


@@ -1288,7 +1298,7 @@ tcp_run_timerlist(void * arg1, void * arg2) {
        LIST_FOREACH_SAFE(te, &listp->lhead, le, next_te) {
                uint32_t offset = 0;
                uint32_t runtime = te->runtime;
        LIST_FOREACH_SAFE(te, &listp->lhead, le, next_te) {
                uint32_t offset = 0;
                uint32_t runtime = te->runtime;

-               if (TSTMP_GT(runtime, tcp_now)) {
+               if (te->index < TCPT_NONE && TSTMP_GT(runtime, tcp_now)) {

                        offset = timer_diff(runtime, 0, tcp_now, 0);
                        if (next_timer == 0 || offset < next_timer) {
                                next_timer = offset;
                        offset = timer_diff(runtime, 0, tcp_now, 0);
                        if (next_timer == 0 || offset < next_timer) {
                                next_timer = offset;


@@ -1384,8 +1394,11 @@ tcp_run_timerlist(void * arg1, void * arg2) {
 
                tcp_sched_timerlist(next_timer);
        } else {
 
                tcp_sched_timerlist(next_timer);
        } else {

-               /* No need to reschedule this timer */
-               listp->runtime = 0;
+               /*
+                * No need to reschedule this timer, but always run
+                * periodically at a much higher granularity.
+                */
+               tcp_sched_timerlist(TCP_TIMERLIST_MAX_OFFSET);

        }
 
        listp->running = FALSE;
        }
 
        listp->running = FALSE;


@@ -1402,7 +1415,7 @@ tcp_sched_timers(struct tcpcb *tp)
        struct tcptimerentry *te = &tp->tentry;
        uint16_t index = te->index;
        struct tcptimerlist *listp = &tcp_timer_list;
        struct tcptimerentry *te = &tp->tentry;
        uint16_t index = te->index;
        struct tcptimerlist *listp = &tcp_timer_list;

-       uint32_t offset = 0;
+       int32_t offset = 0;

        boolean_t is_fast;
        int list_locked = 0;
 
        boolean_t is_fast;
        int list_locked = 0;
 


@@ -1420,8 +1433,8 @@ tcp_sched_timers(struct tcpcb *tp)
        }
 
        is_fast = !(IS_TIMER_SLOW(index));
        }
 
        is_fast = !(IS_TIMER_SLOW(index));

-       offset = te->runtime - tcp_now;
-       if (offset == 0) {
+       offset = timer_diff(te->runtime, 0, tcp_now, 0);
+       if (offset <= 0) {

                offset = 1;
                tcp_timer_advanced++;
        }
                offset = 1;
                tcp_timer_advanced++;
        }


@@ -1442,7 +1455,7 @@ tcp_sched_timers(struct tcpcb *tp)
                        listp->maxentries = listp->entries;
 
                /* if the list is not scheduled, just schedule it */
                        listp->maxentries = listp->entries;
 
                /* if the list is not scheduled, just schedule it */

-               if (listp->runtime == 0)
+               if (!listp->scheduled)

                        goto schedule;
 
        }
                        goto schedule;
 
        }


@@ -1464,15 +1477,22 @@ tcp_sched_timers(struct tcpcb *tp)
                        if (is_fast) {
                                listp->pref_mode = TCP_TIMERLIST_FASTMODE;
                        } else if (listp->pref_offset == 0 ||
                        if (is_fast) {
                                listp->pref_mode = TCP_TIMERLIST_FASTMODE;
                        } else if (listp->pref_offset == 0 ||

-                               ((int)offset) < listp->pref_offset) {
+                               offset < listp->pref_offset) {

                                listp->pref_offset = offset;
                        }
                } else {
                                listp->pref_offset = offset;
                        }
                } else {

-                       int32_t diff;
-                       diff = timer_diff(listp->runtime, 0, tcp_now, offset);
-                       if (diff <= 0) {
-                               /* The list is going to run before this timer */
-                               goto done;
+                       /*
+                        * The list could have got scheduled while this
+                        * thread was waiting for the lock
+                        */
+                       if (listp->scheduled) {
+                               int32_t diff;
+                               diff = timer_diff(listp->runtime, 0,
+                                   tcp_now, offset);
+                               if (diff <= 0)
+                                       goto done;
+                               else
+                                       goto schedule;

                        } else {
                                goto schedule;
                        }
                        } else {
                                goto schedule;
                        }


@@ -1508,8 +1528,8 @@ tcp_set_lotimer_index(struct tcpcb *tp) {
        tp->tentry.index = lo_index;
        if (lo_index != TCPT_NONE) {
                tp->tentry.runtime = tp->tentry.timer_start + tp->t_timer[lo_index];
        tp->tentry.index = lo_index;
        if (lo_index != TCPT_NONE) {
                tp->tentry.runtime = tp->tentry.timer_start + tp->t_timer[lo_index];

-       } else {
-               tp->tentry.runtime = 0;
+               if (tp->tentry.runtime == 0)
+                       tp->tentry.runtime++;

        }
 }
 
        }
 }
 


@@ -1518,6 +1538,9 @@ tcp_check_timer_state(struct tcpcb *tp) {
 
        lck_mtx_assert(&tp->t_inpcb->inpcb_mtx, LCK_MTX_ASSERT_OWNED);
 
 
        lck_mtx_assert(&tp->t_inpcb->inpcb_mtx, LCK_MTX_ASSERT_OWNED);
 

+       if (tp->t_inpcb->inp_flags2 & INP2_TIMEWAIT)
+               return;
+

        tcp_set_lotimer_index(tp);
 
        tcp_sched_timers(tp);
        tcp_set_lotimer_index(tp);
 
        tcp_sched_timers(tp);

diff --git a/bsd/netinet/tcp_timer.h b/bsd/netinet/tcp_timer.h
index 55f4287076f91cd6e70ef7e5e14d78a29c8c3a8a..f7b775db5f23dc3b021e0b89afa6869bb031866b 100644 (file)
--- a/bsd/netinet/tcp_timer.h
+++ b/bsd/netinet/tcp_timer.h
@@ -1,5 +1,5 @@
 /*
 /*

- * Copyright (c) 2000-2010 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2014 Apple Computer, Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 


@@ -208,6 +208,7 @@ struct tcptimerlist {
 
        /* Set desired mode when timer list running */
        boolean_t running;              /* Set when timer list is being processed */
 
        /* Set desired mode when timer list running */
        boolean_t running;              /* Set when timer list is being processed */

+       boolean_t scheduled;            /* Set when timer is scheduled */

 #define TCP_TIMERLIST_FASTMODE 0x1
 #define TCP_TIMERLIST_SLOWMODE 0x2
        uint32_t mode;                  /* Current mode, fast or slow */
 #define TCP_TIMERLIST_FASTMODE 0x1
 #define TCP_TIMERLIST_SLOWMODE 0x2
        uint32_t mode;                  /* Current mode, fast or slow */

diff --git a/config/MasterVersion b/config/MasterVersion
index 394f474c1402d81bb73aa67a72ae32e39eb11d13..8fcb30c965ed71820041a1f6c345544268d51c5b 100644 (file)
--- a/config/MasterVersion
+++ b/config/MasterVersion
@@ -1,4 +1,4 @@
-13.2.0
+13.3.0

 
 # The first line of this file contains the master version number for the kernel.
 # All other instances of the kernel version in xnu are derived from this file.
 
 # The first line of this file contains the master version number for the kernel.
 # All other instances of the kernel version in xnu are derived from this file.

diff --git a/iokit/IOKit/IOHibernatePrivate.h b/iokit/IOKit/IOHibernatePrivate.h
index 525fb5d7a049cc4ebd0ec1231d2635d5d344568b..eddd86c05738131fa0229ccb20af4a9d7126b604 100644 (file)
--- a/iokit/IOKit/IOHibernatePrivate.h
+++ b/iokit/IOKit/IOHibernatePrivate.h
@@ -339,6 +339,9 @@ kern_close_file_for_direct_io(struct kern_direct_file_io_ref_t * ref,
                              off_t discard_offset, off_t discard_end);
 #endif /* _SYS_CONF_H_ */
 
                              off_t discard_offset, off_t discard_end);
 #endif /* _SYS_CONF_H_ */
 

+void
+vm_compressor_do_warmup(void);
+

 hibernate_page_list_t *
 hibernate_page_list_allocate(boolean_t log);
 
 hibernate_page_list_t *
 hibernate_page_list_allocate(boolean_t log);
 

diff --git a/iokit/Kernel/IOHibernateIO.cpp b/iokit/Kernel/IOHibernateIO.cpp
index 3bd133ebac90398348579beafb3e70f45c899272..851adac6035d329bc50785a11bf56e6cfbac0d96 100644 (file)
--- a/iokit/Kernel/IOHibernateIO.cpp
+++ b/iokit/Kernel/IOHibernateIO.cpp
@@ -1262,11 +1262,13 @@ IOHibernateSystemSleep(void)
            bzero(&consoleInfo, sizeof(consoleInfo));
            IOService::getPlatform()->getConsoleInfo(&consoleInfo);
 
            bzero(&consoleInfo, sizeof(consoleInfo));
            IOService::getPlatform()->getConsoleInfo(&consoleInfo);
 

-           // estimate: 5% increase in pages compressed
+           // estimate: 6% increase in pages compressed

            // screen preview 2 images compressed 50%
            // screen preview 2 images compressed 50%

-           setFileSize = ((ptoa_64((105 * pageCount) / 100) * gIOHibernateCompression) >> 8)
+           setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8)

                                + vars->page_list->list_size
                                + vars->page_list->list_size

-                               + (consoleInfo.v_width * consoleInfo.v_height * 4);
+                               + (consoleInfo.v_width * consoleInfo.v_height * 8);
+           enum { setFileRound = 1024*1024ULL };
+           setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1));

 
            HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n", 
                    pageCount, (100ULL * gIOHibernateCompression) >> 8,
 
            HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n", 
                    pageCount, (100ULL * gIOHibernateCompression) >> 8,


@@ -1284,6 +1286,7 @@ IOHibernateSystemSleep(void)
         err = IOPolledFileOpen(gIOHibernateFilename, setFileSize, vars->ioBuffer,
                                 &vars->fileVars, &vars->fileExtents, &data, 
                                 &vars->volumeCryptKey[0]);
         err = IOPolledFileOpen(gIOHibernateFilename, setFileSize, vars->ioBuffer,
                                 &vars->fileVars, &vars->fileExtents, &data, 
                                 &vars->volumeCryptKey[0]);

+

         if (KERN_SUCCESS != err)
         {
            HIBLOG("IOPolledFileOpen(%x)\n", err);
         if (KERN_SUCCESS != err)
         {
            HIBLOG("IOPolledFileOpen(%x)\n", err);


@@ -2658,7 +2661,14 @@ hibernate_write_image(void)
             }
         }
         if (kIOReturnSuccess != err)
             }
         }
         if (kIOReturnSuccess != err)

+        {
+            if (kIOReturnOverrun == err)
+            {
+               // update actual compression ratio on not enough space
+                gIOHibernateCompression = (compressedSize << 8) / uncompressedSize;
+            }

             break;
             break;

+        }

 
         // Header:
     
 
         // Header:
     


@@ -3113,6 +3123,12 @@ void IOHibernateSetWakeCapabilities(uint32_t capability)
     if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
     {
        gIOHibernateStats->wakeCapability = capability;
     if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
     {
        gIOHibernateStats->wakeCapability = capability;

+
+       if (kIOPMSystemCapabilityGraphics & capability)
+       {
+               vm_compressor_do_warmup();
+       }
+

     }
 }
 
     }
 }
 

diff --git a/iokit/Kernel/IOPMrootDomain.cpp b/iokit/Kernel/IOPMrootDomain.cpp
index 60f715588df80a1cadccc4adf36d48bf15e70e84..7221295fa33eced5a72a75b46385170a76ddfcf3 100644 (file)
--- a/iokit/Kernel/IOPMrootDomain.cpp
+++ b/iokit/Kernel/IOPMrootDomain.cpp
@@ -6880,6 +6880,9 @@ void IOPMrootDomain::willEnterFullWake( void )
             (kFullWakeReasonLocalUser == fullWakeReason) ?
                 kOSBooleanTrue : kOSBooleanFalse);
     }
             (kFullWakeReasonLocalUser == fullWakeReason) ?
                 kOSBooleanTrue : kOSBooleanFalse);
     }

+#if HIBERNATION
+    IOHibernateSetWakeCapabilities(_pendingCapability);
+#endif

 
     IOService::setAdvisoryTickleEnable( true );
     tellClients(kIOMessageSystemWillPowerOn);
 
     IOService::setAdvisoryTickleEnable( true );
     tellClients(kIOMessageSystemWillPowerOn);

diff --git a/osfmk/kern/hibernate.c b/osfmk/kern/hibernate.c
index 7b3d385633c132d871c0241a89e5d4294e1bf46a..8d0806589b0879f78954256436c6190c7b21288c 100644 (file)
--- a/osfmk/kern/hibernate.c
+++ b/osfmk/kern/hibernate.c
@@ -160,7 +160,7 @@ hibernate_teardown(hibernate_page_list_t * page_list,
                    need_to_unlock_decompressor = FALSE;
                    vm_decompressor_unlock();
            }
                    need_to_unlock_decompressor = FALSE;
                    vm_decompressor_unlock();
            }

-           vm_compressor_do_warmup();
+           vm_compressor_delay_trim();

     }
     return (KERN_SUCCESS);
 }
     }
     return (KERN_SUCCESS);
 }

diff --git a/osfmk/vm/vm_compressor.c b/osfmk/vm/vm_compressor.c
index ca9ff6c6d69a97e2b08ecf91274db0c9bb93bab5..bc565e3f1c05ad8585bb822289aa6fcbd690e625 100644 (file)
--- a/osfmk/vm/vm_compressor.c
+++ b/osfmk/vm/vm_compressor.c
@@ -775,7 +775,8 @@ c_seg_free_locked(c_segment_t c_seg)
        lck_mtx_unlock_always(&c_seg->c_lock);
 
        if (c_buffer) {
        lck_mtx_unlock_always(&c_seg->c_lock);
 
        if (c_buffer) {

-               kernel_memory_depopulate(kernel_map, (vm_offset_t) c_buffer, pages_populated * PAGE_SIZE, KMA_COMPRESSOR);
+               if (pages_populated)
+                       kernel_memory_depopulate(kernel_map, (vm_offset_t) c_buffer, pages_populated * PAGE_SIZE, KMA_COMPRESSOR);

 
                kmem_free(kernel_map, (vm_offset_t) c_buffer, C_SEG_ALLOCSIZE);
        } else if (c_swap_handle)
 
                kmem_free(kernel_map, (vm_offset_t) c_buffer, C_SEG_ALLOCSIZE);
        } else if (c_swap_handle)


@@ -1450,6 +1451,15 @@ vm_compressor_do_delayed_compactions(boolean_t flush_all)
                c_seg = (c_segment_t)queue_first(&c_minor_list_head);
                
                lck_mtx_lock_spin_always(&c_seg->c_lock);
                c_seg = (c_segment_t)queue_first(&c_minor_list_head);
                
                lck_mtx_lock_spin_always(&c_seg->c_lock);

+
+                if (c_seg->c_busy) {
+
+                        lck_mtx_unlock_always(c_list_lock);
+                        c_seg_wait_on_busy(c_seg);
+                        lck_mtx_lock_spin_always(c_list_lock);
+
+                        continue;
+                }

                c_seg->c_busy = 1;
 
                c_seg_do_minor_compaction_and_unlock(c_seg, TRUE, FALSE, TRUE);
                c_seg->c_busy = 1;
 
                c_seg_do_minor_compaction_and_unlock(c_seg, TRUE, FALSE, TRUE);


@@ -1604,16 +1614,17 @@ vm_compressor_record_warmup_start(void)
 
        lck_mtx_lock_spin_always(c_list_lock);
 
 
        lck_mtx_lock_spin_always(c_list_lock);
 

-       if (!queue_empty(&c_age_list_head)) {
-
-               c_seg = (c_segment_t)queue_last(&c_age_list_head);
+       if (first_c_segment_to_warm_generation_id == 0) {
+               if (!queue_empty(&c_age_list_head)) {

 
 

-               first_c_segment_to_warm_generation_id = c_seg->c_generation_id;
-       } else
-               first_c_segment_to_warm_generation_id = 0;
+                       c_seg = (c_segment_t)queue_last(&c_age_list_head);

 
 

-       fastwake_recording_in_progress = TRUE;
+                       first_c_segment_to_warm_generation_id = c_seg->c_generation_id;
+               } else
+                       first_c_segment_to_warm_generation_id = 0;

 
 

+               fastwake_recording_in_progress = TRUE;
+       }

        lck_mtx_unlock_always(c_list_lock);
 }
 
        lck_mtx_unlock_always(c_list_lock);
 }
 


@@ -1625,16 +1636,20 @@ vm_compressor_record_warmup_end(void)
 
        lck_mtx_lock_spin_always(c_list_lock);
 
 
        lck_mtx_lock_spin_always(c_list_lock);
 

-       if (!queue_empty(&c_age_list_head)) {
+       if (fastwake_recording_in_progress == TRUE) {

 
 

-               c_seg = (c_segment_t)queue_last(&c_age_list_head);
+               if (!queue_empty(&c_age_list_head)) {

 
 

-               last_c_segment_to_warm_generation_id = c_seg->c_generation_id;
-       } else
-               last_c_segment_to_warm_generation_id = first_c_segment_to_warm_generation_id;
+                       c_seg = (c_segment_t)queue_last(&c_age_list_head);
+
+                       last_c_segment_to_warm_generation_id = c_seg->c_generation_id;
+               } else
+                       last_c_segment_to_warm_generation_id = first_c_segment_to_warm_generation_id;

 
 

-       fastwake_recording_in_progress = FALSE;
+               fastwake_recording_in_progress = FALSE;

 
 

+               HIBLOG("vm_compressor_record_warmup (%qd - %qd)\n", first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id);
+       }

        lck_mtx_unlock_always(c_list_lock);
 }
 
        lck_mtx_unlock_always(c_list_lock);
 }
 


@@ -1642,19 +1657,28 @@ vm_compressor_record_warmup_end(void)
 #define DELAY_TRIM_ON_WAKE_SECS                4
 
 void
 #define DELAY_TRIM_ON_WAKE_SECS                4
 
 void

-vm_compressor_do_warmup(void)
+vm_compressor_delay_trim(void)

 {
 {

-       clock_sec_t     sec;
+        clock_sec_t    sec;

        clock_nsec_t    nsec;
 
        clock_get_system_nanotime(&sec, &nsec);
        dont_trim_until_ts = sec + DELAY_TRIM_ON_WAKE_SECS;
        clock_nsec_t    nsec;
 
        clock_get_system_nanotime(&sec, &nsec);
        dont_trim_until_ts = sec + DELAY_TRIM_ON_WAKE_SECS;

+}

 
 

-       if (first_c_segment_to_warm_generation_id == last_c_segment_to_warm_generation_id)
-               return;

 
 

+void
+vm_compressor_do_warmup(void)
+{

        lck_mtx_lock_spin_always(c_list_lock);
        
        lck_mtx_lock_spin_always(c_list_lock);
        

+       if (first_c_segment_to_warm_generation_id == last_c_segment_to_warm_generation_id) {
+               first_c_segment_to_warm_generation_id = last_c_segment_to_warm_generation_id = 0;
+
+               lck_mtx_unlock_always(c_list_lock);
+               return;
+       }
+

        if (compaction_swapper_running == 0) {
 
                fastwake_warmup = TRUE;
        if (compaction_swapper_running == 0) {
 
                fastwake_warmup = TRUE;


@@ -1670,6 +1694,13 @@ do_fastwake_warmup(void)
 {
        uint64_t        my_thread_id;
        c_segment_t     c_seg = NULL;
 {
        uint64_t        my_thread_id;
        c_segment_t     c_seg = NULL;

+       AbsoluteTime    startTime, endTime;
+       uint64_t        nsec;
+
+
+       HIBLOG("vm_compressor_fastwake_warmup (%qd - %qd) - starting\n", first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id);
+
+       clock_get_uptime(&startTime);

 
        lck_mtx_unlock_always(c_list_lock);
 
 
        lck_mtx_unlock_always(c_list_lock);
 


@@ -1692,15 +1723,19 @@ do_fastwake_warmup(void)
                lck_mtx_lock_spin_always(&c_seg->c_lock);
                lck_mtx_unlock_always(c_list_lock);
                
                lck_mtx_lock_spin_always(&c_seg->c_lock);
                lck_mtx_unlock_always(c_list_lock);
                

-               if (c_seg->c_busy)
+               if (c_seg->c_busy) {
+                       PAGE_REPLACEMENT_DISALLOWED(FALSE);

                        c_seg_wait_on_busy(c_seg);
                        c_seg_wait_on_busy(c_seg);

-               else {
+                       PAGE_REPLACEMENT_DISALLOWED(TRUE);
+               } else {

                        c_seg_swapin(c_seg, TRUE);
 
                        lck_mtx_unlock_always(&c_seg->c_lock);
                        c_seg_swapin(c_seg, TRUE);
 
                        lck_mtx_unlock_always(&c_seg->c_lock);

-

                        c_segment_warmup_count++;
                        c_segment_warmup_count++;

+
+                       PAGE_REPLACEMENT_DISALLOWED(FALSE);

                        vm_pageout_io_throttle();
                        vm_pageout_io_throttle();

+                       PAGE_REPLACEMENT_DISALLOWED(TRUE);

                }
                lck_mtx_lock_spin_always(c_list_lock);
        }
                }
                lck_mtx_lock_spin_always(c_list_lock);
        }


@@ -1711,7 +1746,15 @@ do_fastwake_warmup(void)
        proc_set_task_policy_thread(kernel_task, my_thread_id,
                                    TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER0);
 
        proc_set_task_policy_thread(kernel_task, my_thread_id,
                                    TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER0);
 

+        clock_get_uptime(&endTime);
+        SUB_ABSOLUTETIME(&endTime, &startTime);
+        absolutetime_to_nanoseconds(endTime, &nsec);
+
+       HIBLOG("vm_compressor_fastwake_warmup completed - took %qd msecs\n", nsec / 1000000ULL);
+

        lck_mtx_lock_spin_always(c_list_lock);
        lck_mtx_lock_spin_always(c_list_lock);

+
+       first_c_segment_to_warm_generation_id = last_c_segment_to_warm_generation_id = 0;

 }
 
 
 }
 
 


@@ -1735,6 +1778,18 @@ vm_compressor_compact_and_swap(boolean_t flush_all)
                fastwake_warmup = FALSE;
        }
 
                fastwake_warmup = FALSE;
        }
 

+       /*
+        * it's possible for the c_age_list_head to be empty if we
+        * hit our limits for growing the compressor pool and we subsequently
+        * hibernated... on the next hibernation we could see the queue as
+        * empty and not proceeed even though we have a bunch of segments on
+        * the swapped in queue that need to be dealt with.
+        */
+       vm_compressor_do_delayed_compactions(flush_all);
+
+       vm_compressor_age_swapped_in_segments(flush_all);
+
+

        while (!queue_empty(&c_age_list_head) && compaction_swapper_abort == 0) {
 
                if (hibernate_flushing == TRUE) {
        while (!queue_empty(&c_age_list_head) && compaction_swapper_abort == 0) {
 
                if (hibernate_flushing == TRUE) {


@@ -2485,13 +2540,28 @@ c_seg_invalid_data:
        }
        if (!c_seg->c_filling) {
                if (c_seg->c_bytes_used == 0) {
        }
        if (!c_seg->c_filling) {
                if (c_seg->c_bytes_used == 0) {

-                       if (c_seg->c_on_minorcompact_q || c_seg->c_on_swappedout_sparse_q) {
-                               if (c_seg_try_free(c_seg) == TRUE)
-                                       need_unlock = FALSE;
-                       } else {
-                               c_seg_free(c_seg);
-                               need_unlock = FALSE;
-                       }
+                       if (!c_seg->c_ondisk) {
+                               int     pages_populated;
+
+                               pages_populated = (round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE;
+                               c_seg->c_populated_offset = C_SEG_BYTES_TO_OFFSET(0);
+
+                               if (pages_populated) {
+                                       assert(c_seg->c_store.c_buffer != NULL);
+
+                                       c_seg->c_busy = 1;
+                                       lck_mtx_unlock_always(&c_seg->c_lock);
+
+                                       kernel_memory_depopulate(kernel_map, (vm_offset_t) c_seg->c_store.c_buffer, pages_populated * PAGE_SIZE, KMA_COMPRESSOR);
+
+                                       lck_mtx_lock_spin_always(&c_seg->c_lock);
+                                       C_SEG_WAKEUP_DONE(c_seg);
+                               }
+                               if (!c_seg->c_on_minorcompact_q && !c_seg->c_on_swapout_q)
+                                       c_seg_need_delayed_compaction(c_seg);
+                       } else
+                               assert(c_seg->c_on_swappedout_sparse_q);
+

                } else if (c_seg->c_on_minorcompact_q) {
 
                        if (C_SEG_INCORE_IS_SPARSE(c_seg)) {
                } else if (c_seg->c_on_minorcompact_q) {
 
                        if (C_SEG_INCORE_IS_SPARSE(c_seg)) {

diff --git a/osfmk/vm/vm_compressor.h b/osfmk/vm/vm_compressor.h
index c2d85ab0897a897cf311ae24c78960a59cc5be3a..4721f3e321cc3cd1662a7df95c08a9fd4976c858 100644 (file)
--- a/osfmk/vm/vm_compressor.h
+++ b/osfmk/vm/vm_compressor.h
@@ -155,6 +155,7 @@ void c_seg_insert_into_age_q(c_segment_t);
 void vm_decompressor_lock(void);
 void vm_decompressor_unlock(void);
 
 void vm_decompressor_lock(void);
 void vm_decompressor_unlock(void);
 

+void vm_compressor_delay_trim(void);

 void vm_compressor_do_warmup(void);
 void vm_compressor_record_warmup_start(void);
 void vm_compressor_record_warmup_end(void);
 void vm_compressor_do_warmup(void);
 void vm_compressor_record_warmup_start(void);
 void vm_compressor_record_warmup_end(void);

diff --git a/osfmk/vm/vm_compressor_backing_store.c b/osfmk/vm/vm_compressor_backing_store.c
index 80db056d4031928aec436c3dadbe1c81b97083a2..bf57f06e54db1b3f32a9162e2525d5ddf484ce1a 100644 (file)
--- a/osfmk/vm/vm_compressor_backing_store.c
+++ b/osfmk/vm/vm_compressor_backing_store.c
@@ -720,10 +720,9 @@ vm_swapout_thread(void)
                assert(c_seg->c_on_swapout_q);
 
                if (c_seg->c_busy) {
                assert(c_seg->c_on_swapout_q);
 
                if (c_seg->c_busy) {

-                       lck_mtx_unlock_always(&c_seg->c_lock);

                        lck_mtx_unlock_always(c_list_lock);
 
                        lck_mtx_unlock_always(c_list_lock);
 

-                       mutex_pause(2);
+                       c_seg_wait_on_busy(c_seg);

 
                        lck_mtx_lock_spin_always(c_list_lock);
 
 
                        lck_mtx_lock_spin_always(c_list_lock);
 


@@ -733,19 +732,23 @@ vm_swapout_thread(void)
                c_seg->c_on_swapout_q = 0;
                c_swapout_count--;
 
                c_seg->c_on_swapout_q = 0;
                c_swapout_count--;
 

-               c_seg->c_busy = 1;
-               c_seg->c_busy_swapping = 1;
-

                vm_swapout_thread_processed_segments++;
 
                thread_wakeup((event_t)&compaction_swapper_running);
 
                vm_swapout_thread_processed_segments++;
 
                thread_wakeup((event_t)&compaction_swapper_running);
 

+               size = round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset));
+               
+               if (size == 0) {
+                       c_seg_free_locked(c_seg);
+                       goto c_seg_was_freed;
+               }
+               c_seg->c_busy = 1;
+               c_seg->c_busy_swapping = 1;
+

                lck_mtx_unlock_always(c_list_lock);
 
                addr = (vm_offset_t) c_seg->c_store.c_buffer;
 
                lck_mtx_unlock_always(c_list_lock);
 
                addr = (vm_offset_t) c_seg->c_store.c_buffer;
 

-               size = round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset));
-               

                lck_mtx_unlock_always(&c_seg->c_lock);
 
 #if CHECKSUM_THE_SWAP  
                lck_mtx_unlock_always(&c_seg->c_lock);
 
 #if CHECKSUM_THE_SWAP  


@@ -820,7 +823,7 @@ vm_swapout_thread(void)
                        kmem_free(kernel_map, (vm_offset_t) addr, C_SEG_ALLOCSIZE);
 
                vm_pageout_io_throttle();
                        kmem_free(kernel_map, (vm_offset_t) addr, C_SEG_ALLOCSIZE);
 
                vm_pageout_io_throttle();

-
+c_seg_was_freed:

                if (c_swapout_count == 0)
                        vm_swap_consider_defragmenting();
 
                if (c_swapout_count == 0)
                        vm_swap_consider_defragmenting();
 

diff --git a/osfmk/vm/vm_resident.c b/osfmk/vm/vm_resident.c
index a9fd4d0b24d9eb6bc22f8416f505b402cc3f91a4..32271953ab9f3cb2be1f2ada190593db18f6a6fa 100644 (file)
--- a/osfmk/vm/vm_resident.c
+++ b/osfmk/vm/vm_resident.c
@@ -4809,6 +4809,7 @@ struct hibernate_statistics {
        int cd_found_laundry;
        int cd_found_dirty;
        int cd_found_xpmapped;
        int cd_found_laundry;
        int cd_found_dirty;
        int cd_found_xpmapped;

+       int cd_skipped_xpmapped;

        int cd_local_free;
        int cd_total_free;
        int cd_vm_page_wire_count;
        int cd_local_free;
        int cd_total_free;
        int cd_vm_page_wire_count;


@@ -4819,6 +4820,13 @@ struct hibernate_statistics {
 } hibernate_stats;
 
 
 } hibernate_stats;
 
 

+/*
+ * clamp the number of 'xpmapped' pages we'll sweep into the hibernation image
+ * so that we don't overrun the estimated image size, which would
+ * result in a hibernation failure.
+ */
+#define        HIBERNATE_XPMAPPED_LIMIT        40000
+

 
 static int
 hibernate_drain_pageout_queue(struct vm_pageout_queue *q)
 
 static int
 hibernate_drain_pageout_queue(struct vm_pageout_queue *q)


@@ -5360,10 +5368,15 @@ hibernate_consider_discard(vm_page_t m, boolean_t preflight)
         if (discard == FALSE) {
                if (!preflight)
                        hibernate_stats.cd_found_dirty++;
         if (discard == FALSE) {
                if (!preflight)
                        hibernate_stats.cd_found_dirty++;

-        } else if (m->xpmapped && m->reference) {
-                if (!preflight)
-                        hibernate_stats.cd_found_xpmapped++;
-                discard = FALSE;
+        } else if (m->xpmapped && m->reference && !object->internal) {
+               if (hibernate_stats.cd_found_xpmapped < HIBERNATE_XPMAPPED_LIMIT) {
+                       if (!preflight)
+                               hibernate_stats.cd_found_xpmapped++;
+                       discard = FALSE;
+               } else {
+                       if (!preflight)
+                               hibernate_stats.cd_skipped_xpmapped++;
+               }

         }
     }
     while (FALSE);
         }
     }
     while (FALSE);


@@ -5650,8 +5663,8 @@ hibernate_page_list_setall(hibernate_page_list_t * page_list,
        m = next;
     }
 
        m = next;
     }
 

-    m = (vm_page_t) queue_first(&vm_page_queue_inactive);
-    while (m && !queue_end(&vm_page_queue_inactive, (queue_entry_t)m))
+    m = (vm_page_t) queue_first(&vm_page_queue_cleaned);
+    while (m && !queue_end(&vm_page_queue_cleaned, (queue_entry_t)m))

     {
         next = (vm_page_t) m->pageq.next;
        discard = FALSE;
     {
         next = (vm_page_t) m->pageq.next;
        discard = FALSE;


@@ -5662,19 +5675,42 @@ hibernate_page_list_setall(hibernate_page_list_t * page_list,
            if (m->dirty)
                count_discard_purgeable++;
            else
            if (m->dirty)
                count_discard_purgeable++;
            else

-               count_discard_inactive++;
+               count_discard_cleaned++;

             discard = discard_all;
         }
         else
             discard = discard_all;
         }
         else

-            count_inactive++;
+            count_cleaned++;

        count_wire--;
        if (!preflight) hibernate_page_bitset(page_list_wired, TRUE, m->phys_page);
         if (discard)    hibernate_discard_page(m);
        m = next;
     }
 
        count_wire--;
        if (!preflight) hibernate_page_bitset(page_list_wired, TRUE, m->phys_page);
         if (discard)    hibernate_discard_page(m);
        m = next;
     }
 

-    m = (vm_page_t) queue_first(&vm_page_queue_cleaned);
-    while (m && !queue_end(&vm_page_queue_cleaned, (queue_entry_t)m))
+    m = (vm_page_t) queue_first(&vm_page_queue_active);
+    while (m && !queue_end(&vm_page_queue_active, (queue_entry_t)m))
+    {
+        next = (vm_page_t) m->pageq.next;
+       discard = FALSE;
+        if ((kIOHibernateModeDiscardCleanActive & gIOHibernateMode) 
+         && hibernate_consider_discard(m, preflight))
+        {
+            if (!preflight) hibernate_page_bitset(page_list, TRUE, m->phys_page);
+           if (m->dirty)
+               count_discard_purgeable++;
+           else
+               count_discard_active++;
+            discard = discard_all;
+        }
+        else
+            count_active++;
+       count_wire--;
+       if (!preflight) hibernate_page_bitset(page_list_wired, TRUE, m->phys_page);
+        if (discard)    hibernate_discard_page(m);
+       m = next;
+    }
+
+    m = (vm_page_t) queue_first(&vm_page_queue_inactive);
+    while (m && !queue_end(&vm_page_queue_inactive, (queue_entry_t)m))

     {
         next = (vm_page_t) m->pageq.next;
        discard = FALSE;
     {
         next = (vm_page_t) m->pageq.next;
        discard = FALSE;


@@ -5685,11 +5721,11 @@ hibernate_page_list_setall(hibernate_page_list_t * page_list,
            if (m->dirty)
                count_discard_purgeable++;
            else
            if (m->dirty)
                count_discard_purgeable++;
            else

-               count_discard_cleaned++;
+               count_discard_inactive++;

             discard = discard_all;
         }
         else
             discard = discard_all;
         }
         else

-            count_cleaned++;
+            count_inactive++;

        count_wire--;
        if (!preflight) hibernate_page_bitset(page_list_wired, TRUE, m->phys_page);
         if (discard)    hibernate_discard_page(m);
        count_wire--;
        if (!preflight) hibernate_page_bitset(page_list_wired, TRUE, m->phys_page);
         if (discard)    hibernate_discard_page(m);


@@ -5719,29 +5755,6 @@ hibernate_page_list_setall(hibernate_page_list_t * page_list,
        }
     }
 
        }
     }
 

-    m = (vm_page_t) queue_first(&vm_page_queue_active);
-    while (m && !queue_end(&vm_page_queue_active, (queue_entry_t)m))
-    {
-        next = (vm_page_t) m->pageq.next;
-       discard = FALSE;
-        if ((kIOHibernateModeDiscardCleanActive & gIOHibernateMode) 
-         && hibernate_consider_discard(m, preflight))
-        {
-            if (!preflight) hibernate_page_bitset(page_list, TRUE, m->phys_page);
-           if (m->dirty)
-               count_discard_purgeable++;
-           else
-               count_discard_active++;
-            discard = discard_all;
-        }
-        else
-            count_active++;
-       count_wire--;
-       if (!preflight) hibernate_page_bitset(page_list_wired, TRUE, m->phys_page);
-        if (discard)    hibernate_discard_page(m);
-       m = next;
-    }
-

     queue_iterate(&compressor_object->memq, m, vm_page_t, listq)
     {
         count_compressor++;
     queue_iterate(&compressor_object->memq, m, vm_page_t, listq)
     {
         count_compressor++;


@@ -5795,6 +5808,9 @@ hibernate_page_list_setall(hibernate_page_list_t * page_list,
                discard_all ? "did" : "could",
                count_discard_active, count_discard_inactive, count_discard_purgeable, count_discard_speculative, count_discard_cleaned);
 
                discard_all ? "did" : "could",
                count_discard_active, count_discard_inactive, count_discard_purgeable, count_discard_speculative, count_discard_cleaned);
 

+    if (hibernate_stats.cd_skipped_xpmapped)
+           HIBLOG("WARNING: hibernate_page_list_setall skipped %d xpmapped pages\n", hibernate_stats.cd_skipped_xpmapped);
+

     *pagesOut = pages - count_discard_active - count_discard_inactive - count_discard_purgeable - count_discard_speculative - count_discard_cleaned;
 
     if (preflight && will_discard) *pagesOut -= count_compressor + count_throttled + count_anonymous + count_inactive + count_cleaned + count_speculative + count_active;
     *pagesOut = pages - count_discard_active - count_discard_inactive - count_discard_purgeable - count_discard_speculative - count_discard_cleaned;
 
     if (preflight && will_discard) *pagesOut -= count_compressor + count_throttled + count_anonymous + count_inactive + count_cleaned + count_speculative + count_active;