]> git.saurik.com Git - apple/xnu.git/blobdiff - osfmk/vm/vm_compressor.c
xnu-7195.101.1.tar.gz
[apple/xnu.git] / osfmk / vm / vm_compressor.c
index 64b7d7bd9be50829648fa8d1641435c6b86fb8eb..315c8b429173900981277393e75256c411f3be3e 100644 (file)
@@ -1,8 +1,8 @@
 /*
 /*
- * Copyright (c) 2000-2013 Apple Inc. All rights reserved.
+ * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
- * 
+ *
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.
- * 
+ *
  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.
- * 
+ *
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
@@ -22,7 +22,7 @@
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.
- * 
+ *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <vm/vm_map.h>
 #include <vm/vm_pageout.h>
 #include <vm/memory_object.h>
 #include <vm/vm_map.h>
 #include <vm/vm_pageout.h>
 #include <vm/memory_object.h>
-#include <mach/mach_host.h>            /* for host_info() */
+#include <vm/vm_compressor_algorithms.h>
+#include <vm/vm_fault.h>
+#include <vm/vm_protos.h>
+#include <mach/mach_host.h>             /* for host_info() */
 #include <kern/ledger.h>
 #include <kern/ledger.h>
+#include <kern/policy_internal.h>
+#include <kern/thread_group.h>
+#include <san/kasan.h>
 
 
+#if defined(__x86_64__)
 #include <i386/misc_protos.h>
 #include <i386/misc_protos.h>
-
-#include <default_pager/default_pager_alerts.h>
-#include <default_pager/default_pager_object_server.h>
+#endif
+#if defined(__arm64__)
+#include <arm/machine_routines.h>
+#endif
 
 #include <IOKit/IOHibernatePrivate.h>
 
 
 #include <IOKit/IOHibernatePrivate.h>
 
+extern boolean_t vm_darkwake_mode;
+extern zone_t vm_page_zone;
+
+#if DEVELOPMENT || DEBUG
+/* sysctl defined in bsd/dev/arm64/sysctl.c */
+int do_cseg_wedge_thread(void);
+int do_cseg_unwedge_thread(void);
+static event_t debug_cseg_wait_event = NULL;
+#endif /* DEVELOPMENT || DEBUG */
+
+#if CONFIG_FREEZE
+bool freezer_incore_cseg_acct = TRUE; /* Only count incore compressed memory for jetsams. */
+void task_disown_frozen_csegs(task_t owner_task);
+#endif /* CONFIG_FREEZE */
+
+#if POPCOUNT_THE_COMPRESSED_DATA
+boolean_t popcount_c_segs = TRUE;
+
+static inline uint32_t
+vmc_pop(uintptr_t ins, int sz)
+{
+       uint32_t rv = 0;
+
+       if (__probable(popcount_c_segs == FALSE)) {
+               return 0xDEAD707C;
+       }
+
+       while (sz >= 16) {
+               uint32_t rv1, rv2;
+               uint64_t *ins64 = (uint64_t *) ins;
+               uint64_t *ins642 = (uint64_t *) (ins + 8);
+               rv1 = __builtin_popcountll(*ins64);
+               rv2 = __builtin_popcountll(*ins642);
+               rv += rv1 + rv2;
+               sz -= 16;
+               ins += 16;
+       }
+
+       while (sz >= 4) {
+               uint32_t *ins32 = (uint32_t *) ins;
+               rv += __builtin_popcount(*ins32);
+               sz -= 4;
+               ins += 4;
+       }
+
+       while (sz > 0) {
+               char *ins8 = (char *)ins;
+               rv += __builtin_popcount(*ins8);
+               sz--;
+               ins++;
+       }
+       return rv;
+}
+#endif
+
+#if VALIDATE_C_SEGMENTS
+boolean_t validate_c_segs = TRUE;
+#endif
 /*
  * vm_compressor_mode has a heirarchy of control to set its value.
  * boot-args are checked first, then device-tree, and finally
 /*
  * vm_compressor_mode has a heirarchy of control to set its value.
  * boot-args are checked first, then device-tree, and finally
  * the boot-arg & device-tree code.
  */
 
  * the boot-arg & device-tree code.
  */
 
+#if !XNU_TARGET_OS_OSX
 
 
-int            vm_compressor_mode = VM_PAGER_COMPRESSOR_WITH_SWAP;
-int            vm_scale = 16;
+#if CONFIG_FREEZE
+int     vm_compressor_mode = VM_PAGER_FREEZER_DEFAULT;
+struct  freezer_context freezer_context_global;
+#else /* CONFIG_FREEZE */
+int     vm_compressor_mode = VM_PAGER_NOT_CONFIGURED;
+#endif /* CONFIG_FREEZE */
 
 
+#else /* !XNU_TARGET_OS_OSX */
+int             vm_compressor_mode = VM_PAGER_COMPRESSOR_WITH_SWAP;
 
 
-int            vm_compressor_is_active = 0;
-int            vm_compression_limit = 0;
-int            vm_compressor_available = 0;
+#endif /* !XNU_TARGET_OS_OSX */
 
 
-extern boolean_t vm_swap_up;
-extern void    vm_pageout_io_throttle(void);
-extern int     not_in_kdp;
+TUNABLE(uint32_t, vm_compression_limit, "vm_compression_limit", 0);
+int             vm_compressor_is_active = 0;
+int             vm_compressor_available = 0;
+
+extern uint64_t vm_swap_get_max_configured_space(void);
+extern void     vm_pageout_io_throttle(void);
 
 #if CHECKSUM_THE_DATA || CHECKSUM_THE_SWAP || CHECKSUM_THE_COMPRESSED_DATA
 extern unsigned int hash_string(char *cp, int len);
 
 #if CHECKSUM_THE_DATA || CHECKSUM_THE_SWAP || CHECKSUM_THE_COMPRESSED_DATA
 extern unsigned int hash_string(char *cp, int len);
+static unsigned int vmc_hash(char *, int);
+boolean_t checksum_c_segs = TRUE;
+
+unsigned int
+vmc_hash(char *cp, int len)
+{
+       if (__probable(checksum_c_segs == FALSE)) {
+               return 0xDEAD7A37;
+       }
+       return hash_string(cp, len);
+}
 #endif
 
 #endif
 
-#define UNPACK_C_SIZE(cs)      ((cs->c_size == (PAGE_SIZE-1)) ? PAGE_SIZE : cs->c_size)
-#define PACK_C_SIZE(cs, size)  (cs->c_size = ((size == PAGE_SIZE) ? PAGE_SIZE - 1 : size))
+#define UNPACK_C_SIZE(cs)       ((cs->c_size == (PAGE_SIZE-1)) ? PAGE_SIZE : cs->c_size)
+#define PACK_C_SIZE(cs, size)   (cs->c_size = ((size == PAGE_SIZE) ? PAGE_SIZE - 1 : size))
 
 
 struct c_sv_hash_entry {
        union {
 
 
 struct c_sv_hash_entry {
        union {
-               struct  {
-                       uint32_t        c_sv_he_ref;
-                       uint32_t        c_sv_he_data;
+               struct  {
+                       uint32_t        c_sv_he_ref;
+                       uint32_t        c_sv_he_data;
                } c_sv_he;
                } c_sv_he;
-               uint64_t        c_sv_he_record;
-
-       } c_sv_he_un;   
+               uint64_t        c_sv_he_record;
+       } c_sv_he_un;
 };
 
 };
 
-#define        he_ref  c_sv_he_un.c_sv_he.c_sv_he_ref
-#define        he_data c_sv_he_un.c_sv_he.c_sv_he_data
-#define        he_record c_sv_he_un.c_sv_he_record
-
-#define C_SV_HASH_MAX_MISS     32
-#define C_SV_HASH_SIZE         ((1 << 10))
-#define C_SV_HASH_MASK         ((1 << 10) - 1)
-#define C_SV_CSEG_ID           ((1 << 22) - 1)
+#define he_ref  c_sv_he_un.c_sv_he.c_sv_he_ref
+#define he_data c_sv_he_un.c_sv_he.c_sv_he_data
+#define he_record c_sv_he_un.c_sv_he_record
 
 
-
-struct  c_slot_mapping {
-        uint32_t        s_cseg:22,     /* segment number + 1 */
-                       s_cindx:10;     /* index in the segment */
-};
-#define C_SLOT_MAX_INDEX       (1 << 10)
-
-typedef struct c_slot_mapping *c_slot_mapping_t;
+#define C_SV_HASH_MAX_MISS      32
+#define C_SV_HASH_SIZE          ((1 << 10))
+#define C_SV_HASH_MASK          ((1 << 10) - 1)
+#define C_SV_CSEG_ID            ((1 << 22) - 1)
 
 
 union c_segu {
 
 
 union c_segu {
-       c_segment_t     c_seg;
-       uint32_t        c_segno;
+       c_segment_t     c_seg;
+       uintptr_t       c_segno;
 };
 
 };
 
+#define C_SLOT_ASSERT_PACKABLE(ptr) \
+       VM_ASSERT_POINTER_PACKABLE((vm_offset_t)(ptr), C_SLOT_PACKED_PTR);
 
 
+#define C_SLOT_PACK_PTR(ptr) \
+       VM_PACK_POINTER((vm_offset_t)(ptr), C_SLOT_PACKED_PTR)
 
 
-#define C_SLOT_PACK_PTR(ptr)           (((uintptr_t)ptr - (uintptr_t) VM_MIN_KERNEL_AND_KEXT_ADDRESS) >> 2)
-#define C_SLOT_UNPACK_PTR(cslot)       ((uintptr_t)(cslot->c_packed_ptr << 2) + (uintptr_t) VM_MIN_KERNEL_AND_KEXT_ADDRESS)
+#define C_SLOT_UNPACK_PTR(cslot) \
+       (c_slot_mapping_t)VM_UNPACK_POINTER((cslot)->c_packed_ptr, C_SLOT_PACKED_PTR)
 
 
+/* for debugging purposes */
+SECURITY_READ_ONLY_EARLY(vm_packing_params_t) c_slot_packing_params =
+    VM_PACKING_PARAMS(C_SLOT_PACKED_PTR);
 
 
-uint32_t       c_segment_count = 0;
-uint32_t       c_segment_count_max = 0;
+uint32_t        c_segment_count = 0;
+uint32_t        c_segment_count_max = 0;
 
 
-uint64_t       c_generation_id = 0;
-uint64_t       c_generation_id_flush_barrier;
+uint64_t        c_generation_id = 0;
+uint64_t        c_generation_id_flush_barrier;
 
 
 
 
-#define                HIBERNATE_FLUSHING_SECS_TO_COMPLETE     120
+#define         HIBERNATE_FLUSHING_SECS_TO_COMPLETE     120
 
 
-boolean_t      hibernate_no_swapspace = FALSE;
-clock_sec_t    hibernate_flushing_deadline = 0;
+boolean_t       hibernate_no_swapspace = FALSE;
+clock_sec_t     hibernate_flushing_deadline = 0;
 
 
 #if RECORD_THE_COMPRESSED_DATA
 
 
 #if RECORD_THE_COMPRESSED_DATA
-char   *c_compressed_record_sbuf;
-char   *c_compressed_record_ebuf;
-char   *c_compressed_record_cptr;
+char    *c_compressed_record_sbuf;
+char    *c_compressed_record_ebuf;
+char    *c_compressed_record_cptr;
 #endif
 
 
 #endif
 
 
-queue_head_t   c_age_list_head;
-queue_head_t   c_swapout_list_head;
-queue_head_t   c_swappedin_list_head;
-queue_head_t   c_swappedout_list_head;
-queue_head_t   c_swappedout_sparse_list_head;
-queue_head_t   c_major_list_head;
-queue_head_t   c_filling_list_head;
-queue_head_t   c_bad_list_head;
-
-uint32_t       c_age_count = 0;
-uint32_t       c_swapout_count = 0;
-uint32_t       c_swappedin_count = 0;
-uint32_t       c_swappedout_count = 0;
-uint32_t       c_swappedout_sparse_count = 0;
-uint32_t       c_major_count = 0;
-uint32_t       c_filling_count = 0;
-uint32_t       c_empty_count = 0;
-uint32_t       c_bad_count = 0;
-
-
-queue_head_t   c_minor_list_head;
-uint32_t       c_minor_count = 0;
-
-int            c_overage_swapped_count = 0;
-int            c_overage_swapped_limit = 0;
-
-int            c_seg_fixed_array_len;
-union  c_segu  *c_segments;
-vm_offset_t    c_buffers;
+queue_head_t    c_age_list_head;
+queue_head_t    c_swappedin_list_head;
+queue_head_t    c_swapout_list_head;
+queue_head_t    c_swapio_list_head;
+queue_head_t    c_swappedout_list_head;
+queue_head_t    c_swappedout_sparse_list_head;
+queue_head_t    c_major_list_head;
+queue_head_t    c_filling_list_head;
+queue_head_t    c_bad_list_head;
+
+uint32_t        c_age_count = 0;
+uint32_t        c_swappedin_count = 0;
+uint32_t        c_swapout_count = 0;
+uint32_t        c_swapio_count = 0;
+uint32_t        c_swappedout_count = 0;
+uint32_t        c_swappedout_sparse_count = 0;
+uint32_t        c_major_count = 0;
+uint32_t        c_filling_count = 0;
+uint32_t        c_empty_count = 0;
+uint32_t        c_bad_count = 0;
+
+
+queue_head_t    c_minor_list_head;
+uint32_t        c_minor_count = 0;
+
+int             c_overage_swapped_count = 0;
+int             c_overage_swapped_limit = 0;
+
+int             c_seg_fixed_array_len;
+union  c_segu   *c_segments;
+vm_offset_t     c_buffers;
 vm_size_t       c_buffers_size;
 vm_size_t       c_buffers_size;
-caddr_t                c_segments_next_page;
-boolean_t      c_segments_busy;
-uint32_t       c_segments_available;
-uint32_t       c_segments_limit;
-uint32_t       c_segments_nearing_limit;
-
-uint32_t       c_segment_svp_in_hash;
-uint32_t       c_segment_svp_hash_succeeded;
-uint32_t       c_segment_svp_hash_failed;
-uint32_t       c_segment_svp_zero_compressions;
-uint32_t       c_segment_svp_nonzero_compressions;
-uint32_t       c_segment_svp_zero_decompressions;
-uint32_t       c_segment_svp_nonzero_decompressions;
-
-uint32_t       c_segment_noncompressible_pages;
-
-uint32_t       c_segment_pages_compressed;
-uint32_t       c_segment_pages_compressed_limit;
-uint32_t       c_segment_pages_compressed_nearing_limit;
-uint32_t       c_free_segno_head = (uint32_t)-1;
+caddr_t         c_segments_next_page;
+boolean_t       c_segments_busy;
+uint32_t        c_segments_available;
+uint32_t        c_segments_limit;
+uint32_t        c_segments_nearing_limit;
+
+uint32_t        c_segment_svp_in_hash;
+uint32_t        c_segment_svp_hash_succeeded;
+uint32_t        c_segment_svp_hash_failed;
+uint32_t        c_segment_svp_zero_compressions;
+uint32_t        c_segment_svp_nonzero_compressions;
+uint32_t        c_segment_svp_zero_decompressions;
+uint32_t        c_segment_svp_nonzero_decompressions;
+
+uint32_t        c_segment_noncompressible_pages;
+
+uint32_t        c_segment_pages_compressed = 0; /* Tracks # of uncompressed pages fed into the compressor */
+#if CONFIG_FREEZE
+int32_t        c_segment_pages_compressed_incore = 0; /* Tracks # of uncompressed pages fed into the compressor that are in memory */
+uint32_t        c_segments_incore_limit = 0; /* Tracks # of segments allowed to be in-core. Based on compressor pool size */
+#endif /* CONFIG_FREEZE */
 
 
-uint32_t       vm_compressor_minorcompact_threshold_divisor = 10;
-uint32_t       vm_compressor_majorcompact_threshold_divisor = 10;
-uint32_t       vm_compressor_unthrottle_threshold_divisor = 10;
-uint32_t       vm_compressor_catchup_threshold_divisor = 10;
+uint32_t        c_segment_pages_compressed_limit;
+uint32_t        c_segment_pages_compressed_nearing_limit;
+uint32_t        c_free_segno_head = (uint32_t)-1;
 
 
-#define                C_SEGMENTS_PER_PAGE     (PAGE_SIZE / sizeof(union c_segu))
+uint32_t        vm_compressor_minorcompact_threshold_divisor = 10;
+uint32_t        vm_compressor_majorcompact_threshold_divisor = 10;
+uint32_t        vm_compressor_unthrottle_threshold_divisor = 10;
+uint32_t        vm_compressor_catchup_threshold_divisor = 10;
 
 
+uint32_t        vm_compressor_minorcompact_threshold_divisor_overridden = 0;
+uint32_t        vm_compressor_majorcompact_threshold_divisor_overridden = 0;
+uint32_t        vm_compressor_unthrottle_threshold_divisor_overridden = 0;
+uint32_t        vm_compressor_catchup_threshold_divisor_overridden = 0;
 
 
-lck_grp_attr_t vm_compressor_lck_grp_attr;
-lck_attr_t     vm_compressor_lck_attr;
-lck_grp_t      vm_compressor_lck_grp;
+#define         C_SEGMENTS_PER_PAGE     (PAGE_SIZE / sizeof(union c_segu))
 
 
-#if __i386__ || __x86_64__
-lck_mtx_t      *c_list_lock;
-#else /* __i386__ || __x86_64__ */
-lck_spin_t     *c_list_lock;
-#endif /* __i386__ || __x86_64__ */
+LCK_GRP_DECLARE(vm_compressor_lck_grp, "vm_compressor");
+LCK_RW_DECLARE(c_master_lock, &vm_compressor_lck_grp);
+LCK_MTX_DECLARE(c_list_lock_storage, &vm_compressor_lck_grp);
 
 
-lck_rw_t       c_master_lock;
-boolean_t      decompressions_blocked = FALSE;
+boolean_t       decompressions_blocked = FALSE;
 
 
-zone_t         compressor_segment_zone;
-int            c_compressor_swap_trigger = 0;
+zone_t          compressor_segment_zone;
+int             c_compressor_swap_trigger = 0;
 
 
-uint32_t       compressor_cpus;
-char           *compressor_scratch_bufs;
-char           *kdp_compressor_scratch_buf;
-char           *kdp_compressor_decompressed_page;
-addr64_t       kdp_compressor_decompressed_page_paddr;
-ppnum_t                kdp_compressor_decompressed_page_ppnum;
+uint32_t        compressor_cpus;
+char            *compressor_scratch_bufs;
+char            *kdp_compressor_scratch_buf;
+char            *kdp_compressor_decompressed_page;
+addr64_t        kdp_compressor_decompressed_page_paddr;
+ppnum_t         kdp_compressor_decompressed_page_ppnum;
 
 
-clock_sec_t    start_of_sample_period_sec = 0;
-clock_nsec_t   start_of_sample_period_nsec = 0;
-clock_sec_t    start_of_eval_period_sec = 0;
-clock_nsec_t   start_of_eval_period_nsec = 0;
-uint32_t       sample_period_decompression_count = 0;
-uint32_t       sample_period_compression_count = 0;
-uint32_t       last_eval_decompression_count = 0;
-uint32_t       last_eval_compression_count = 0;
+clock_sec_t     start_of_sample_period_sec = 0;
+clock_nsec_t    start_of_sample_period_nsec = 0;
+clock_sec_t     start_of_eval_period_sec = 0;
+clock_nsec_t    start_of_eval_period_nsec = 0;
+uint32_t        sample_period_decompression_count = 0;
+uint32_t        sample_period_compression_count = 0;
+uint32_t        last_eval_decompression_count = 0;
+uint32_t        last_eval_compression_count = 0;
 
 
-#define                DECOMPRESSION_SAMPLE_MAX_AGE            (60 * 30)
+#define         DECOMPRESSION_SAMPLE_MAX_AGE            (60 * 30)
 
 
-boolean_t      vm_swapout_ripe_segments = FALSE;
-uint32_t       vm_ripe_target_age = (60 * 60 * 48);
+boolean_t       vm_swapout_ripe_segments = FALSE;
+uint32_t        vm_ripe_target_age = (60 * 60 * 48);
 
 
-uint32_t       swapout_target_age = 0;
-uint32_t       age_of_decompressions_during_sample_period[DECOMPRESSION_SAMPLE_MAX_AGE];
-uint32_t       overage_decompressions_during_sample_period = 0;
+uint32_t        swapout_target_age = 0;
+uint32_t        age_of_decompressions_during_sample_period[DECOMPRESSION_SAMPLE_MAX_AGE];
+uint32_t        overage_decompressions_during_sample_period = 0;
 
 
-void           do_fastwake_warmup(void);
-boolean_t      fastwake_warmup = FALSE;
-boolean_t      fastwake_recording_in_progress = FALSE;
-clock_sec_t    dont_trim_until_ts = 0;
 
 
-uint64_t       c_segment_warmup_count;
-uint64_t       first_c_segment_to_warm_generation_id = 0;
-uint64_t       last_c_segment_to_warm_generation_id = 0;
-boolean_t      hibernate_flushing = FALSE;
+void            do_fastwake_warmup(queue_head_t *, boolean_t);
+boolean_t       fastwake_warmup = FALSE;
+boolean_t       fastwake_recording_in_progress = FALSE;
+clock_sec_t     dont_trim_until_ts = 0;
 
 
-int64_t                c_segment_input_bytes __attribute__((aligned(8))) = 0;
-int64_t                c_segment_compressed_bytes __attribute__((aligned(8))) = 0;
-int64_t                compressor_bytes_used __attribute__((aligned(8))) = 0;
+uint64_t        c_segment_warmup_count;
+uint64_t        first_c_segment_to_warm_generation_id = 0;
+uint64_t        last_c_segment_to_warm_generation_id = 0;
+boolean_t       hibernate_flushing = FALSE;
 
 
+int64_t         c_segment_input_bytes __attribute__((aligned(8))) = 0;
+int64_t         c_segment_compressed_bytes __attribute__((aligned(8))) = 0;
+int64_t         compressor_bytes_used __attribute__((aligned(8))) = 0;
 
 
-struct c_sv_hash_entry c_segment_sv_hash_table[C_SV_HASH_SIZE]  __attribute__ ((aligned (8)));
 
 
+struct c_sv_hash_entry c_segment_sv_hash_table[C_SV_HASH_SIZE]  __attribute__ ((aligned(8)));
 
 static boolean_t compressor_needs_to_swap(void);
 static void vm_compressor_swap_trigger_thread(void);
 
 static boolean_t compressor_needs_to_swap(void);
 static void vm_compressor_swap_trigger_thread(void);
@@ -258,9 +344,9 @@ static void vm_compressor_do_delayed_compactions(boolean_t);
 static void vm_compressor_compact_and_swap(boolean_t);
 static void vm_compressor_age_swapped_in_segments(boolean_t);
 
 static void vm_compressor_compact_and_swap(boolean_t);
 static void vm_compressor_age_swapped_in_segments(boolean_t);
 
+#if XNU_TARGET_OS_OSX
 static void vm_compressor_take_paging_space_action(void);
 static void vm_compressor_take_paging_space_action(void);
-
-boolean_t vm_compressor_low_on_space(void);
+#endif /* XNU_TARGET_OS_OSX */
 
 void compute_swapout_target_age(void);
 
 
 void compute_swapout_target_age(void);
 
@@ -270,7 +356,6 @@ boolean_t c_seg_major_compact_ok(c_segment_t, c_segment_t);
 int  c_seg_minor_compaction_and_unlock(c_segment_t, boolean_t);
 int  c_seg_do_minor_compaction_and_unlock(c_segment_t, boolean_t, boolean_t, boolean_t);
 void c_seg_try_minor_compaction_and_unlock(c_segment_t c_seg);
 int  c_seg_minor_compaction_and_unlock(c_segment_t, boolean_t);
 int  c_seg_do_minor_compaction_and_unlock(c_segment_t, boolean_t, boolean_t, boolean_t);
 void c_seg_try_minor_compaction_and_unlock(c_segment_t c_seg);
-void c_seg_need_delayed_compaction(c_segment_t);
 
 void c_seg_move_to_sparse_list(c_segment_t);
 void c_seg_insert_into_q(queue_head_t *, c_segment_t);
 
 void c_seg_move_to_sparse_list(c_segment_t);
 void c_seg_insert_into_q(queue_head_t *, c_segment_t);
@@ -278,80 +363,146 @@ void c_seg_insert_into_q(queue_head_t *, c_segment_t);
 uint64_t vm_available_memory(void);
 uint64_t vm_compressor_pages_compressed(void);
 
 uint64_t vm_available_memory(void);
 uint64_t vm_compressor_pages_compressed(void);
 
-extern unsigned int dp_pages_free, dp_pages_reserve;
+/*
+ * indicate the need to do a major compaction if
+ * the overall set of in-use compression segments
+ * becomes sparse... on systems that support pressure
+ * driven swapping, this will also cause swapouts to
+ * be initiated.
+ */
+static inline boolean_t
+vm_compressor_needs_to_major_compact()
+{
+       uint32_t        incore_seg_count;
+
+       incore_seg_count = c_segment_count - c_swappedout_count - c_swappedout_sparse_count;
+
+       if ((c_segment_count >= (c_segments_nearing_limit / 8)) &&
+           ((incore_seg_count * C_SEG_MAX_PAGES) - VM_PAGE_COMPRESSOR_COUNT) >
+           ((incore_seg_count / 8) * C_SEG_MAX_PAGES)) {
+               return 1;
+       }
+       return 0;
+}
+
 
 uint64_t
 vm_available_memory(void)
 {
 
 uint64_t
 vm_available_memory(void)
 {
-       return (((uint64_t)AVAILABLE_NON_COMPRESSED_MEMORY) * PAGE_SIZE_64);
+       return ((uint64_t)AVAILABLE_NON_COMPRESSED_MEMORY) * PAGE_SIZE_64;
 }
 
 
 uint64_t
 vm_compressor_pages_compressed(void)
 {
 }
 
 
 uint64_t
 vm_compressor_pages_compressed(void)
 {
-       return (c_segment_pages_compressed * PAGE_SIZE_64);
+       return c_segment_pages_compressed * PAGE_SIZE_64;
 }
 
 
 boolean_t
 }
 
 
 boolean_t
-vm_compression_available(void)
+vm_compressor_low_on_space(void)
 {
 {
-       if ( !(COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE))
-               return (FALSE);
-
-       if (c_segments_available >= c_segments_limit || c_segment_pages_compressed >= c_segment_pages_compressed_limit)
-               return (FALSE);
+#if CONFIG_FREEZE
+       uint64_t incore_seg_count;
+       uint32_t incore_compressed_pages;
+       if (freezer_incore_cseg_acct) {
+               incore_seg_count = c_segment_count - c_swappedout_count - c_swappedout_sparse_count;
+               incore_compressed_pages = c_segment_pages_compressed_incore;
+       } else {
+               incore_seg_count = c_segment_count;
+               incore_compressed_pages = c_segment_pages_compressed;
+       }
 
 
-       return (TRUE);
+       if ((incore_compressed_pages > c_segment_pages_compressed_nearing_limit) ||
+           (incore_seg_count > c_segments_nearing_limit)) {
+               return TRUE;
+       }
+#else /* CONFIG_FREEZE */
+       if ((c_segment_pages_compressed > c_segment_pages_compressed_nearing_limit) ||
+           (c_segment_count > c_segments_nearing_limit)) {
+               return TRUE;
+       }
+#endif /* CONFIG_FREEZE */
+       return FALSE;
 }
 
 
 boolean_t
 }
 
 
 boolean_t
-vm_compressor_low_on_space(void)
+vm_compressor_out_of_space(void)
 {
 {
-       if ((c_segment_pages_compressed > c_segment_pages_compressed_nearing_limit) ||
-           (c_segment_count > c_segments_nearing_limit))
-               return (TRUE);
+#if CONFIG_FREEZE
+       uint64_t incore_seg_count;
+       uint32_t incore_compressed_pages;
+       if (freezer_incore_cseg_acct) {
+               incore_seg_count = c_segment_count - c_swappedout_count - c_swappedout_sparse_count;
+               incore_compressed_pages = c_segment_pages_compressed_incore;
+       } else {
+               incore_seg_count = c_segment_count;
+               incore_compressed_pages = c_segment_pages_compressed;
+       }
 
 
-       return (FALSE);
+       if ((incore_compressed_pages >= c_segment_pages_compressed_limit) ||
+           (incore_seg_count > c_segments_incore_limit)) {
+               return TRUE;
+       }
+#else /* CONFIG_FREEZE */
+       if ((c_segment_pages_compressed >= c_segment_pages_compressed_limit) ||
+           (c_segment_count >= c_segments_limit)) {
+               return TRUE;
+       }
+#endif /* CONFIG_FREEZE */
+       return FALSE;
 }
 }
-           
+
 
 int
 vm_wants_task_throttled(task_t task)
 {
 
 int
 vm_wants_task_throttled(task_t task)
 {
-       if (task == kernel_task)
-               return (0);
-
-       if (COMPRESSED_PAGER_IS_SWAPLESS || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_SWAPLESS)
-               return (0);
+       if (task == kernel_task) {
+               return 0;
+       }
 
 
-       if (COMPRESSED_PAGER_IS_SWAPBACKED || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_SWAPBACKED) {
+       if (VM_CONFIG_SWAP_IS_ACTIVE) {
                if ((vm_compressor_low_on_space() || HARD_THROTTLE_LIMIT_REACHED()) &&
                if ((vm_compressor_low_on_space() || HARD_THROTTLE_LIMIT_REACHED()) &&
-                   (unsigned int)pmap_compressed(task->map->pmap) > (c_segment_pages_compressed / 4))
-                       return (1);
-       } else {
-               if (((dp_pages_free + dp_pages_reserve < 2000) && VM_DYNAMIC_PAGING_ENABLED(memory_manager_default)) &&
-                   get_task_resident_size(task) > (((AVAILABLE_NON_COMPRESSED_MEMORY) * PAGE_SIZE) / 5))
-                       return (1);
+                   (unsigned int)pmap_compressed(task->map->pmap) > (c_segment_pages_compressed / 4)) {
+                       return 1;
+               }
        }
        }
-       return (0);
+       return 0;
 }
 
 
 }
 
 
+#if DEVELOPMENT || DEBUG
+/*
+ * On compressor/swap exhaustion, kill the largest process regardless of
+ * its chosen process policy.
+ */
+TUNABLE(bool, kill_on_no_paging_space, "-kill_on_no_paging_space", false);
+#endif /* DEVELOPMENT || DEBUG */
+
+#if XNU_TARGET_OS_OSX
 
 
-static uint32_t        no_paging_space_action_in_progress = 0;
+static uint32_t no_paging_space_action_in_progress = 0;
 extern void memorystatus_send_low_swap_note(void);
 
 static void
 vm_compressor_take_paging_space_action(void)
 {
        if (no_paging_space_action_in_progress == 0) {
 extern void memorystatus_send_low_swap_note(void);
 
 static void
 vm_compressor_take_paging_space_action(void)
 {
        if (no_paging_space_action_in_progress == 0) {
-
                if (OSCompareAndSwap(0, 1, (UInt32 *)&no_paging_space_action_in_progress)) {
                if (OSCompareAndSwap(0, 1, (UInt32 *)&no_paging_space_action_in_progress)) {
-
                        if (no_paging_space_action()) {
                        if (no_paging_space_action()) {
+#if DEVELOPMENT || DEBUG
+                               if (kill_on_no_paging_space) {
+                                       /*
+                                        * Since we are choosing to always kill a process, we don't need the
+                                        * "out of application memory" dialog box in this mode. And, hence we won't
+                                        * send the knote.
+                                        */
+                                       no_paging_space_action_in_progress = 0;
+                                       return;
+                               }
+#endif /* DEVELOPMENT || DEBUG */
                                memorystatus_send_low_swap_note();
                        }
 
                                memorystatus_send_low_swap_note();
                        }
 
@@ -359,18 +510,7 @@ vm_compressor_take_paging_space_action(void)
                }
        }
 }
                }
        }
 }
-
-
-
-void
-vm_compressor_init_locks(void)
-{
-       lck_grp_attr_setdefault(&vm_compressor_lck_grp_attr);
-       lck_grp_init(&vm_compressor_lck_grp, "vm_compressor", &vm_compressor_lck_grp_attr);
-       lck_attr_setdefault(&vm_compressor_lck_attr);
-
-       lck_rw_init(&c_master_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
-}
+#endif /* XNU_TARGET_OS_OSX */
 
 
 void
 
 
 void
@@ -379,7 +519,7 @@ vm_decompressor_lock(void)
        PAGE_REPLACEMENT_ALLOWED(TRUE);
 
        decompressions_blocked = TRUE;
        PAGE_REPLACEMENT_ALLOWED(TRUE);
 
        decompressions_blocked = TRUE;
-       
+
        PAGE_REPLACEMENT_ALLOWED(FALSE);
 }
 
        PAGE_REPLACEMENT_ALLOWED(FALSE);
 }
 
@@ -395,36 +535,100 @@ vm_decompressor_unlock(void)
        thread_wakeup((event_t)&decompressions_blocked);
 }
 
        thread_wakeup((event_t)&decompressions_blocked);
 }
 
+static inline void
+cslot_copy(c_slot_t cdst, c_slot_t csrc)
+{
+#if CHECKSUM_THE_DATA
+       cdst->c_hash_data = csrc->c_hash_data;
+#endif
+#if CHECKSUM_THE_COMPRESSED_DATA
+       cdst->c_hash_compressed_data = csrc->c_hash_compressed_data;
+#endif
+#if POPCOUNT_THE_COMPRESSED_DATA
+       cdst->c_pop_cdata = csrc->c_pop_cdata;
+#endif
+       cdst->c_size = csrc->c_size;
+       cdst->c_packed_ptr = csrc->c_packed_ptr;
+#if defined(__arm__) || defined(__arm64__)
+       cdst->c_codec = csrc->c_codec;
+#endif
+#if __ARM_WKDM_POPCNT__
+       cdst->c_inline_popcount = csrc->c_inline_popcount;
+#endif
+}
 
 
+vm_map_t compressor_map;
+uint64_t compressor_pool_max_size;
+uint64_t compressor_pool_size;
+uint32_t compressor_pool_multiplier;
+
+#if DEVELOPMENT || DEBUG
+/*
+ * Compressor segments are write-protected in development/debug
+ * kernels to help debug memory corruption.
+ * In cases where performance is a concern, this can be disabled
+ * via the boot-arg "-disable_cseg_write_protection".
+ */
+boolean_t write_protect_c_segs = TRUE;
+int vm_compressor_test_seg_wp;
+uint32_t vm_ktrace_enabled;
+#endif /* DEVELOPMENT || DEBUG */
 
 void
 vm_compressor_init(void)
 {
 
 void
 vm_compressor_init(void)
 {
-       thread_t        thread;
-       struct c_slot   cs_dummy;
-       c_slot_t cs  = &cs_dummy;
-       int             c_segment_min_size;
-       int             c_segment_padded_size;
-
-       /*
-        * ensure that any pointer that gets created from
-        * the vm_page zone can be packed properly
-        */
-       cs->c_packed_ptr = C_SLOT_PACK_PTR(zone_map_min_address);
+       thread_t        thread;
+       int             attempts = 1;
+       kern_return_t   retval = KERN_SUCCESS;
+       vm_offset_t     start_addr = 0;
+       vm_size_t       c_segments_arr_size = 0, compressor_submap_size = 0;
+       vm_map_kernel_flags_t vmk_flags;
+#if RECORD_THE_COMPRESSED_DATA
+       vm_size_t       c_compressed_record_sbuf_size = 0;
+#endif /* RECORD_THE_COMPRESSED_DATA */
 
 
-       if (C_SLOT_UNPACK_PTR(cs) != (uintptr_t)zone_map_min_address)
-               panic("C_SLOT_UNPACK_PTR failed on zone_map_min_address - %p", (void *)zone_map_min_address);
+#if DEVELOPMENT || DEBUG || CONFIG_FREEZE
+       char bootarg_name[32];
+#endif /* DEVELOPMENT || DEBUG || CONFIG_FREEZE */
 
 
-       cs->c_packed_ptr = C_SLOT_PACK_PTR(zone_map_max_address);
+#if DEVELOPMENT || DEBUG
+       if (PE_parse_boot_argn("-disable_cseg_write_protection", bootarg_name, sizeof(bootarg_name))) {
+               write_protect_c_segs = FALSE;
+       }
+       int vmcval = 1;
+       PE_parse_boot_argn("vm_compressor_validation", &vmcval, sizeof(vmcval));
 
 
-       if (C_SLOT_UNPACK_PTR(cs) != (uintptr_t)zone_map_max_address)
-               panic("C_SLOT_UNPACK_PTR failed on zone_map_max_address - %p", (void *)zone_map_max_address);
+       if (kern_feature_override(KF_COMPRSV_OVRD)) {
+               vmcval = 0;
+       }
+       if (vmcval == 0) {
+#if POPCOUNT_THE_COMPRESSED_DATA
+               popcount_c_segs = FALSE;
+#endif
+#if CHECKSUM_THE_DATA || CHECKSUM_THE_COMPRESSED_DATA
+               checksum_c_segs = FALSE;
+#endif
+#if VALIDATE_C_SEGMENTS
+               validate_c_segs = FALSE;
+#endif
+               write_protect_c_segs = FALSE;
+       }
+#endif /* DEVELOPMENT || DEBUG */
 
 
+#if CONFIG_FREEZE
+       if (PE_parse_boot_argn("-disable_freezer_cseg_acct", bootarg_name, sizeof(bootarg_name))) {
+               freezer_incore_cseg_acct = FALSE;
+       }
+#endif /* CONFIG_FREEZE */
 
        assert((C_SEGMENTS_PER_PAGE * sizeof(union c_segu)) == PAGE_SIZE);
 
 
        assert((C_SEGMENTS_PER_PAGE * sizeof(union c_segu)) == PAGE_SIZE);
 
-       PE_parse_boot_argn("vm_compression_limit", &vm_compression_limit, sizeof (vm_compression_limit));
-
+#if !XNU_TARGET_OS_OSX
+       vm_compressor_minorcompact_threshold_divisor = 20;
+       vm_compressor_majorcompact_threshold_divisor = 30;
+       vm_compressor_unthrottle_threshold_divisor = 40;
+       vm_compressor_catchup_threshold_divisor = 60;
+#else /* !XNU_TARGET_OS_OSX */
        if (max_mem <= (3ULL * 1024ULL * 1024ULL * 1024ULL)) {
                vm_compressor_minorcompact_threshold_divisor = 11;
                vm_compressor_majorcompact_threshold_divisor = 13;
        if (max_mem <= (3ULL * 1024ULL * 1024ULL * 1024ULL)) {
                vm_compressor_minorcompact_threshold_divisor = 11;
                vm_compressor_majorcompact_threshold_divisor = 13;
@@ -436,18 +640,7 @@ vm_compressor_init(void)
                vm_compressor_unthrottle_threshold_divisor = 35;
                vm_compressor_catchup_threshold_divisor = 50;
        }
                vm_compressor_unthrottle_threshold_divisor = 35;
                vm_compressor_catchup_threshold_divisor = 50;
        }
-       /*
-        * vm_page_init_lck_grp is now responsible for calling vm_compressor_init_locks
-        * c_master_lock needs to be available early so that "vm_page_find_contiguous" can
-        * use PAGE_REPLACEMENT_ALLOWED to coordinate with the compressor.
-        */
-
-#if __i386__ || __x86_64__
-       c_list_lock = lck_mtx_alloc_init(&vm_compressor_lck_grp, &vm_compressor_lck_attr);
-#else /* __i386__ || __x86_64__ */
-       c_list_lock = lck_spin_alloc_init(&vm_compressor_lck_grp, &vm_compressor_lck_attr);
-#endif /* __i386__ || __x86_64__ */
-
+#endif /* !XNU_TARGET_OS_OSX */
 
        queue_init(&c_bad_list_head);
        queue_init(&c_age_list_head);
 
        queue_init(&c_bad_list_head);
        queue_init(&c_age_list_head);
@@ -455,106 +648,272 @@ vm_compressor_init(void)
        queue_init(&c_major_list_head);
        queue_init(&c_filling_list_head);
        queue_init(&c_swapout_list_head);
        queue_init(&c_major_list_head);
        queue_init(&c_filling_list_head);
        queue_init(&c_swapout_list_head);
+       queue_init(&c_swapio_list_head);
        queue_init(&c_swappedin_list_head);
        queue_init(&c_swappedout_list_head);
        queue_init(&c_swappedout_sparse_list_head);
 
        queue_init(&c_swappedin_list_head);
        queue_init(&c_swappedout_list_head);
        queue_init(&c_swappedout_sparse_list_head);
 
-       c_segment_min_size = sizeof(struct c_segment) + (C_SEG_SLOT_VAR_ARRAY_MIN_LEN * sizeof(struct c_slot));
-       
-       for (c_segment_padded_size = 128; c_segment_padded_size < c_segment_min_size; c_segment_padded_size = c_segment_padded_size << 1);
-
-       compressor_segment_zone = zinit(c_segment_padded_size, 128000 * c_segment_padded_size, PAGE_SIZE, "compressor_segment");
-       zone_change(compressor_segment_zone, Z_CALLERACCT, FALSE);
-       zone_change(compressor_segment_zone, Z_NOENCRYPT, TRUE);
-
-       c_seg_fixed_array_len = (c_segment_padded_size - sizeof(struct c_segment)) / sizeof(struct c_slot);
-       
        c_free_segno_head = -1;
        c_segments_available = 0;
 
        c_free_segno_head = -1;
        c_segments_available = 0;
 
-       if (vm_compression_limit == 0) {
-               c_segment_pages_compressed_limit = (uint32_t)((max_mem / PAGE_SIZE)) * vm_scale;
+       if (vm_compression_limit) {
+               compressor_pool_size = ptoa_64(vm_compression_limit);
+       }
+
+       compressor_pool_max_size = C_SEG_MAX_LIMIT;
+       compressor_pool_max_size *= C_SEG_BUFSIZE;
 
 
-#define        OLD_SWAP_LIMIT  (1024 * 1024 * 16)
-#define MAX_SWAP_LIMIT (1024 * 1024 * 128)
-       
-               if (c_segment_pages_compressed_limit > (OLD_SWAP_LIMIT))
-                       c_segment_pages_compressed_limit = OLD_SWAP_LIMIT;
+#if XNU_TARGET_OS_OSX
 
 
-               if (c_segment_pages_compressed_limit < (uint32_t)(max_mem / PAGE_SIZE_64))
-                       c_segment_pages_compressed_limit = (uint32_t)(max_mem / PAGE_SIZE_64);
+       if (vm_compression_limit == 0) {
+               if (max_mem <= (4ULL * 1024ULL * 1024ULL * 1024ULL)) {
+                       compressor_pool_size = 16ULL * max_mem;
+               } else if (max_mem <= (8ULL * 1024ULL * 1024ULL * 1024ULL)) {
+                       compressor_pool_size = 8ULL * max_mem;
+               } else if (max_mem <= (32ULL * 1024ULL * 1024ULL * 1024ULL)) {
+                       compressor_pool_size = 4ULL * max_mem;
+               } else {
+                       compressor_pool_size = 2ULL * max_mem;
+               }
+       }
+       if (max_mem <= (8ULL * 1024ULL * 1024ULL * 1024ULL)) {
+               compressor_pool_multiplier = 1;
+       } else if (max_mem <= (32ULL * 1024ULL * 1024ULL * 1024ULL)) {
+               compressor_pool_multiplier = 2;
        } else {
        } else {
-               if (vm_compression_limit < MAX_SWAP_LIMIT)
-                       c_segment_pages_compressed_limit = vm_compression_limit;
-               else
-                       c_segment_pages_compressed_limit = MAX_SWAP_LIMIT;
+               compressor_pool_multiplier = 4;
        }
        }
-       if ((c_segments_limit = c_segment_pages_compressed_limit / (C_SEG_BUFSIZE / PAGE_SIZE)) > C_SEG_MAX_LIMIT)
-               c_segments_limit = C_SEG_MAX_LIMIT;
 
 
-       c_segment_pages_compressed_nearing_limit = (c_segment_pages_compressed_limit * 98) / 100;
-       c_segments_nearing_limit = (c_segments_limit * 98) / 100;
+#elif defined(__arm__)
 
 
-       c_segments_busy = FALSE;
+#define VM_RESERVE_SIZE                 (1024 * 1024 * 256)
+#define MAX_COMPRESSOR_POOL_SIZE        (1024 * 1024 * 450)
+
+       if (compressor_pool_max_size > MAX_COMPRESSOR_POOL_SIZE) {
+               compressor_pool_max_size = MAX_COMPRESSOR_POOL_SIZE;
+       }
 
 
-       if (kernel_memory_allocate(kernel_map, (vm_offset_t *)(&c_segments), (sizeof(union c_segu) * c_segments_limit), 0, KMA_KOBJECT | KMA_VAONLY | KMA_PERMANENT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS)
+       if (vm_compression_limit == 0) {
+               compressor_pool_size = ((kernel_map->max_offset - kernel_map->min_offset) - kernel_map->size) - VM_RESERVE_SIZE;
+       }
+       compressor_pool_multiplier = 1;
+
+#elif defined(__arm64__) && defined(XNU_TARGET_OS_WATCH)
+
+       /*
+        * On M9 watches the compressor can become big and can lead to
+        * churn in workingset resulting in audio drops. Setting a cap
+        * on the compressor size favors reclaiming unused memory
+        * sitting in idle band via jetsams
+        */
+
+#define COMPRESSOR_CAP_PERCENTAGE        37ULL
+
+       if (compressor_pool_max_size > max_mem) {
+               compressor_pool_max_size = max_mem;
+       }
+
+       if (vm_compression_limit == 0) {
+               compressor_pool_size = (max_mem * COMPRESSOR_CAP_PERCENTAGE) / 100ULL;
+       }
+       compressor_pool_multiplier = 1;
+
+#else
+
+       if (compressor_pool_max_size > max_mem) {
+               compressor_pool_max_size = max_mem;
+       }
+
+       if (vm_compression_limit == 0) {
+               compressor_pool_size = max_mem;
+       }
+       compressor_pool_multiplier = 1;
+#endif
+       if (compressor_pool_size > compressor_pool_max_size) {
+               compressor_pool_size = compressor_pool_max_size;
+       }
+
+try_again:
+       c_segments_limit = (uint32_t)(compressor_pool_size / (vm_size_t)(C_SEG_ALLOCSIZE));
+       c_segments_nearing_limit = (uint32_t)(((uint64_t)c_segments_limit * 98ULL) / 100ULL);
+
+       c_segment_pages_compressed_limit = (c_segments_limit * (C_SEG_BUFSIZE / PAGE_SIZE) * compressor_pool_multiplier);
+
+       if (c_segment_pages_compressed_limit < (uint32_t)(max_mem / PAGE_SIZE)) {
+               if (!vm_compression_limit) {
+                       c_segment_pages_compressed_limit = (uint32_t)(max_mem / PAGE_SIZE);
+               }
+       }
+
+       c_segment_pages_compressed_nearing_limit = (uint32_t)(((uint64_t)c_segment_pages_compressed_limit * 98ULL) / 100ULL);
+
+#if CONFIG_FREEZE
+       /*
+        * Our in-core limits are based on the size of the compressor pool.
+        * The c_segments_nearing_limit is also based on the compressor pool
+        * size and calculated above.
+        */
+       c_segments_incore_limit = c_segments_limit;
+
+       if (freezer_incore_cseg_acct) {
+               /*
+                * Add enough segments to track all frozen c_segs that can be stored in swap.
+                */
+               c_segments_limit += (uint32_t)(vm_swap_get_max_configured_space() / (vm_size_t)(C_SEG_ALLOCSIZE));
+       }
+#endif
+       /*
+        * Submap needs space for:
+        * - c_segments
+        * - c_buffers
+        * - swap reclaimations -- C_SEG_BUFSIZE
+        */
+       c_segments_arr_size = vm_map_round_page((sizeof(union c_segu) * c_segments_limit), VM_MAP_PAGE_MASK(kernel_map));
+       c_buffers_size = vm_map_round_page(((vm_size_t)C_SEG_ALLOCSIZE * (vm_size_t)c_segments_limit), VM_MAP_PAGE_MASK(kernel_map));
+
+       compressor_submap_size = c_segments_arr_size + c_buffers_size + C_SEG_BUFSIZE;
+
+#if RECORD_THE_COMPRESSED_DATA
+       c_compressed_record_sbuf_size = (vm_size_t)C_SEG_ALLOCSIZE + (PAGE_SIZE * 2);
+       compressor_submap_size += c_compressed_record_sbuf_size;
+#endif /* RECORD_THE_COMPRESSED_DATA */
+
+       vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
+       vmk_flags.vmkf_permanent = TRUE;
+       retval = kmem_suballoc(kernel_map, &start_addr, compressor_submap_size,
+           FALSE, VM_FLAGS_ANYWHERE, vmk_flags, VM_KERN_MEMORY_COMPRESSOR,
+           &compressor_map);
+
+       if (retval != KERN_SUCCESS) {
+               if (++attempts > 3) {
+                       panic("vm_compressor_init: kmem_suballoc failed - 0x%llx", (uint64_t)compressor_submap_size);
+               }
+
+               compressor_pool_size = compressor_pool_size / 2;
+
+               kprintf("retrying creation of the compressor submap at 0x%llx bytes\n", compressor_pool_size);
+               goto try_again;
+       }
+       if (kernel_memory_allocate(compressor_map, (vm_offset_t *)(&c_segments),
+           (sizeof(union c_segu) * c_segments_limit), 0,
+           KMA_KOBJECT | KMA_VAONLY | KMA_PERMANENT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS) {
                panic("vm_compressor_init: kernel_memory_allocate failed - c_segments\n");
                panic("vm_compressor_init: kernel_memory_allocate failed - c_segments\n");
-       c_buffers_size = (vm_size_t)C_SEG_ALLOCSIZE * (vm_size_t)c_segments_limit;
-       if (kernel_memory_allocate(kernel_map, &c_buffers, c_buffers_size, 0, KMA_COMPRESSOR | KMA_VAONLY | KMA_PERMANENT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS)
+       }
+       if (kernel_memory_allocate(compressor_map, &c_buffers, c_buffers_size, 0,
+           KMA_COMPRESSOR | KMA_VAONLY | KMA_PERMANENT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS) {
                panic("vm_compressor_init: kernel_memory_allocate failed - c_buffers\n");
                panic("vm_compressor_init: kernel_memory_allocate failed - c_buffers\n");
+       }
+
+
+       /*
+        * Pick a good size that will minimize fragmentation in zalloc
+        * by minimizing the fragmentation in a 16k run.
+        *
+        * C_SEG_SLOT_VAR_ARRAY_MIN_LEN is larger on 4k systems than 16k ones,
+        * making the fragmentation in a 4k page terrible. Using 16k for all
+        * systems matches zalloc() and will minimize fragmentation.
+        */
+       uint32_t c_segment_size = sizeof(struct c_segment) + (C_SEG_SLOT_VAR_ARRAY_MIN_LEN * sizeof(struct c_slot));
+       uint32_t cnt  = (16 << 10) / c_segment_size;
+       uint32_t frag = (16 << 10) % c_segment_size;
+
+       c_seg_fixed_array_len = C_SEG_SLOT_VAR_ARRAY_MIN_LEN;
+
+       while (cnt * sizeof(struct c_slot) < frag) {
+               c_segment_size += sizeof(struct c_slot);
+               c_seg_fixed_array_len++;
+               frag -= cnt * sizeof(struct c_slot);
+       }
+
+       compressor_segment_zone = zone_create("compressor_segment",
+           c_segment_size, ZC_NOENCRYPT | ZC_ZFREE_CLEARMEM);
+
+       c_segments_busy = FALSE;
 
        c_segments_next_page = (caddr_t)c_segments;
 
        c_segments_next_page = (caddr_t)c_segments;
+       vm_compressor_algorithm_init();
 
        {
                host_basic_info_data_t hinfo;
                mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
 
        {
                host_basic_info_data_t hinfo;
                mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
+               size_t bufsize;
+               char *buf;
 
 #define BSD_HOST 1
                host_info((host_t)BSD_HOST, HOST_BASIC_INFO, (host_info_t)&hinfo, &count);
 
                compressor_cpus = hinfo.max_cpus;
 
 
 #define BSD_HOST 1
                host_info((host_t)BSD_HOST, HOST_BASIC_INFO, (host_info_t)&hinfo, &count);
 
                compressor_cpus = hinfo.max_cpus;
 
-               compressor_scratch_bufs = kalloc_tag(compressor_cpus * WKdm_SCRATCH_BUF_SIZE, VM_KERN_MEMORY_COMPRESSOR);
+               bufsize = PAGE_SIZE;
+               bufsize += compressor_cpus * vm_compressor_get_decode_scratch_size();
+               bufsize += vm_compressor_get_decode_scratch_size();
+#if CONFIG_FREEZE
+               bufsize += vm_compressor_get_encode_scratch_size();
+#endif
+#if RECORD_THE_COMPRESSED_DATA
+               bufsize += c_compressed_record_sbuf_size;
+#endif
 
 
-               kdp_compressor_scratch_buf = kalloc_tag(WKdm_SCRATCH_BUF_SIZE, VM_KERN_MEMORY_COMPRESSOR);
-               kdp_compressor_decompressed_page = kalloc_tag(PAGE_SIZE, VM_KERN_MEMORY_COMPRESSOR);
+               if (kernel_memory_allocate(kernel_map, (vm_offset_t *)&buf, bufsize,
+                   PAGE_MASK, KMA_KOBJECT | KMA_PERMANENT, VM_KERN_MEMORY_COMPRESSOR)) {
+                       panic("vm_compressor_init: Unable to allocate %zd bytes", bufsize);
+               }
+
+               /*
+                * kdp_compressor_decompressed_page must be page aligned because we access
+                * it through the physical apperture by page number.
+                */
+               kdp_compressor_decompressed_page = buf;
                kdp_compressor_decompressed_page_paddr = kvtophys((vm_offset_t)kdp_compressor_decompressed_page);
                kdp_compressor_decompressed_page_ppnum = (ppnum_t) atop(kdp_compressor_decompressed_page_paddr);
                kdp_compressor_decompressed_page_paddr = kvtophys((vm_offset_t)kdp_compressor_decompressed_page);
                kdp_compressor_decompressed_page_ppnum = (ppnum_t) atop(kdp_compressor_decompressed_page_paddr);
-       }
-#if CONFIG_FREEZE              
-       freezer_compressor_scratch_buf = kalloc_tag(WKdm_SCRATCH_BUF_SIZE, VM_KERN_MEMORY_COMPRESSOR);
+               buf += PAGE_SIZE;
+               bufsize -= PAGE_SIZE;
+
+               compressor_scratch_bufs = buf;
+               buf += compressor_cpus * vm_compressor_get_decode_scratch_size();
+               bufsize -= compressor_cpus * vm_compressor_get_decode_scratch_size();
+
+               kdp_compressor_scratch_buf = buf;
+               buf += vm_compressor_get_decode_scratch_size();
+               bufsize -= vm_compressor_get_decode_scratch_size();
+
+#if CONFIG_FREEZE
+               freezer_context_global.freezer_ctx_compressor_scratch_buf = buf;
+               buf += vm_compressor_get_encode_scratch_size();
+               bufsize -= vm_compressor_get_encode_scratch_size();
 #endif
 
 #if RECORD_THE_COMPRESSED_DATA
 #endif
 
 #if RECORD_THE_COMPRESSED_DATA
-       if (kernel_memory_allocate(kernel_map, (vm_offset_t *)&c_compressed_record_sbuf, (vm_size_t)C_SEG_ALLOCSIZE + (PAGE_SIZE * 2), 0, KMA_KOBJECT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS)
-               panic("vm_compressor_init: kernel_memory_allocate failed - c_compressed_record_sbuf\n");
-
-       c_compressed_record_cptr = c_compressed_record_sbuf;
-       c_compressed_record_ebuf = c_compressed_record_sbuf + C_SEG_ALLOCSIZE + (PAGE_SIZE * 2);
+               c_compressed_record_sbuf = buf;
+               c_compressed_record_cptr = buf;
+               c_compressed_record_ebuf = c_compressed_record_sbuf + c_compressed_record_sbuf_size;
+               buf += c_compressed_record_sbuf_size;
+               bufsize -= c_compressed_record_sbuf_size;
 #endif
 #endif
+               assert(bufsize == 0);
+       }
 
        if (kernel_thread_start_priority((thread_continue_t)vm_compressor_swap_trigger_thread, NULL,
 
        if (kernel_thread_start_priority((thread_continue_t)vm_compressor_swap_trigger_thread, NULL,
-                                        BASEPRI_PREEMPT - 1, &thread) != KERN_SUCCESS) {
+           BASEPRI_VM, &thread) != KERN_SUCCESS) {
                panic("vm_compressor_swap_trigger_thread: create failed");
        }
        thread_deallocate(thread);
 
                panic("vm_compressor_swap_trigger_thread: create failed");
        }
        thread_deallocate(thread);
 
-       assert(default_pager_init_flag == 0);
-               
        if (vm_pageout_internal_start() != KERN_SUCCESS) {
                panic("vm_compressor_init: Failed to start the internal pageout thread.\n");
        }
        if (vm_pageout_internal_start() != KERN_SUCCESS) {
                panic("vm_compressor_init: Failed to start the internal pageout thread.\n");
        }
-       if (COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE)
+       if (VM_CONFIG_SWAP_IS_PRESENT) {
                vm_compressor_swap_init();
                vm_compressor_swap_init();
+       }
 
 
-       if (COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_SWAPBACKED)
+       if (VM_CONFIG_COMPRESSOR_IS_ACTIVE) {
                vm_compressor_is_active = 1;
                vm_compressor_is_active = 1;
+       }
 
 #if CONFIG_FREEZE
        memorystatus_freeze_enabled = TRUE;
 #endif /* CONFIG_FREEZE */
 
 
 #if CONFIG_FREEZE
        memorystatus_freeze_enabled = TRUE;
 #endif /* CONFIG_FREEZE */
 
-       default_pager_init_flag = 1;
        vm_compressor_available = 1;
 
        vm_page_reactivate_all_throttled();
        vm_compressor_available = 1;
 
        vm_page_reactivate_all_throttled();
@@ -566,28 +925,30 @@ vm_compressor_init(void)
 static void
 c_seg_validate(c_segment_t c_seg, boolean_t must_be_compact)
 {
 static void
 c_seg_validate(c_segment_t c_seg, boolean_t must_be_compact)
 {
-       int             c_indx;
-       int32_t         bytes_used;
-       int32_t         bytes_unused;
-       uint32_t        c_rounded_size;
-       uint32_t        c_size;
-       c_slot_t        cs;
+       uint16_t        c_indx;
+       int32_t         bytes_used;
+       uint32_t        c_rounded_size;
+       uint32_t        c_size;
+       c_slot_t        cs;
 
 
+       if (__probable(validate_c_segs == FALSE)) {
+               return;
+       }
        if (c_seg->c_firstemptyslot < c_seg->c_nextslot) {
                c_indx = c_seg->c_firstemptyslot;
                cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
 
        if (c_seg->c_firstemptyslot < c_seg->c_nextslot) {
                c_indx = c_seg->c_firstemptyslot;
                cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
 
-               if (cs == NULL)
+               if (cs == NULL) {
                        panic("c_seg_validate:  no slot backing c_firstemptyslot");
                        panic("c_seg_validate:  no slot backing c_firstemptyslot");
-       
-               if (cs->c_size)
+               }
+
+               if (cs->c_size) {
                        panic("c_seg_validate:  c_firstemptyslot has non-zero size (%d)\n", cs->c_size);
                        panic("c_seg_validate:  c_firstemptyslot has non-zero size (%d)\n", cs->c_size);
+               }
        }
        bytes_used = 0;
        }
        bytes_used = 0;
-       bytes_unused = 0;
 
        for (c_indx = 0; c_indx < c_seg->c_nextslot; c_indx++) {
 
        for (c_indx = 0; c_indx < c_seg->c_nextslot; c_indx++) {
-               
                cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
 
                c_size = UNPACK_C_SIZE(cs);
                cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
 
                c_size = UNPACK_C_SIZE(cs);
@@ -597,22 +958,37 @@ c_seg_validate(c_segment_t c_seg, boolean_t must_be_compact)
                bytes_used += c_rounded_size;
 
 #if CHECKSUM_THE_COMPRESSED_DATA
                bytes_used += c_rounded_size;
 
 #if CHECKSUM_THE_COMPRESSED_DATA
-               if (c_size && cs->c_hash_compressed_data != hash_string((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size))
-                       panic("compressed data doesn't match original");
+               unsigned csvhash;
+               if (c_size && cs->c_hash_compressed_data != (csvhash = vmc_hash((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size))) {
+                       addr64_t csvphys = kvtophys((vm_offset_t)&c_seg->c_store.c_buffer[cs->c_offset]);
+                       panic("Compressed data doesn't match original %p phys: 0x%llx %d %p %d %d 0x%x 0x%x", c_seg, csvphys, cs->c_offset, cs, c_indx, c_size, cs->c_hash_compressed_data, csvhash);
+               }
+#endif
+#if POPCOUNT_THE_COMPRESSED_DATA
+               unsigned csvpop;
+               if (c_size) {
+                       uintptr_t csvaddr = (uintptr_t) &c_seg->c_store.c_buffer[cs->c_offset];
+                       if (cs->c_pop_cdata != (csvpop = vmc_pop(csvaddr, c_size))) {
+                               panic("Compressed data popcount doesn't match original, bit distance: %d %p (phys: %p) %p %p 0x%llx 0x%x 0x%x 0x%x", (csvpop - cs->c_pop_cdata), (void *)csvaddr, (void *) kvtophys(csvaddr), c_seg, cs, (uint64_t)cs->c_offset, c_size, csvpop, cs->c_pop_cdata);
+                       }
+               }
 #endif
        }
 
 #endif
        }
 
-       if (bytes_used != c_seg->c_bytes_used)
+       if (bytes_used != c_seg->c_bytes_used) {
                panic("c_seg_validate: bytes_used mismatch - found %d, segment has %d\n", bytes_used, c_seg->c_bytes_used);
                panic("c_seg_validate: bytes_used mismatch - found %d, segment has %d\n", bytes_used, c_seg->c_bytes_used);
+       }
 
 
-       if (c_seg->c_bytes_used > C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset))
+       if (c_seg->c_bytes_used > C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset)) {
                panic("c_seg_validate: c_bytes_used > c_nextoffset - c_nextoffset = %d,  c_bytes_used = %d\n",
                panic("c_seg_validate: c_bytes_used > c_nextoffset - c_nextoffset = %d,  c_bytes_used = %d\n",
-                     (int32_t)C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset), c_seg->c_bytes_used);
+                   (int32_t)C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset), c_seg->c_bytes_used);
+       }
 
        if (must_be_compact) {
 
        if (must_be_compact) {
-               if (c_seg->c_bytes_used != C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset))
+               if (c_seg->c_bytes_used != C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset)) {
                        panic("c_seg_validate: c_bytes_used doesn't match c_nextoffset - c_nextoffset = %d,  c_bytes_used = %d\n",
                        panic("c_seg_validate: c_bytes_used doesn't match c_nextoffset - c_nextoffset = %d,  c_bytes_used = %d\n",
-                             (int32_t)C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset), c_seg->c_bytes_used);
+                           (int32_t)C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset), c_seg->c_bytes_used);
+               }
        }
 }
 
        }
 }
 
@@ -620,30 +996,35 @@ c_seg_validate(c_segment_t c_seg, boolean_t must_be_compact)
 
 
 void
 
 
 void
-c_seg_need_delayed_compaction(c_segment_t c_seg)
+c_seg_need_delayed_compaction(c_segment_t c_seg, boolean_t c_list_lock_held)
 {
 {
-       boolean_t       clear_busy = FALSE;
+       boolean_t       clear_busy = FALSE;
 
 
-       if ( !lck_mtx_try_lock_spin_always(c_list_lock)) {
-               C_SEG_BUSY(c_seg);
-               
-               lck_mtx_unlock_always(&c_seg->c_lock);
-               lck_mtx_lock_spin_always(c_list_lock);
-               lck_mtx_lock_spin_always(&c_seg->c_lock);
+       if (c_list_lock_held == FALSE) {
+               if (!lck_mtx_try_lock_spin_always(c_list_lock)) {
+                       C_SEG_BUSY(c_seg);
 
 
-               clear_busy = TRUE;
+                       lck_mtx_unlock_always(&c_seg->c_lock);
+                       lck_mtx_lock_spin_always(c_list_lock);
+                       lck_mtx_lock_spin_always(&c_seg->c_lock);
+
+                       clear_busy = TRUE;
+               }
        }
        assert(c_seg->c_state != C_IS_FILLING);
 
        }
        assert(c_seg->c_state != C_IS_FILLING);
 
-       if (!c_seg->c_on_minorcompact_q && !(C_SEG_IS_ONDISK(c_seg))) {
+       if (!c_seg->c_on_minorcompact_q && !(C_SEG_IS_ON_DISK_OR_SOQ(c_seg))) {
                queue_enter(&c_minor_list_head, c_seg, c_segment_t, c_list);
                c_seg->c_on_minorcompact_q = 1;
                c_minor_count++;
        }
                queue_enter(&c_minor_list_head, c_seg, c_segment_t, c_list);
                c_seg->c_on_minorcompact_q = 1;
                c_minor_count++;
        }
-       lck_mtx_unlock_always(c_list_lock);
-       
-       if (clear_busy == TRUE)
+       if (c_list_lock_held == FALSE) {
+               lck_mtx_unlock_always(c_list_lock);
+       }
+
+       if (clear_busy == TRUE) {
                C_SEG_WAKEUP_DONE(c_seg);
                C_SEG_WAKEUP_DONE(c_seg);
+       }
 }
 
 
 }
 
 
@@ -652,15 +1033,15 @@ unsigned int c_seg_moved_to_sparse_list = 0;
 void
 c_seg_move_to_sparse_list(c_segment_t c_seg)
 {
 void
 c_seg_move_to_sparse_list(c_segment_t c_seg)
 {
-       boolean_t       clear_busy = FALSE;
+       boolean_t       clear_busy = FALSE;
 
 
-       if ( !lck_mtx_try_lock_spin_always(c_list_lock)) {
+       if (!lck_mtx_try_lock_spin_always(c_list_lock)) {
                C_SEG_BUSY(c_seg);
 
                lck_mtx_unlock_always(&c_seg->c_lock);
                lck_mtx_lock_spin_always(c_list_lock);
                lck_mtx_lock_spin_always(&c_seg->c_lock);
                C_SEG_BUSY(c_seg);
 
                lck_mtx_unlock_always(&c_seg->c_lock);
                lck_mtx_lock_spin_always(c_list_lock);
                lck_mtx_lock_spin_always(&c_seg->c_lock);
-               
+
                clear_busy = TRUE;
        }
        c_seg_switch_state(c_seg, C_ON_SWAPPEDOUTSPARSE_Q, FALSE);
                clear_busy = TRUE;
        }
        c_seg_switch_state(c_seg, C_ON_SWAPPEDOUTSPARSE_Q, FALSE);
@@ -669,8 +1050,9 @@ c_seg_move_to_sparse_list(c_segment_t c_seg)
 
        lck_mtx_unlock_always(c_list_lock);
 
 
        lck_mtx_unlock_always(c_list_lock);
 
-       if (clear_busy == TRUE)
+       if (clear_busy == TRUE) {
                C_SEG_WAKEUP_DONE(c_seg);
                C_SEG_WAKEUP_DONE(c_seg);
+       }
 }
 
 
 }
 
 
@@ -685,13 +1067,12 @@ c_seg_insert_into_q(queue_head_t *qhead, c_segment_t c_seg)
                c_seg_next = (c_segment_t)queue_first(qhead);
 
                while (TRUE) {
                c_seg_next = (c_segment_t)queue_first(qhead);
 
                while (TRUE) {
-
                        if (c_seg->c_generation_id < c_seg_next->c_generation_id) {
                                queue_insert_before(qhead, c_seg, c_seg_next, c_segment_t, c_age_list);
                                break;
                        }
                        c_seg_next = (c_segment_t) queue_next(&c_seg_next->c_age_list);
                        if (c_seg->c_generation_id < c_seg_next->c_generation_id) {
                                queue_insert_before(qhead, c_seg, c_seg_next, c_segment_t, c_age_list);
                                break;
                        }
                        c_seg_next = (c_segment_t) queue_next(&c_seg_next->c_age_list);
-               
+
                        if (queue_end(qhead, (queue_entry_t) c_seg_next)) {
                                queue_enter(qhead, c_seg, c_segment_t, c_age_list);
                                break;
                        if (queue_end(qhead, (queue_entry_t) c_seg_next)) {
                                queue_enter(qhead, c_seg, c_segment_t, c_age_list);
                                break;
@@ -707,7 +1088,6 @@ int try_minor_compaction_succeeded = 0;
 void
 c_seg_try_minor_compaction_and_unlock(c_segment_t c_seg)
 {
 void
 c_seg_try_minor_compaction_and_unlock(c_segment_t c_seg)
 {
-
        assert(c_seg->c_on_minorcompact_q);
        /*
         * c_seg is currently on the delayed minor compaction
        assert(c_seg->c_on_minorcompact_q);
        /*
         * c_seg is currently on the delayed minor compaction
@@ -716,7 +1096,7 @@ c_seg_try_minor_compaction_and_unlock(c_segment_t c_seg)
         * because the lock order is c_list_lock then c_seg's lock)
         * we'll pull it from the delayed list and free it directly
         */
         * because the lock order is c_list_lock then c_seg's lock)
         * we'll pull it from the delayed list and free it directly
         */
-       if ( !lck_mtx_try_lock_spin_always(c_list_lock)) {
+       if (!lck_mtx_try_lock_spin_always(c_list_lock)) {
                /*
                 * c_list_lock is held, we need to bail
                 */
                /*
                 * c_list_lock is held, we need to bail
                 */
@@ -735,9 +1115,10 @@ c_seg_try_minor_compaction_and_unlock(c_segment_t c_seg)
 int
 c_seg_do_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy, boolean_t need_list_lock, boolean_t disallow_page_replacement)
 {
 int
 c_seg_do_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy, boolean_t need_list_lock, boolean_t disallow_page_replacement)
 {
-       int     c_seg_freed;
+       int     c_seg_freed;
 
        assert(c_seg->c_busy);
 
        assert(c_seg->c_busy);
+       assert(!C_SEG_IS_ON_DISK_OR_SOQ(c_seg));
 
        /*
         * check for the case that can occur when we are not swapping
 
        /*
         * check for the case that can occur when we are not swapping
@@ -748,22 +1129,22 @@ c_seg_do_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy, bo
         * in the next major compaction sweep... if we don't do this
         * we will eventually run into the c_segments_limit
         */
         * in the next major compaction sweep... if we don't do this
         * we will eventually run into the c_segments_limit
         */
-       if (c_seg->c_state == C_ON_MAJORCOMPACT_Q && C_SEG_SHOULD_MAJORCOMPACT(c_seg)) {
-               
+       if (c_seg->c_state == C_ON_MAJORCOMPACT_Q && C_SEG_SHOULD_MAJORCOMPACT_NOW(c_seg)) {
                c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
        }
        if (!c_seg->c_on_minorcompact_q) {
                c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
        }
        if (!c_seg->c_on_minorcompact_q) {
-               if (clear_busy == TRUE)
+               if (clear_busy == TRUE) {
                        C_SEG_WAKEUP_DONE(c_seg);
                        C_SEG_WAKEUP_DONE(c_seg);
+               }
 
                lck_mtx_unlock_always(&c_seg->c_lock);
 
 
                lck_mtx_unlock_always(&c_seg->c_lock);
 
-               return (0);
+               return 0;
        }
        queue_remove(&c_minor_list_head, c_seg, c_segment_t, c_list);
        c_seg->c_on_minorcompact_q = 0;
        c_minor_count--;
        }
        queue_remove(&c_minor_list_head, c_seg, c_segment_t, c_list);
        c_seg->c_on_minorcompact_q = 0;
        c_minor_count--;
-       
+
        lck_mtx_unlock_always(c_list_lock);
 
        if (disallow_page_replacement == TRUE) {
        lck_mtx_unlock_always(c_list_lock);
 
        if (disallow_page_replacement == TRUE) {
@@ -775,201 +1156,370 @@ c_seg_do_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy, bo
        }
        c_seg_freed = c_seg_minor_compaction_and_unlock(c_seg, clear_busy);
 
        }
        c_seg_freed = c_seg_minor_compaction_and_unlock(c_seg, clear_busy);
 
-       if (disallow_page_replacement == TRUE)
-               PAGE_REPLACEMENT_DISALLOWED(FALSE);
+       if (disallow_page_replacement == TRUE) {
+               PAGE_REPLACEMENT_DISALLOWED(FALSE);
+       }
+
+       if (need_list_lock == TRUE) {
+               lck_mtx_lock_spin_always(c_list_lock);
+       }
+
+       return c_seg_freed;
+}
+
+void
+kdp_compressor_busy_find_owner(event64_t wait_event, thread_waitinfo_t *waitinfo)
+{
+       c_segment_t c_seg = (c_segment_t) wait_event;
+
+       waitinfo->owner = thread_tid(c_seg->c_busy_for_thread);
+       waitinfo->context = VM_KERNEL_UNSLIDE_OR_PERM(c_seg);
+}
+
+#if DEVELOPMENT || DEBUG
+int
+do_cseg_wedge_thread(void)
+{
+       struct c_segment c_seg;
+       c_seg.c_busy_for_thread = current_thread();
+
+       debug_cseg_wait_event = (event_t) &c_seg;
+
+       thread_set_pending_block_hint(current_thread(), kThreadWaitCompressor);
+       assert_wait((event_t) (&c_seg), THREAD_INTERRUPTIBLE);
+
+       thread_block(THREAD_CONTINUE_NULL);
+
+       return 0;
+}
+
+int
+do_cseg_unwedge_thread(void)
+{
+       thread_wakeup(debug_cseg_wait_event);
+       debug_cseg_wait_event = NULL;
+
+       return 0;
+}
+#endif /* DEVELOPMENT || DEBUG */
+
+void
+c_seg_wait_on_busy(c_segment_t c_seg)
+{
+       c_seg->c_wanted = 1;
+
+       thread_set_pending_block_hint(current_thread(), kThreadWaitCompressor);
+       assert_wait((event_t) (c_seg), THREAD_UNINT);
+
+       lck_mtx_unlock_always(&c_seg->c_lock);
+       thread_block(THREAD_CONTINUE_NULL);
+}
+
+#if CONFIG_FREEZE
+/*
+ * We don't have the task lock held while updating the task's
+ * c_seg queues. We can do that because of the following restrictions:
+ *
+ * - SINGLE FREEZER CONTEXT:
+ *   We 'insert' c_segs into the task list on the task_freeze path.
+ *   There can only be one such freeze in progress and the task
+ *   isn't disappearing because we have the VM map lock held throughout
+ *   and we have a reference on the proc too.
+ *
+ * - SINGLE TASK DISOWN CONTEXT:
+ *   We 'disown' c_segs of a task ONLY from the task_terminate context. So
+ *   we don't need the task lock but we need the c_list_lock and the
+ *   compressor master lock (shared). We also hold the individual
+ *   c_seg locks (exclusive).
+ *
+ *   If we either:
+ *   - can't get the c_seg lock on a try, then we start again because maybe
+ *   the c_seg is part of a compaction and might get freed. So we can't trust
+ *   that linkage and need to restart our queue traversal.
+ *   - OR, we run into a busy c_seg (say being swapped in or free-ing) we
+ *   drop all locks again and wait and restart our queue traversal.
+ *
+ * - The new_owner_task below is currently only the kernel or NULL.
+ *
+ */
+void
+c_seg_update_task_owner(c_segment_t c_seg, task_t new_owner_task)
+{
+       task_t          owner_task = c_seg->c_task_owner;
+       uint64_t        uncompressed_bytes = ((c_seg->c_slots_used) * PAGE_SIZE_64);
+
+       LCK_MTX_ASSERT(c_list_lock, LCK_MTX_ASSERT_OWNED);
+       LCK_MTX_ASSERT(&c_seg->c_lock, LCK_MTX_ASSERT_OWNED);
+
+       if (owner_task) {
+               task_update_frozen_to_swap_acct(owner_task, uncompressed_bytes, DEBIT_FROM_SWAP);
+               queue_remove(&owner_task->task_frozen_cseg_q, c_seg,
+                   c_segment_t, c_task_list_next_cseg);
+       }
 
 
-       if (need_list_lock == TRUE)
-               lck_mtx_lock_spin_always(c_list_lock);
+       if (new_owner_task) {
+               queue_enter(&new_owner_task->task_frozen_cseg_q, c_seg,
+                   c_segment_t, c_task_list_next_cseg);
+               task_update_frozen_to_swap_acct(new_owner_task, uncompressed_bytes, CREDIT_TO_SWAP);
+       }
 
 
-       return (c_seg_freed);
+       c_seg->c_task_owner = new_owner_task;
 }
 
 }
 
-
 void
 void
-c_seg_wait_on_busy(c_segment_t c_seg)
+task_disown_frozen_csegs(task_t owner_task)
 {
 {
-       c_seg->c_wanted = 1;
-       assert_wait((event_t) (c_seg), THREAD_UNINT);
+       c_segment_t c_seg = NULL, next_cseg = NULL;
 
 
-       lck_mtx_unlock_always(&c_seg->c_lock);
-       thread_block(THREAD_CONTINUE_NULL);
-}
+again:
+       PAGE_REPLACEMENT_DISALLOWED(TRUE);
+       lck_mtx_lock_spin_always(c_list_lock);
+
+       for (c_seg = (c_segment_t) queue_first(&owner_task->task_frozen_cseg_q);
+           !queue_end(&owner_task->task_frozen_cseg_q, (queue_entry_t) c_seg);
+           c_seg = next_cseg) {
+               next_cseg = (c_segment_t) queue_next(&c_seg->c_task_list_next_cseg);;
+
+               if (!lck_mtx_try_lock_spin_always(&c_seg->c_lock)) {
+                       lck_mtx_unlock(c_list_lock);
+                       PAGE_REPLACEMENT_DISALLOWED(FALSE);
+                       goto again;
+               }
+
+               if (c_seg->c_busy) {
+                       lck_mtx_unlock(c_list_lock);
+                       PAGE_REPLACEMENT_DISALLOWED(FALSE);
+
+                       c_seg_wait_on_busy(c_seg);
+
+                       goto again;
+               }
+               assert(c_seg->c_task_owner == owner_task);
+               c_seg_update_task_owner(c_seg, kernel_task);
+               lck_mtx_unlock_always(&c_seg->c_lock);
+       }
 
 
+       lck_mtx_unlock(c_list_lock);
+       PAGE_REPLACEMENT_DISALLOWED(FALSE);
+}
+#endif /* CONFIG_FREEZE */
 
 void
 c_seg_switch_state(c_segment_t c_seg, int new_state, boolean_t insert_head)
 {
 
 void
 c_seg_switch_state(c_segment_t c_seg, int new_state, boolean_t insert_head)
 {
-       int     old_state = c_seg->c_state;
+       int     old_state = c_seg->c_state;
 
 
-#if DEVELOPMENT || DEBUG
-#if __i386__ || __x86_64__
-       if (new_state != C_IS_FILLING)
-               lck_mtx_assert(&c_seg->c_lock, LCK_MTX_ASSERT_OWNED);
-       lck_mtx_assert(c_list_lock, LCK_MTX_ASSERT_OWNED);
-#endif
+#if XNU_TARGET_OS_OSX
+#if     DEVELOPMENT || DEBUG
+       if (new_state != C_IS_FILLING) {
+               LCK_MTX_ASSERT(&c_seg->c_lock, LCK_MTX_ASSERT_OWNED);
+       }
+       LCK_MTX_ASSERT(c_list_lock, LCK_MTX_ASSERT_OWNED);
 #endif
 #endif
+#endif /* XNU_TARGET_OS_OSX */
        switch (old_state) {
        switch (old_state) {
+       case C_IS_EMPTY:
+               assert(new_state == C_IS_FILLING || new_state == C_IS_FREE);
 
 
-               case C_IS_EMPTY:
-                       assert(new_state == C_IS_FILLING || new_state == C_IS_FREE);
+               c_empty_count--;
+               break;
 
 
-                       c_empty_count--;
-                       break;
+       case C_IS_FILLING:
+               assert(new_state == C_ON_AGE_Q || new_state == C_ON_SWAPOUT_Q);
 
 
-               case C_IS_FILLING:
-                       assert(new_state == C_ON_AGE_Q || new_state == C_ON_SWAPOUT_Q);
+               queue_remove(&c_filling_list_head, c_seg, c_segment_t, c_age_list);
+               c_filling_count--;
+               break;
 
 
-                       queue_remove(&c_filling_list_head, c_seg, c_segment_t, c_age_list);
-                       c_filling_count--;
-                       break;
+       case C_ON_AGE_Q:
+               assert(new_state == C_ON_SWAPOUT_Q || new_state == C_ON_MAJORCOMPACT_Q ||
+                   new_state == C_IS_FREE);
 
 
-               case C_ON_AGE_Q:
-                       assert(new_state == C_ON_SWAPOUT_Q || new_state == C_ON_MAJORCOMPACT_Q ||
-                              new_state == C_IS_FREE);
+               queue_remove(&c_age_list_head, c_seg, c_segment_t, c_age_list);
+               c_age_count--;
+               break;
 
 
-                       queue_remove(&c_age_list_head, c_seg, c_segment_t, c_age_list);
-                       c_age_count--;
-                       break;
-               
-               case C_ON_SWAPPEDIN_Q:
-                       assert(new_state == C_ON_AGE_Q || new_state == C_IS_FREE);
+       case C_ON_SWAPPEDIN_Q:
+               assert(new_state == C_ON_AGE_Q || new_state == C_IS_FREE);
 
 
-                       queue_remove(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
-                       c_swappedin_count--;
-                       break;
+               queue_remove(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
+               c_swappedin_count--;
+               break;
 
 
-               case C_ON_SWAPOUT_Q:
-                       assert(new_state == C_ON_SWAPPEDOUT_Q || new_state == C_ON_SWAPPEDOUTSPARSE_Q ||
-                              new_state == C_ON_AGE_Q || new_state == C_IS_FREE || new_state == C_IS_EMPTY);
+       case C_ON_SWAPOUT_Q:
+               assert(new_state == C_ON_AGE_Q || new_state == C_IS_FREE || new_state == C_IS_EMPTY || new_state == C_ON_SWAPIO_Q);
 
 
-                       queue_remove(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
-                       thread_wakeup((event_t)&compaction_swapper_running);
-                       c_swapout_count--;
-                       break;
+#if CONFIG_FREEZE
+               if (c_seg->c_task_owner && (new_state != C_ON_SWAPIO_Q)) {
+                       c_seg_update_task_owner(c_seg, NULL);
+               }
+#endif /* CONFIG_FREEZE */
 
 
-               case C_ON_SWAPPEDOUT_Q:
-                       assert(new_state == C_ON_SWAPPEDIN_Q || new_state == C_ON_SWAPPEDOUTSPARSE_Q ||
-                              new_state == C_ON_BAD_Q || new_state == C_IS_EMPTY || new_state == C_IS_FREE);
+               queue_remove(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
+               thread_wakeup((event_t)&compaction_swapper_running);
+               c_swapout_count--;
+               break;
 
 
-                       queue_remove(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
-                       c_swappedout_count--;
-                       break;
+       case C_ON_SWAPIO_Q:
+               assert(new_state == C_ON_SWAPPEDOUT_Q || new_state == C_ON_SWAPPEDOUTSPARSE_Q || new_state == C_ON_AGE_Q);
 
 
-               case C_ON_SWAPPEDOUTSPARSE_Q:
-                       assert(new_state == C_ON_SWAPPEDIN_Q ||
-                              new_state == C_ON_BAD_Q || new_state == C_IS_EMPTY || new_state == C_IS_FREE);
+               queue_remove(&c_swapio_list_head, c_seg, c_segment_t, c_age_list);
+               c_swapio_count--;
+               break;
 
 
-                       queue_remove(&c_swappedout_sparse_list_head, c_seg, c_segment_t, c_age_list);
-                       c_swappedout_sparse_count--;
-                       break;
+       case C_ON_SWAPPEDOUT_Q:
+               assert(new_state == C_ON_SWAPPEDIN_Q || new_state == C_ON_AGE_Q ||
+                   new_state == C_ON_SWAPPEDOUTSPARSE_Q ||
+                   new_state == C_ON_BAD_Q || new_state == C_IS_EMPTY || new_state == C_IS_FREE);
 
 
-               case C_ON_MAJORCOMPACT_Q:
-                       assert(new_state == C_ON_AGE_Q || new_state == C_IS_FREE);
+               queue_remove(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
+               c_swappedout_count--;
+               break;
 
 
-                       queue_remove(&c_major_list_head, c_seg, c_segment_t, c_age_list);
-                       c_major_count--;
-                       break;
+       case C_ON_SWAPPEDOUTSPARSE_Q:
+               assert(new_state == C_ON_SWAPPEDIN_Q || new_state == C_ON_AGE_Q ||
+                   new_state == C_ON_BAD_Q || new_state == C_IS_EMPTY || new_state == C_IS_FREE);
 
 
-               case C_ON_BAD_Q:
-                       assert(new_state == C_IS_FREE);
+               queue_remove(&c_swappedout_sparse_list_head, c_seg, c_segment_t, c_age_list);
+               c_swappedout_sparse_count--;
+               break;
 
 
-                       queue_remove(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
-                       c_bad_count--;
-                       break;
+       case C_ON_MAJORCOMPACT_Q:
+               assert(new_state == C_ON_AGE_Q || new_state == C_IS_FREE);
 
 
-               default:
-                       panic("c_seg %p has bad c_state = %d\n", c_seg, old_state);
+               queue_remove(&c_major_list_head, c_seg, c_segment_t, c_age_list);
+               c_major_count--;
+               break;
+
+       case C_ON_BAD_Q:
+               assert(new_state == C_IS_FREE);
+
+               queue_remove(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
+               c_bad_count--;
+               break;
+
+       default:
+               panic("c_seg %p has bad c_state = %d\n", c_seg, old_state);
        }
 
        }
 
-       switch(new_state) {
-               case C_IS_FREE:
-                       assert(old_state != C_IS_FILLING);
+       switch (new_state) {
+       case C_IS_FREE:
+               assert(old_state != C_IS_FILLING);
 
 
-                       break;
+               break;
 
 
-               case C_IS_EMPTY:
-                       assert(old_state == C_ON_SWAPOUT_Q || old_state == C_ON_SWAPPEDOUT_Q || old_state == C_ON_SWAPPEDOUTSPARSE_Q);
+       case C_IS_EMPTY:
+               assert(old_state == C_ON_SWAPOUT_Q || old_state == C_ON_SWAPPEDOUT_Q || old_state == C_ON_SWAPPEDOUTSPARSE_Q);
 
 
-                       c_empty_count++;
-                       break;
+               c_empty_count++;
+               break;
 
 
-               case C_IS_FILLING:
-                       assert(old_state == C_IS_EMPTY);
+       case C_IS_FILLING:
+               assert(old_state == C_IS_EMPTY);
 
 
-                       queue_enter(&c_filling_list_head, c_seg, c_segment_t, c_age_list);
-                       c_filling_count++;
-                       break;
+               queue_enter(&c_filling_list_head, c_seg, c_segment_t, c_age_list);
+               c_filling_count++;
+               break;
 
 
-               case C_ON_AGE_Q:
-                       assert(old_state == C_IS_FILLING || old_state == C_ON_SWAPPEDIN_Q ||
-                              old_state == C_ON_MAJORCOMPACT_Q || old_state == C_ON_SWAPOUT_Q);
+       case C_ON_AGE_Q:
+               assert(old_state == C_IS_FILLING || old_state == C_ON_SWAPPEDIN_Q ||
+                   old_state == C_ON_SWAPOUT_Q || old_state == C_ON_SWAPIO_Q ||
+                   old_state == C_ON_MAJORCOMPACT_Q || old_state == C_ON_SWAPPEDOUT_Q || old_state == C_ON_SWAPPEDOUTSPARSE_Q);
 
 
-                       if (old_state == C_IS_FILLING)
-                               queue_enter(&c_age_list_head, c_seg, c_segment_t, c_age_list);
-                       else
-                               c_seg_insert_into_q(&c_age_list_head, c_seg);
-                       c_age_count++;
-                       break;
-               
-               case C_ON_SWAPPEDIN_Q:
-                       assert(c_seg->c_state == C_ON_SWAPPEDOUT_Q || c_seg->c_state == C_ON_SWAPPEDOUTSPARSE_Q);
-
-                       if (insert_head == TRUE)
-                               queue_enter_first(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
-                       else
-                               queue_enter(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
-                       c_swappedin_count++;
-                       break;
+               if (old_state == C_IS_FILLING) {
+                       queue_enter(&c_age_list_head, c_seg, c_segment_t, c_age_list);
+               } else {
+                       if (!queue_empty(&c_age_list_head)) {
+                               c_segment_t     c_first;
 
 
-               case C_ON_SWAPOUT_Q:
-                       assert(old_state == C_ON_AGE_Q || old_state == C_IS_FILLING);
+                               c_first = (c_segment_t)queue_first(&c_age_list_head);
+                               c_seg->c_creation_ts = c_first->c_creation_ts;
+                       }
+                       queue_enter_first(&c_age_list_head, c_seg, c_segment_t, c_age_list);
+               }
+               c_age_count++;
+               break;
 
 
-                       if (insert_head == TRUE)
-                               queue_enter_first(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
-                       else
-                               queue_enter(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
-                       c_swapout_count++;
-                       break;
+       case C_ON_SWAPPEDIN_Q:
+               assert(old_state == C_ON_SWAPPEDOUT_Q || old_state == C_ON_SWAPPEDOUTSPARSE_Q);
 
 
-               case C_ON_SWAPPEDOUT_Q:
-                       assert(c_seg->c_state == C_ON_SWAPOUT_Q);
+               if (insert_head == TRUE) {
+                       queue_enter_first(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
+               } else {
+                       queue_enter(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
+               }
+               c_swappedin_count++;
+               break;
 
 
-                       if (insert_head == TRUE)
-                               queue_enter_first(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
-                       else
-                               queue_enter(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
-                       c_swappedout_count++;
-                       break;
+       case C_ON_SWAPOUT_Q:
+               assert(old_state == C_ON_AGE_Q || old_state == C_IS_FILLING);
 
 
-               case C_ON_SWAPPEDOUTSPARSE_Q:
-                       assert(c_seg->c_state == C_ON_SWAPOUT_Q || c_seg->c_state == C_ON_SWAPPEDOUT_Q);
-                       
-                       c_seg_insert_into_q(&c_swappedout_sparse_list_head, c_seg);
-                       c_swappedout_sparse_count++;
-                       break;
+               if (insert_head == TRUE) {
+                       queue_enter_first(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
+               } else {
+                       queue_enter(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
+               }
+               c_swapout_count++;
+               break;
 
 
-               case C_ON_MAJORCOMPACT_Q:
-                       assert(c_seg->c_state == C_ON_AGE_Q);
+       case C_ON_SWAPIO_Q:
+               assert(old_state == C_ON_SWAPOUT_Q);
 
 
-                       if (insert_head == TRUE)
-                               queue_enter_first(&c_major_list_head, c_seg, c_segment_t, c_age_list);
-                       else
-                               queue_enter(&c_major_list_head, c_seg, c_segment_t, c_age_list);
-                       c_major_count++;
-                       break;
+               if (insert_head == TRUE) {
+                       queue_enter_first(&c_swapio_list_head, c_seg, c_segment_t, c_age_list);
+               } else {
+                       queue_enter(&c_swapio_list_head, c_seg, c_segment_t, c_age_list);
+               }
+               c_swapio_count++;
+               break;
 
 
-               case C_ON_BAD_Q:
-                       assert(c_seg->c_state == C_ON_SWAPPEDOUT_Q || c_seg->c_state == C_ON_SWAPPEDOUTSPARSE_Q);
+       case C_ON_SWAPPEDOUT_Q:
+               assert(old_state == C_ON_SWAPIO_Q);
 
 
-                       if (insert_head == TRUE)
-                               queue_enter_first(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
-                       else
-                               queue_enter(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
-                       c_bad_count++;
-                       break;
+               if (insert_head == TRUE) {
+                       queue_enter_first(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
+               } else {
+                       queue_enter(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
+               }
+               c_swappedout_count++;
+               break;
+
+       case C_ON_SWAPPEDOUTSPARSE_Q:
+               assert(old_state == C_ON_SWAPIO_Q || old_state == C_ON_SWAPPEDOUT_Q);
+
+               if (insert_head == TRUE) {
+                       queue_enter_first(&c_swappedout_sparse_list_head, c_seg, c_segment_t, c_age_list);
+               } else {
+                       queue_enter(&c_swappedout_sparse_list_head, c_seg, c_segment_t, c_age_list);
+               }
+
+               c_swappedout_sparse_count++;
+               break;
+
+       case C_ON_MAJORCOMPACT_Q:
+               assert(old_state == C_ON_AGE_Q);
+
+               if (insert_head == TRUE) {
+                       queue_enter_first(&c_major_list_head, c_seg, c_segment_t, c_age_list);
+               } else {
+                       queue_enter(&c_major_list_head, c_seg, c_segment_t, c_age_list);
+               }
+               c_major_count++;
+               break;
+
+       case C_ON_BAD_Q:
+               assert(old_state == C_ON_SWAPPEDOUT_Q || old_state == C_ON_SWAPPEDOUTSPARSE_Q);
+
+               if (insert_head == TRUE) {
+                       queue_enter_first(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
+               } else {
+                       queue_enter(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
+               }
+               c_bad_count++;
+               break;
 
 
-               default:
-                      panic("c_seg %p requesting bad c_state = %d\n", c_seg, new_state);
+       default:
+               panic("c_seg %p requesting bad c_state = %d\n", c_seg, new_state);
        }
        c_seg->c_state = new_state;
 }
        }
        c_seg->c_state = new_state;
 }
@@ -992,48 +1542,59 @@ c_seg_free(c_segment_t c_seg)
 void
 c_seg_free_locked(c_segment_t c_seg)
 {
 void
 c_seg_free_locked(c_segment_t c_seg)
 {
-       int             segno;
-       int             pages_populated = 0;
-       int32_t         *c_buffer = NULL;
-       uint64_t        c_swap_handle = 0;
+       int             segno;
+       int             pages_populated = 0;
+       int32_t         *c_buffer = NULL;
+       uint64_t        c_swap_handle = 0;
 
        assert(c_seg->c_busy);
 
        assert(c_seg->c_busy);
+       assert(c_seg->c_slots_used == 0);
        assert(!c_seg->c_on_minorcompact_q);
        assert(!c_seg->c_busy_swapping);
 
        if (c_seg->c_overage_swap == TRUE) {
                c_overage_swapped_count--;
                c_seg->c_overage_swap = FALSE;
        assert(!c_seg->c_on_minorcompact_q);
        assert(!c_seg->c_busy_swapping);
 
        if (c_seg->c_overage_swap == TRUE) {
                c_overage_swapped_count--;
                c_seg->c_overage_swap = FALSE;
-       }       
-       if ( !(C_SEG_IS_ONDISK(c_seg)))
+       }
+       if (!(C_SEG_IS_ONDISK(c_seg))) {
                c_buffer = c_seg->c_store.c_buffer;
                c_buffer = c_seg->c_store.c_buffer;
-       else
+       } else {
                c_swap_handle = c_seg->c_store.c_swap_handle;
                c_swap_handle = c_seg->c_store.c_swap_handle;
+       }
 
        c_seg_switch_state(c_seg, C_IS_FREE, FALSE);
 
 
        c_seg_switch_state(c_seg, C_IS_FREE, FALSE);
 
-       lck_mtx_unlock_always(c_list_lock);
-
        if (c_buffer) {
                pages_populated = (round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE;
                c_seg->c_store.c_buffer = NULL;
        if (c_buffer) {
                pages_populated = (round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE;
                c_seg->c_store.c_buffer = NULL;
-       } else
+       } else {
+#if CONFIG_FREEZE
+               c_seg_update_task_owner(c_seg, NULL);
+#endif /* CONFIG_FREEZE */
+
                c_seg->c_store.c_swap_handle = (uint64_t)-1;
                c_seg->c_store.c_swap_handle = (uint64_t)-1;
+       }
 
        lck_mtx_unlock_always(&c_seg->c_lock);
 
 
        lck_mtx_unlock_always(&c_seg->c_lock);
 
-       if (c_buffer) {
-               if (pages_populated)
-                       kernel_memory_depopulate(kernel_map, (vm_offset_t) c_buffer, pages_populated * PAGE_SIZE, KMA_COMPRESSOR);
+       lck_mtx_unlock_always(c_list_lock);
 
 
+       if (c_buffer) {
+               if (pages_populated) {
+                       kernel_memory_depopulate(compressor_map, (vm_offset_t)c_buffer,
+                           pages_populated * PAGE_SIZE, KMA_COMPRESSOR, VM_KERN_MEMORY_COMPRESSOR);
+               }
        } else if (c_swap_handle) {
        } else if (c_swap_handle) {
-                /*
-                 * Free swap space on disk.
+               /*
+                * Free swap space on disk.
                 */
                vm_swap_free(c_swap_handle);
        }
        lck_mtx_lock_spin_always(&c_seg->c_lock);
                 */
                vm_swap_free(c_swap_handle);
        }
        lck_mtx_lock_spin_always(&c_seg->c_lock);
-
+       /*
+        * c_seg must remain busy until
+        * after the call to vm_swap_free
+        */
        C_SEG_WAKEUP_DONE(c_seg);
        lck_mtx_unlock_always(&c_seg->c_lock);
 
        C_SEG_WAKEUP_DONE(c_seg);
        lck_mtx_unlock_always(&c_seg->c_lock);
 
@@ -1043,8 +1604,8 @@ c_seg_free_locked(c_segment_t c_seg)
        /*
         * because the c_buffer is now associated with the segno,
         * we can't put the segno back on the free list until
        /*
         * because the c_buffer is now associated with the segno,
         * we can't put the segno back on the free list until
-        * after we have depopulated the c_buffer range, or 
-        * we run the risk of depopulating a range that is 
+        * after we have depopulated the c_buffer range, or
+        * we run the risk of depopulating a range that is
         * now being used in one of the compressor heads
         */
        c_segments[segno].c_segno = c_free_segno_head;
         * now being used in one of the compressor heads
         */
        c_segments[segno].c_segno = c_free_segno_head;
@@ -1053,38 +1614,37 @@ c_seg_free_locked(c_segment_t c_seg)
 
        lck_mtx_unlock_always(c_list_lock);
 
 
        lck_mtx_unlock_always(c_list_lock);
 
-#if __i386__ || __x86_64__
        lck_mtx_destroy(&c_seg->c_lock, &vm_compressor_lck_grp);
        lck_mtx_destroy(&c_seg->c_lock, &vm_compressor_lck_grp);
-#else /* __i386__ || __x86_64__ */
-       lck_spin_destroy(&c_seg->c_lock, &vm_compressor_lck_grp);
-#endif /* __i386__ || __x86_64__ */
 
 
-       if (c_seg->c_slot_var_array_len)
-               kfree(c_seg->c_slot_var_array, sizeof(struct c_slot) * c_seg->c_slot_var_array_len);
+       if (c_seg->c_slot_var_array_len) {
+               kheap_free(KHEAP_DATA_BUFFERS, c_seg->c_slot_var_array,
+                   sizeof(struct c_slot) * c_seg->c_slot_var_array_len);
+       }
 
        zfree(compressor_segment_zone, c_seg);
 }
 
 
        zfree(compressor_segment_zone, c_seg);
 }
 
-
+#if DEVELOPMENT || DEBUG
 int c_seg_trim_page_count = 0;
 int c_seg_trim_page_count = 0;
+#endif
 
 void
 c_seg_trim_tail(c_segment_t c_seg)
 {
 
 void
 c_seg_trim_tail(c_segment_t c_seg)
 {
-       c_slot_t        cs;
-       uint32_t        c_size;
-       uint32_t        c_offset;
-       uint32_t        c_rounded_size;
-       uint16_t        current_nextslot;
-       uint32_t        current_populated_offset;
+       c_slot_t        cs;
+       uint32_t        c_size;
+       uint32_t        c_offset;
+       uint32_t        c_rounded_size;
+       uint16_t        current_nextslot;
+       uint32_t        current_populated_offset;
 
 
-       if (c_seg->c_bytes_used == 0)
+       if (c_seg->c_bytes_used == 0) {
                return;
                return;
+       }
        current_nextslot = c_seg->c_nextslot;
        current_populated_offset = c_seg->c_populated_offset;
        current_nextslot = c_seg->c_nextslot;
        current_populated_offset = c_seg->c_populated_offset;
-               
-       while (c_seg->c_nextslot) {
 
 
+       while (c_seg->c_nextslot) {
                cs = C_SEG_SLOT_FROM_INDEX(c_seg, (c_seg->c_nextslot - 1));
 
                c_size = UNPACK_C_SIZE(cs);
                cs = C_SEG_SLOT_FROM_INDEX(c_seg, (c_seg->c_nextslot - 1));
 
                c_size = UNPACK_C_SIZE(cs);
@@ -1095,16 +1655,20 @@ c_seg_trim_tail(c_segment_t c_seg)
                                c_offset = cs->c_offset + C_SEG_BYTES_TO_OFFSET(c_rounded_size);
 
                                c_seg->c_nextoffset = c_offset;
                                c_offset = cs->c_offset + C_SEG_BYTES_TO_OFFSET(c_rounded_size);
 
                                c_seg->c_nextoffset = c_offset;
-                               c_seg->c_populated_offset = (c_offset + (C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1)) & ~(C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1);
+                               c_seg->c_populated_offset = (c_offset + (C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1)) &
+                                   ~(C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1);
 
 
-                               if (c_seg->c_firstemptyslot > c_seg->c_nextslot)
+                               if (c_seg->c_firstemptyslot > c_seg->c_nextslot) {
                                        c_seg->c_firstemptyslot = c_seg->c_nextslot;
                                        c_seg->c_firstemptyslot = c_seg->c_nextslot;
-
+                               }
+#if DEVELOPMENT || DEBUG
                                c_seg_trim_page_count += ((round_page_32(C_SEG_OFFSET_TO_BYTES(current_populated_offset)) -
                                c_seg_trim_page_count += ((round_page_32(C_SEG_OFFSET_TO_BYTES(current_populated_offset)) -
-                                                          round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE);
+                                   round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) /
+                                   PAGE_SIZE);
+#endif
                        }
                        break;
                        }
                        break;
-               }               
+               }
                c_seg->c_nextslot--;
        }
        assert(c_seg->c_nextslot);
                c_seg->c_nextslot--;
        }
        assert(c_seg->c_nextslot);
@@ -1115,15 +1679,14 @@ int
 c_seg_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy)
 {
        c_slot_mapping_t slot_ptr;
 c_seg_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy)
 {
        c_slot_mapping_t slot_ptr;
-       uint32_t        c_offset = 0;
-       uint32_t        old_populated_offset;
-       uint32_t        c_rounded_size;
-       uint32_t        c_size;
-       int             c_indx = 0;
-       int             i;
-       c_slot_t        c_dst;
-       c_slot_t        c_src;
-       boolean_t       need_unlock = TRUE;
+       uint32_t        c_offset = 0;
+       uint32_t        old_populated_offset;
+       uint32_t        c_rounded_size;
+       uint32_t        c_size;
+       uint16_t        c_indx = 0;
+       int             i;
+       c_slot_t        c_dst;
+       c_slot_t        c_src;
 
        assert(c_seg->c_busy);
 
 
        assert(c_seg->c_busy);
 
@@ -1132,46 +1695,48 @@ c_seg_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy)
 #endif
        if (c_seg->c_bytes_used == 0) {
                c_seg_free(c_seg);
 #endif
        if (c_seg->c_bytes_used == 0) {
                c_seg_free(c_seg);
-               return (1);
+               return 1;
        }
        }
-       if (c_seg->c_firstemptyslot >= c_seg->c_nextslot || C_SEG_UNUSED_BYTES(c_seg) < PAGE_SIZE)
+       lck_mtx_unlock_always(&c_seg->c_lock);
+
+       if (c_seg->c_firstemptyslot >= c_seg->c_nextslot || C_SEG_UNUSED_BYTES(c_seg) < PAGE_SIZE) {
                goto done;
                goto done;
-               
+       }
+
+/* TODO: assert first emptyslot's c_size is actually 0 */
+
+#if DEVELOPMENT || DEBUG
+       C_SEG_MAKE_WRITEABLE(c_seg);
+#endif
+
 #if VALIDATE_C_SEGMENTS
        c_seg->c_was_minor_compacted++;
 #endif
        c_indx = c_seg->c_firstemptyslot;
        c_dst = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
 #if VALIDATE_C_SEGMENTS
        c_seg->c_was_minor_compacted++;
 #endif
        c_indx = c_seg->c_firstemptyslot;
        c_dst = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
-       
+
        old_populated_offset = c_seg->c_populated_offset;
        c_offset = c_dst->c_offset;
 
        for (i = c_indx + 1; i < c_seg->c_nextslot && c_offset < c_seg->c_nextoffset; i++) {
        old_populated_offset = c_seg->c_populated_offset;
        c_offset = c_dst->c_offset;
 
        for (i = c_indx + 1; i < c_seg->c_nextslot && c_offset < c_seg->c_nextoffset; i++) {
-
                c_src = C_SEG_SLOT_FROM_INDEX(c_seg, i);
 
                c_size = UNPACK_C_SIZE(c_src);
 
                c_src = C_SEG_SLOT_FROM_INDEX(c_seg, i);
 
                c_size = UNPACK_C_SIZE(c_src);
 
-               if (c_size == 0)
+               if (c_size == 0) {
                        continue;
                        continue;
+               }
 
 
-               memcpy(&c_seg->c_store.c_buffer[c_offset], &c_seg->c_store.c_buffer[c_src->c_offset], c_size);
+               c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
+/* N.B.: This memcpy may be an overlapping copy */
+               memcpy(&c_seg->c_store.c_buffer[c_offset], &c_seg->c_store.c_buffer[c_src->c_offset], c_rounded_size);
 
 
-#if CHECKSUM_THE_DATA
-               c_dst->c_hash_data = c_src->c_hash_data;
-#endif
-#if CHECKSUM_THE_COMPRESSED_DATA
-               c_dst->c_hash_compressed_data = c_src->c_hash_compressed_data;
-#endif
-               c_dst->c_size = c_src->c_size;
-               c_dst->c_packed_ptr = c_src->c_packed_ptr;
+               cslot_copy(c_dst, c_src);
                c_dst->c_offset = c_offset;
 
                c_dst->c_offset = c_offset;
 
-               slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
+               slot_ptr = C_SLOT_UNPACK_PTR(c_dst);
                slot_ptr->s_cindx = c_indx;
 
                slot_ptr->s_cindx = c_indx;
 
-               c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
-
                c_offset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
                PACK_C_SIZE(c_src, 0);
                c_indx++;
                c_offset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
                PACK_C_SIZE(c_src, 0);
                c_indx++;
@@ -1187,73 +1752,77 @@ c_seg_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy)
 #if VALIDATE_C_SEGMENTS
        c_seg_validate(c_seg, TRUE);
 #endif
 #if VALIDATE_C_SEGMENTS
        c_seg_validate(c_seg, TRUE);
 #endif
-
        if (old_populated_offset > c_seg->c_populated_offset) {
        if (old_populated_offset > c_seg->c_populated_offset) {
-               uint32_t        gc_size;
-               int32_t         *gc_ptr;
+               uint32_t        gc_size;
+               int32_t         *gc_ptr;
 
                gc_size = C_SEG_OFFSET_TO_BYTES(old_populated_offset - c_seg->c_populated_offset);
                gc_ptr = &c_seg->c_store.c_buffer[c_seg->c_populated_offset];
 
 
                gc_size = C_SEG_OFFSET_TO_BYTES(old_populated_offset - c_seg->c_populated_offset);
                gc_ptr = &c_seg->c_store.c_buffer[c_seg->c_populated_offset];
 
-               lck_mtx_unlock_always(&c_seg->c_lock);
+               kernel_memory_depopulate(compressor_map, (vm_offset_t)gc_ptr, gc_size,
+                   KMA_COMPRESSOR, VM_KERN_MEMORY_COMPRESSOR);
+       }
 
 
-               kernel_memory_depopulate(kernel_map, (vm_offset_t)gc_ptr, gc_size, KMA_COMPRESSOR);
+#if DEVELOPMENT || DEBUG
+       C_SEG_WRITE_PROTECT(c_seg);
+#endif
 
 
-               if (clear_busy == TRUE)
-                       lck_mtx_lock_spin_always(&c_seg->c_lock);
-               else
-                       need_unlock = FALSE;
-       }
 done:
 done:
-       if (need_unlock == TRUE) {
-               if (clear_busy == TRUE)
-                       C_SEG_WAKEUP_DONE(c_seg);
-
+       if (clear_busy == TRUE) {
+               lck_mtx_lock_spin_always(&c_seg->c_lock);
+               C_SEG_WAKEUP_DONE(c_seg);
                lck_mtx_unlock_always(&c_seg->c_lock);
        }
                lck_mtx_unlock_always(&c_seg->c_lock);
        }
-       return (0);
+       return 0;
 }
 
 
 static void
 c_seg_alloc_nextslot(c_segment_t c_seg)
 {
 }
 
 
 static void
 c_seg_alloc_nextslot(c_segment_t c_seg)
 {
-       struct c_slot   *old_slot_array = NULL;
-       struct c_slot   *new_slot_array = NULL;
-       int             newlen;
-       int             oldlen;
+       struct c_slot   *old_slot_array = NULL;
+       struct c_slot   *new_slot_array = NULL;
+       int             newlen;
+       int             oldlen;
 
 
-       if (c_seg->c_nextslot < c_seg_fixed_array_len)
+       if (c_seg->c_nextslot < c_seg_fixed_array_len) {
                return;
                return;
+       }
 
        if ((c_seg->c_nextslot - c_seg_fixed_array_len) >= c_seg->c_slot_var_array_len) {
 
        if ((c_seg->c_nextslot - c_seg_fixed_array_len) >= c_seg->c_slot_var_array_len) {
-
                oldlen = c_seg->c_slot_var_array_len;
                old_slot_array = c_seg->c_slot_var_array;
 
                oldlen = c_seg->c_slot_var_array_len;
                old_slot_array = c_seg->c_slot_var_array;
 
-               if (oldlen == 0)
+               if (oldlen == 0) {
                        newlen = C_SEG_SLOT_VAR_ARRAY_MIN_LEN;
                        newlen = C_SEG_SLOT_VAR_ARRAY_MIN_LEN;
-               else
+               } else {
                        newlen = oldlen * 2;
                        newlen = oldlen * 2;
+               }
 
 
-               new_slot_array = (struct c_slot *)kalloc(sizeof(struct c_slot) * newlen);
+               new_slot_array = kheap_alloc(KHEAP_DATA_BUFFERS,
+                   sizeof(struct c_slot) * newlen, Z_WAITOK);
 
                lck_mtx_lock_spin_always(&c_seg->c_lock);
 
 
                lck_mtx_lock_spin_always(&c_seg->c_lock);
 
-               if (old_slot_array)
-                       memcpy((char *)new_slot_array, (char *)old_slot_array, sizeof(struct c_slot) * oldlen);
+               if (old_slot_array) {
+                       memcpy(new_slot_array, old_slot_array,
+                           sizeof(struct c_slot) * oldlen);
+               }
 
                c_seg->c_slot_var_array_len = newlen;
                c_seg->c_slot_var_array = new_slot_array;
 
                lck_mtx_unlock_always(&c_seg->c_lock);
 
                c_seg->c_slot_var_array_len = newlen;
                c_seg->c_slot_var_array = new_slot_array;
 
                lck_mtx_unlock_always(&c_seg->c_lock);
-               
-               if (old_slot_array)
-                       kfree(old_slot_array, sizeof(struct c_slot) * oldlen);
+
+               if (old_slot_array) {
+                       kheap_free(KHEAP_DATA_BUFFERS, old_slot_array,
+                           sizeof(struct c_slot) * oldlen);
+               }
        }
 }
 
 
        }
 }
 
 
+#define C_SEG_MAJOR_COMPACT_STATS_MAX   (30)
 
 struct {
        uint64_t asked_permission;
 
 struct {
        uint64_t asked_permission;
@@ -1263,10 +1832,14 @@ struct {
        uint64_t wasted_space_in_swapouts;
        uint64_t count_of_swapouts;
        uint64_t count_of_freed_segs;
        uint64_t wasted_space_in_swapouts;
        uint64_t count_of_swapouts;
        uint64_t count_of_freed_segs;
-} c_seg_major_compact_stats;
+       uint64_t bailed_compactions;
+       uint64_t bytes_freed_rate_us;
+} c_seg_major_compact_stats[C_SEG_MAJOR_COMPACT_STATS_MAX];
+
+int c_seg_major_compact_stats_now = 0;
 
 
 
 
-#define C_MAJOR_COMPACTION_SIZE_APPROPRIATE    ((C_SEG_BUFSIZE * 90) / 100)
+#define C_MAJOR_COMPACTION_SIZE_APPROPRIATE     ((C_SEG_BUFSIZE * 90) / 100)
 
 
 boolean_t
 
 
 boolean_t
@@ -1274,21 +1847,21 @@ c_seg_major_compact_ok(
        c_segment_t c_seg_dst,
        c_segment_t c_seg_src)
 {
        c_segment_t c_seg_dst,
        c_segment_t c_seg_src)
 {
-
-       c_seg_major_compact_stats.asked_permission++;
+       c_seg_major_compact_stats[c_seg_major_compact_stats_now].asked_permission++;
 
        if (c_seg_src->c_bytes_used >= C_MAJOR_COMPACTION_SIZE_APPROPRIATE &&
 
        if (c_seg_src->c_bytes_used >= C_MAJOR_COMPACTION_SIZE_APPROPRIATE &&
-           c_seg_dst->c_bytes_used >= C_MAJOR_COMPACTION_SIZE_APPROPRIATE)
-               return (FALSE);
+           c_seg_dst->c_bytes_used >= C_MAJOR_COMPACTION_SIZE_APPROPRIATE) {
+               return FALSE;
+       }
 
        if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX_INDEX) {
                /*
                 * destination segment is full... can't compact
                 */
 
        if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX_INDEX) {
                /*
                 * destination segment is full... can't compact
                 */
-               return (FALSE);
+               return FALSE;
        }
 
        }
 
-       return (TRUE);
+       return TRUE;
 }
 
 
 }
 
 
@@ -1298,14 +1871,14 @@ c_seg_major_compact(
        c_segment_t c_seg_src)
 {
        c_slot_mapping_t slot_ptr;
        c_segment_t c_seg_src)
 {
        c_slot_mapping_t slot_ptr;
-       uint32_t        c_rounded_size;
-       uint32_t        c_size;
-       uint16_t        dst_slot;
-       int             i;
-       c_slot_t        c_dst;
-       c_slot_t        c_src;
-       boolean_t       keep_compacting = TRUE;
-       
+       uint32_t        c_rounded_size;
+       uint32_t        c_size;
+       uint16_t        dst_slot;
+       int             i;
+       c_slot_t        c_dst;
+       c_slot_t        c_src;
+       boolean_t       keep_compacting = TRUE;
+
        /*
         * segments are not locked but they are both marked c_busy
         * which keeps c_decompress from working on them...
        /*
         * segments are not locked but they are both marked c_busy
         * which keeps c_decompress from working on them...
@@ -1313,17 +1886,19 @@ c_seg_major_compact(
         * from c_seg_src to c_seg_dst and update both c_segment's
         * state w/o holding the master lock
         */
         * from c_seg_src to c_seg_dst and update both c_segment's
         * state w/o holding the master lock
         */
+#if DEVELOPMENT || DEBUG
+       C_SEG_MAKE_WRITEABLE(c_seg_dst);
+#endif
 
 #if VALIDATE_C_SEGMENTS
        c_seg_dst->c_was_major_compacted++;
        c_seg_src->c_was_major_donor++;
 #endif
 
 #if VALIDATE_C_SEGMENTS
        c_seg_dst->c_was_major_compacted++;
        c_seg_src->c_was_major_donor++;
 #endif
-       c_seg_major_compact_stats.compactions++;
+       c_seg_major_compact_stats[c_seg_major_compact_stats_now].compactions++;
 
        dst_slot = c_seg_dst->c_nextslot;
 
        for (i = 0; i < c_seg_src->c_nextslot; i++) {
 
        dst_slot = c_seg_dst->c_nextslot;
 
        for (i = 0; i < c_seg_src->c_nextslot; i++) {
-
                c_src = C_SEG_SLOT_FROM_INDEX(c_seg_src, i);
 
                c_size = UNPACK_C_SIZE(c_src);
                c_src = C_SEG_SLOT_FROM_INDEX(c_seg_src, i);
 
                c_size = UNPACK_C_SIZE(c_src);
@@ -1334,24 +1909,25 @@ c_seg_major_compact(
                }
 
                if (C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset - c_seg_dst->c_nextoffset) < (unsigned) c_size) {
                }
 
                if (C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset - c_seg_dst->c_nextoffset) < (unsigned) c_size) {
-                       int     size_to_populate;
+                       int     size_to_populate;
 
                        /* doesn't fit */
                        size_to_populate = C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset);
 
 
                        /* doesn't fit */
                        size_to_populate = C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset);
 
-                       if (size_to_populate == 0) {
+                       if (size_to_populate == 0) {
                                /* can't fit */
                                keep_compacting = FALSE;
                                break;
                        }
                                /* can't fit */
                                keep_compacting = FALSE;
                                break;
                        }
-                       if (size_to_populate > C_SEG_MAX_POPULATE_SIZE)
+                       if (size_to_populate > C_SEG_MAX_POPULATE_SIZE) {
                                size_to_populate = C_SEG_MAX_POPULATE_SIZE;
                                size_to_populate = C_SEG_MAX_POPULATE_SIZE;
+                       }
 
 
-                       kernel_memory_populate(kernel_map,
-                                              (vm_offset_t) &c_seg_dst->c_store.c_buffer[c_seg_dst->c_populated_offset],
-                                              size_to_populate,
-                                              KMA_COMPRESSOR, 
-                                              VM_KERN_MEMORY_COMPRESSOR);
+                       kernel_memory_populate(compressor_map,
+                           (vm_offset_t) &c_seg_dst->c_store.c_buffer[c_seg_dst->c_populated_offset],
+                           size_to_populate,
+                           KMA_COMPRESSOR,
+                           VM_KERN_MEMORY_COMPRESSOR);
 
                        c_seg_dst->c_populated_offset += C_SEG_BYTES_TO_OFFSET(size_to_populate);
                        assert(C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset) <= C_SEG_BUFSIZE);
 
                        c_seg_dst->c_populated_offset += C_SEG_BYTES_TO_OFFSET(size_to_populate);
                        assert(C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset) <= C_SEG_BUFSIZE);
@@ -1364,21 +1940,16 @@ c_seg_major_compact(
 
                c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
 
 
                c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
 
-               c_seg_major_compact_stats.moved_slots++;
-               c_seg_major_compact_stats.moved_bytes += c_size;
+               c_seg_major_compact_stats[c_seg_major_compact_stats_now].moved_slots++;
+               c_seg_major_compact_stats[c_seg_major_compact_stats_now].moved_bytes += c_size;
 
 
-#if CHECKSUM_THE_DATA
-               c_dst->c_hash_data = c_src->c_hash_data;
-#endif
-#if CHECKSUM_THE_COMPRESSED_DATA
-               c_dst->c_hash_compressed_data = c_src->c_hash_compressed_data;
-#endif
-               c_dst->c_size = c_src->c_size;
-               c_dst->c_packed_ptr = c_src->c_packed_ptr;
+               cslot_copy(c_dst, c_src);
                c_dst->c_offset = c_seg_dst->c_nextoffset;
 
                c_dst->c_offset = c_seg_dst->c_nextoffset;
 
-               if (c_seg_dst->c_firstemptyslot == c_seg_dst->c_nextslot)
+               if (c_seg_dst->c_firstemptyslot == c_seg_dst->c_nextslot) {
                        c_seg_dst->c_firstemptyslot++;
                        c_seg_dst->c_firstemptyslot++;
+               }
+               c_seg_dst->c_slots_used++;
                c_seg_dst->c_nextslot++;
                c_seg_dst->c_bytes_used += c_rounded_size;
                c_seg_dst->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
                c_seg_dst->c_nextslot++;
                c_seg_dst->c_bytes_used += c_rounded_size;
                c_seg_dst->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
@@ -1389,47 +1960,51 @@ c_seg_major_compact(
                c_seg_src->c_bytes_unused += c_rounded_size;
                c_seg_src->c_firstemptyslot = 0;
 
                c_seg_src->c_bytes_unused += c_rounded_size;
                c_seg_src->c_firstemptyslot = 0;
 
+               assert(c_seg_src->c_slots_used);
+               c_seg_src->c_slots_used--;
+
                if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX_INDEX) {
                        /* dest segment is now full */
                        keep_compacting = FALSE;
                        break;
                }
        }
                if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX_INDEX) {
                        /* dest segment is now full */
                        keep_compacting = FALSE;
                        break;
                }
        }
+#if DEVELOPMENT || DEBUG
+       C_SEG_WRITE_PROTECT(c_seg_dst);
+#endif
        if (dst_slot < c_seg_dst->c_nextslot) {
        if (dst_slot < c_seg_dst->c_nextslot) {
-
                PAGE_REPLACEMENT_ALLOWED(TRUE);
                /*
                PAGE_REPLACEMENT_ALLOWED(TRUE);
                /*
-                * we've now locked out c_decompress from 
+                * we've now locked out c_decompress from
                 * converting the slot passed into it into
                 * converting the slot passed into it into
-                * a c_segment_t which allows us to use 
+                * a c_segment_t which allows us to use
                 * the backptr to change which c_segment and
                 * index the slot points to
                 */
                while (dst_slot < c_seg_dst->c_nextslot) {
                 * the backptr to change which c_segment and
                 * index the slot points to
                 */
                while (dst_slot < c_seg_dst->c_nextslot) {
-
                        c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, dst_slot);
                        c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, dst_slot);
-                       
-                       slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
+
+                       slot_ptr = C_SLOT_UNPACK_PTR(c_dst);
                        /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
                        slot_ptr->s_cseg = c_seg_dst->c_mysegno + 1;
                        slot_ptr->s_cindx = dst_slot++;
                }
                PAGE_REPLACEMENT_ALLOWED(FALSE);
        }
                        /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
                        slot_ptr->s_cseg = c_seg_dst->c_mysegno + 1;
                        slot_ptr->s_cindx = dst_slot++;
                }
                PAGE_REPLACEMENT_ALLOWED(FALSE);
        }
-       return (keep_compacting);
+       return keep_compacting;
 }
 
 
 uint64_t
 vm_compressor_compute_elapsed_msecs(clock_sec_t end_sec, clock_nsec_t end_nsec, clock_sec_t start_sec, clock_nsec_t start_nsec)
 {
 }
 
 
 uint64_t
 vm_compressor_compute_elapsed_msecs(clock_sec_t end_sec, clock_nsec_t end_nsec, clock_sec_t start_sec, clock_nsec_t start_nsec)
 {
-        uint64_t end_msecs;
-        uint64_t start_msecs;
-  
+       uint64_t end_msecs;
+       uint64_t start_msecs;
+
        end_msecs = (end_sec * 1000) + end_nsec / 1000000;
        start_msecs = (start_sec * 1000) + start_nsec / 1000000;
 
        end_msecs = (end_sec * 1000) + end_nsec / 1000000;
        start_msecs = (start_sec * 1000) + start_nsec / 1000000;
 
-       return (end_msecs - start_msecs);
+       return end_msecs - start_msecs;
 }
 
 
 }
 
 
@@ -1441,20 +2016,18 @@ uint32_t compressor_thrashing_threshold_per_10msecs = 50;
 uint32_t compressor_thrashing_min_per_10msecs = 20;
 
 /* When true, reset sample data next chance we get. */
 uint32_t compressor_thrashing_min_per_10msecs = 20;
 
 /* When true, reset sample data next chance we get. */
-static boolean_t       compressor_need_sample_reset = FALSE;
-
-extern uint32_t vm_page_filecache_min;
+static boolean_t        compressor_need_sample_reset = FALSE;
 
 
 void
 compute_swapout_target_age(void)
 {
 
 
 void
 compute_swapout_target_age(void)
 {
-        clock_sec_t    cur_ts_sec;
-        clock_nsec_t   cur_ts_nsec;
-       uint32_t        min_operations_needed_in_this_sample;
-       uint64_t        elapsed_msecs_in_eval;
-       uint64_t        elapsed_msecs_in_sample;
-       boolean_t       need_eval_reset = FALSE;
+       clock_sec_t     cur_ts_sec;
+       clock_nsec_t    cur_ts_nsec;
+       uint32_t        min_operations_needed_in_this_sample;
+       uint64_t        elapsed_msecs_in_eval;
+       uint64_t        elapsed_msecs_in_sample;
+       boolean_t       need_eval_reset = FALSE;
 
        clock_get_system_nanotime(&cur_ts_sec, &cur_ts_nsec);
 
 
        clock_get_system_nanotime(&cur_ts_sec, &cur_ts_nsec);
 
@@ -1467,9 +2040,10 @@ compute_swapout_target_age(void)
                goto done;
        }
        elapsed_msecs_in_eval = vm_compressor_compute_elapsed_msecs(cur_ts_sec, cur_ts_nsec, start_of_eval_period_sec, start_of_eval_period_nsec);
                goto done;
        }
        elapsed_msecs_in_eval = vm_compressor_compute_elapsed_msecs(cur_ts_sec, cur_ts_nsec, start_of_eval_period_sec, start_of_eval_period_nsec);
-       
-       if (elapsed_msecs_in_eval < compressor_eval_period_in_msecs)
+
+       if (elapsed_msecs_in_eval < compressor_eval_period_in_msecs) {
                goto done;
                goto done;
+       }
        need_eval_reset = TRUE;
 
        KERNEL_DEBUG(0xe0400020 | DBG_FUNC_START, elapsed_msecs_in_eval, sample_period_compression_count, sample_period_decompression_count, 0, 0);
        need_eval_reset = TRUE;
 
        KERNEL_DEBUG(0xe0400020 | DBG_FUNC_START, elapsed_msecs_in_eval, sample_period_compression_count, sample_period_decompression_count, 0, 0);
@@ -1478,9 +2052,8 @@ compute_swapout_target_age(void)
 
        if ((sample_period_compression_count - last_eval_compression_count) < min_operations_needed_in_this_sample ||
            (sample_period_decompression_count - last_eval_decompression_count) < min_operations_needed_in_this_sample) {
 
        if ((sample_period_compression_count - last_eval_compression_count) < min_operations_needed_in_this_sample ||
            (sample_period_decompression_count - last_eval_decompression_count) < min_operations_needed_in_this_sample) {
-               
                KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, sample_period_compression_count - last_eval_compression_count,
                KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, sample_period_compression_count - last_eval_compression_count,
-                            sample_period_decompression_count - last_eval_decompression_count, 0, 1, 0);
+                   sample_period_decompression_count - last_eval_decompression_count, 0, 1, 0);
 
                swapout_target_age = 0;
 
 
                swapout_target_age = 0;
 
@@ -1492,46 +2065,43 @@ compute_swapout_target_age(void)
        last_eval_decompression_count = sample_period_decompression_count;
 
        if (elapsed_msecs_in_sample < compressor_sample_min_in_msecs) {
        last_eval_decompression_count = sample_period_decompression_count;
 
        if (elapsed_msecs_in_sample < compressor_sample_min_in_msecs) {
-
                KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, swapout_target_age, 0, 0, 5, 0);
                goto done;
        }
        if (sample_period_decompression_count > ((compressor_thrashing_threshold_per_10msecs * elapsed_msecs_in_sample) / 10)) {
                KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, swapout_target_age, 0, 0, 5, 0);
                goto done;
        }
        if (sample_period_decompression_count > ((compressor_thrashing_threshold_per_10msecs * elapsed_msecs_in_sample) / 10)) {
-
-               uint64_t        running_total;
-               uint64_t        working_target;
-               uint64_t        aging_target;
-               uint32_t        oldest_age_of_csegs_sampled = 0;
-               uint64_t        working_set_approximation = 0;
+               uint64_t        running_total;
+               uint64_t        working_target;
+               uint64_t        aging_target;
+               uint32_t        oldest_age_of_csegs_sampled = 0;
+               uint64_t        working_set_approximation = 0;
 
                swapout_target_age = 0;
 
 
                swapout_target_age = 0;
 
-               working_target = (sample_period_decompression_count / 100) * 95;                /* 95 percent */
-               aging_target = (sample_period_decompression_count / 100) * 1;                   /* 1 percent */
+               working_target = (sample_period_decompression_count / 100) * 95;                /* 95 percent */
+               aging_target = (sample_period_decompression_count / 100) * 1;                   /* 1 percent */
                running_total = 0;
 
                for (oldest_age_of_csegs_sampled = 0; oldest_age_of_csegs_sampled < DECOMPRESSION_SAMPLE_MAX_AGE; oldest_age_of_csegs_sampled++) {
                running_total = 0;
 
                for (oldest_age_of_csegs_sampled = 0; oldest_age_of_csegs_sampled < DECOMPRESSION_SAMPLE_MAX_AGE; oldest_age_of_csegs_sampled++) {
-
                        running_total += age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
 
                        working_set_approximation += oldest_age_of_csegs_sampled * age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
 
                        running_total += age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
 
                        working_set_approximation += oldest_age_of_csegs_sampled * age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
 
-                       if (running_total >= working_target)
+                       if (running_total >= working_target) {
                                break;
                                break;
+                       }
                }
                if (oldest_age_of_csegs_sampled < DECOMPRESSION_SAMPLE_MAX_AGE) {
                }
                if (oldest_age_of_csegs_sampled < DECOMPRESSION_SAMPLE_MAX_AGE) {
-
                        working_set_approximation = (working_set_approximation * 1000) / elapsed_msecs_in_sample;
 
                        if (working_set_approximation < VM_PAGE_COMPRESSOR_COUNT) {
                        working_set_approximation = (working_set_approximation * 1000) / elapsed_msecs_in_sample;
 
                        if (working_set_approximation < VM_PAGE_COMPRESSOR_COUNT) {
-
                                running_total = overage_decompressions_during_sample_period;
 
                                for (oldest_age_of_csegs_sampled = DECOMPRESSION_SAMPLE_MAX_AGE - 1; oldest_age_of_csegs_sampled; oldest_age_of_csegs_sampled--) {
                                        running_total += age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
 
                                running_total = overage_decompressions_during_sample_period;
 
                                for (oldest_age_of_csegs_sampled = DECOMPRESSION_SAMPLE_MAX_AGE - 1; oldest_age_of_csegs_sampled; oldest_age_of_csegs_sampled--) {
                                        running_total += age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
 
-                                       if (running_total >= aging_target)
+                                       if (running_total >= aging_target) {
                                                break;
                                                break;
+                                       }
                                }
                                swapout_target_age = (uint32_t)cur_ts_sec - oldest_age_of_csegs_sampled;
 
                                }
                                swapout_target_age = (uint32_t)cur_ts_sec - oldest_age_of_csegs_sampled;
 
@@ -1539,13 +2109,15 @@ compute_swapout_target_age(void)
                        } else {
                                KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, working_set_approximation, VM_PAGE_COMPRESSOR_COUNT, 0, 3, 0);
                        }
                        } else {
                                KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, working_set_approximation, VM_PAGE_COMPRESSOR_COUNT, 0, 3, 0);
                        }
-               } else
+               } else {
                        KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, working_target, running_total, 0, 4, 0);
                        KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, working_target, running_total, 0, 4, 0);
+               }
 
                compressor_need_sample_reset = TRUE;
                need_eval_reset = TRUE;
 
                compressor_need_sample_reset = TRUE;
                need_eval_reset = TRUE;
-       } else
+       } else {
                KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, sample_period_decompression_count, (compressor_thrashing_threshold_per_10msecs * elapsed_msecs_in_sample) / 10, 0, 6, 0);
                KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, sample_period_decompression_count, (compressor_thrashing_threshold_per_10msecs * elapsed_msecs_in_sample) / 10, 0, 6, 0);
+       }
 done:
        if (compressor_need_sample_reset == TRUE) {
                bzero(age_of_decompressions_during_sample_period, sizeof(age_of_decompressions_during_sample_period));
 done:
        if (compressor_need_sample_reset == TRUE) {
                bzero(age_of_decompressions_during_sample_period, sizeof(age_of_decompressions_during_sample_period));
@@ -1566,87 +2138,110 @@ done:
 }
 
 
 }
 
 
-int            compaction_swapper_inited = 0;
-int            compaction_swapper_init_now = 0;
-int            compaction_swapper_running = 0;
-int            compaction_swapper_abort = 0;
+int             compaction_swapper_init_now = 0;
+int             compaction_swapper_running = 0;
+int             compaction_swapper_awakened = 0;
+int             compaction_swapper_abort = 0;
 
 
 #if CONFIG_JETSAM
 
 
 #if CONFIG_JETSAM
-boolean_t      memorystatus_kill_on_VM_thrashing(boolean_t);
-boolean_t      memorystatus_kill_on_FC_thrashing(boolean_t);
-int            compressor_thrashing_induced_jetsam = 0;
-int            filecache_thrashing_induced_jetsam = 0;
-static boolean_t       vm_compressor_thrashing_detected = FALSE;
+boolean_t       memorystatus_kill_on_VM_compressor_thrashing(boolean_t);
+boolean_t       memorystatus_kill_on_VM_compressor_space_shortage(boolean_t);
+boolean_t       memorystatus_kill_on_FC_thrashing(boolean_t);
+int             compressor_thrashing_induced_jetsam = 0;
+int             filecache_thrashing_induced_jetsam = 0;
+static boolean_t        vm_compressor_thrashing_detected = FALSE;
 #endif /* CONFIG_JETSAM */
 
 static boolean_t
 compressor_needs_to_swap(void)
 {
 #endif /* CONFIG_JETSAM */
 
 static boolean_t
 compressor_needs_to_swap(void)
 {
-       boolean_t       should_swap = FALSE;
+       boolean_t       should_swap = FALSE;
 
        if (vm_swapout_ripe_segments == TRUE && c_overage_swapped_count < c_overage_swapped_limit) {
 
        if (vm_swapout_ripe_segments == TRUE && c_overage_swapped_count < c_overage_swapped_limit) {
-               c_segment_t     c_seg;
-               clock_sec_t     now;
-               clock_sec_t     age;
-               clock_nsec_t    nsec;
-               
-               clock_get_system_nanotime(&now,  &nsec);
+               c_segment_t     c_seg;
+               clock_sec_t     now;
+               clock_sec_t     age;
+               clock_nsec_t    nsec;
+
+               clock_get_system_nanotime(&now, &nsec);
                age = 0;
 
                lck_mtx_lock_spin_always(c_list_lock);
 
                age = 0;
 
                lck_mtx_lock_spin_always(c_list_lock);
 
-               if ( !queue_empty(&c_age_list_head)) {
+               if (!queue_empty(&c_age_list_head)) {
                        c_seg = (c_segment_t) queue_first(&c_age_list_head);
 
                        age = now - c_seg->c_creation_ts;
                }
                lck_mtx_unlock_always(c_list_lock);
 
                        c_seg = (c_segment_t) queue_first(&c_age_list_head);
 
                        age = now - c_seg->c_creation_ts;
                }
                lck_mtx_unlock_always(c_list_lock);
 
-               if (age >= vm_ripe_target_age)
-                       return (TRUE);
+               if (age >= vm_ripe_target_age) {
+                       should_swap = TRUE;
+                       goto check_if_low_space;
+               }
        }
        }
-       if ((vm_compressor_mode == VM_PAGER_COMPRESSOR_WITH_SWAP) && vm_swap_up == TRUE) {
+       if (VM_CONFIG_SWAP_IS_ACTIVE) {
                if (COMPRESSOR_NEEDS_TO_SWAP()) {
                if (COMPRESSOR_NEEDS_TO_SWAP()) {
-                       return (TRUE);
+                       should_swap = TRUE;
+                       goto check_if_low_space;
                }
                if (VM_PAGE_Q_THROTTLED(&vm_pageout_queue_external) && vm_page_anonymous_count < (vm_page_inactive_count / 20)) {
                }
                if (VM_PAGE_Q_THROTTLED(&vm_pageout_queue_external) && vm_page_anonymous_count < (vm_page_inactive_count / 20)) {
-                       return (TRUE);
+                       should_swap = TRUE;
+                       goto check_if_low_space;
+               }
+               if (vm_page_free_count < (vm_page_free_reserved - (COMPRESSOR_FREE_RESERVED_LIMIT * 2))) {
+                       should_swap = TRUE;
+                       goto check_if_low_space;
                }
                }
-               if (vm_page_free_count < (vm_page_free_reserved - (COMPRESSOR_FREE_RESERVED_LIMIT * 2)))
-                       return (TRUE);
        }
        }
+
+#if (XNU_TARGET_OS_OSX && __arm64__)
+       /*
+        * Thrashing detection disabled.
+        */
+#else /* (XNU_TARGET_OS_OSX && __arm64__) */
+
        compute_swapout_target_age();
        compute_swapout_target_age();
-                       
+
        if (swapout_target_age) {
        if (swapout_target_age) {
-               c_segment_t     c_seg;
+               c_segment_t     c_seg;
 
                lck_mtx_lock_spin_always(c_list_lock);
 
                if (!queue_empty(&c_age_list_head)) {
 
                lck_mtx_lock_spin_always(c_list_lock);
 
                if (!queue_empty(&c_age_list_head)) {
-
                        c_seg = (c_segment_t) queue_first(&c_age_list_head);
 
                        c_seg = (c_segment_t) queue_first(&c_age_list_head);
 
-                       if (c_seg->c_creation_ts > swapout_target_age)
+                       if (c_seg->c_creation_ts > swapout_target_age) {
                                swapout_target_age = 0;
                                swapout_target_age = 0;
+                       }
                }
                lck_mtx_unlock_always(c_list_lock);
        }
 #if CONFIG_PHANTOM_CACHE
                }
                lck_mtx_unlock_always(c_list_lock);
        }
 #if CONFIG_PHANTOM_CACHE
-       if (vm_phantom_cache_check_pressure())
+       if (vm_phantom_cache_check_pressure()) {
                should_swap = TRUE;
                should_swap = TRUE;
+       }
 #endif
 #endif
-       if (swapout_target_age)
+       if (swapout_target_age) {
                should_swap = TRUE;
                should_swap = TRUE;
+       }
+#endif /* (XNU_TARGET_OS_OSX && __arm64__) */
 
 
-#if CONFIG_JETSAM
-       if (should_swap || c_segment_pages_compressed > c_segment_pages_compressed_nearing_limit) {
+check_if_low_space:
 
 
+#if CONFIG_JETSAM
+       if (should_swap || vm_compressor_low_on_space() == TRUE) {
                if (vm_compressor_thrashing_detected == FALSE) {
                        vm_compressor_thrashing_detected = TRUE;
                if (vm_compressor_thrashing_detected == FALSE) {
                        vm_compressor_thrashing_detected = TRUE;
-                               
-                       if (swapout_target_age || c_segment_pages_compressed > c_segment_pages_compressed_nearing_limit) {
-                               memorystatus_kill_on_VM_thrashing(TRUE /* async */);
+
+                       if (swapout_target_age) {
+                               /* The compressor is thrashing. */
+                               memorystatus_kill_on_VM_compressor_thrashing(TRUE /* async */);
+                               compressor_thrashing_induced_jetsam++;
+                       } else if (vm_compressor_low_on_space() == TRUE) {
+                               /* The compressor is running low on space. */
+                               memorystatus_kill_on_VM_compressor_space_shortage(TRUE /* async */);
                                compressor_thrashing_induced_jetsam++;
                        } else {
                                memorystatus_kill_on_FC_thrashing(TRUE /* async */);
                                compressor_thrashing_induced_jetsam++;
                        } else {
                                memorystatus_kill_on_FC_thrashing(TRUE /* async */);
@@ -1664,11 +2259,15 @@ compressor_needs_to_swap(void)
                should_swap = FALSE;
        }
 
                should_swap = FALSE;
        }
 
+#else /* CONFIG_JETSAM */
+       if (should_swap && vm_swap_low_on_space()) {
+               vm_compressor_take_paging_space_action();
+       }
 #endif /* CONFIG_JETSAM */
 
        if (should_swap == FALSE) {
                /*
 #endif /* CONFIG_JETSAM */
 
        if (should_swap == FALSE) {
                /*
-                * COMPRESSOR_NEEDS_TO_MAJOR_COMPACT returns true only if we're
+                * vm_compressor_needs_to_major_compact returns true only if we're
                 * about to run out of available compressor segments... in this
                 * case, we absolutely need to run a major compaction even if
                 * we've just kicked off a jetsam or we don't otherwise need to
                 * about to run out of available compressor segments... in this
                 * case, we absolutely need to run a major compaction even if
                 * we've just kicked off a jetsam or we don't otherwise need to
@@ -1676,7 +2275,7 @@ compressor_needs_to_swap(void)
                 * pages back to the uncompressed cache, but does not guarantee
                 * that we will free up even a single compression segment
                 */
                 * pages back to the uncompressed cache, but does not guarantee
                 * that we will free up even a single compression segment
                 */
-               should_swap = COMPRESSOR_NEEDS_TO_MAJOR_COMPACT();
+               should_swap = vm_compressor_needs_to_major_compact();
        }
 
        /*
        }
 
        /*
@@ -1686,7 +2285,7 @@ compressor_needs_to_swap(void)
         * segments that have been major compacted
         * will be moved to the majorcompact queue
         */
         * segments that have been major compacted
         * will be moved to the majorcompact queue
         */
-       return (should_swap);
+       return should_swap;
 }
 
 #if CONFIG_JETSAM
 }
 
 #if CONFIG_JETSAM
@@ -1717,26 +2316,75 @@ vm_thrashing_jetsam_done(void)
 #endif /* CONFIG_JETSAM */
 
 uint32_t vm_wake_compactor_swapper_calls = 0;
 #endif /* CONFIG_JETSAM */
 
 uint32_t vm_wake_compactor_swapper_calls = 0;
+uint32_t vm_run_compactor_already_running = 0;
+uint32_t vm_run_compactor_empty_minor_q = 0;
+uint32_t vm_run_compactor_did_compact = 0;
+uint32_t vm_run_compactor_waited = 0;
 
 void
 
 void
-vm_wake_compactor_swapper(void)
+vm_run_compactor(void)
 {
 {
-       if (compaction_swapper_running || c_segment_count == 0)
+       if (c_segment_count == 0) {
+               return;
+       }
+
+       lck_mtx_lock_spin_always(c_list_lock);
+
+       if (c_minor_count == 0) {
+               vm_run_compactor_empty_minor_q++;
+
+               lck_mtx_unlock_always(c_list_lock);
+               return;
+       }
+       if (compaction_swapper_running) {
+               if (vm_pageout_state.vm_restricted_to_single_processor == FALSE) {
+                       vm_run_compactor_already_running++;
+
+                       lck_mtx_unlock_always(c_list_lock);
+                       return;
+               }
+               vm_run_compactor_waited++;
+
+               assert_wait((event_t)&compaction_swapper_running, THREAD_UNINT);
+
+               lck_mtx_unlock_always(c_list_lock);
+
+               thread_block(THREAD_CONTINUE_NULL);
+
                return;
                return;
+       }
+       vm_run_compactor_did_compact++;
+
+       fastwake_warmup = FALSE;
+       compaction_swapper_running = 1;
+
+       vm_compressor_do_delayed_compactions(FALSE);
+
+       compaction_swapper_running = 0;
+
+       lck_mtx_unlock_always(c_list_lock);
+
+       thread_wakeup((event_t)&compaction_swapper_running);
+}
 
 
-       if (c_minor_count || COMPRESSOR_NEEDS_TO_MAJOR_COMPACT()) {
 
 
+void
+vm_wake_compactor_swapper(void)
+{
+       if (compaction_swapper_running || compaction_swapper_awakened || c_segment_count == 0) {
+               return;
+       }
+
+       if (c_minor_count || vm_compressor_needs_to_major_compact()) {
                lck_mtx_lock_spin_always(c_list_lock);
 
                fastwake_warmup = FALSE;
 
                lck_mtx_lock_spin_always(c_list_lock);
 
                fastwake_warmup = FALSE;
 
-               if (compaction_swapper_running == 0) {
-
+               if (compaction_swapper_running == 0 && compaction_swapper_awakened == 0) {
                        vm_wake_compactor_swapper_calls++;
 
                        vm_wake_compactor_swapper_calls++;
 
+                       compaction_swapper_awakened = 1;
                        thread_wakeup((event_t)&c_compressor_swap_trigger);
                        thread_wakeup((event_t)&c_compressor_swap_trigger);
-                       
-                       compaction_swapper_running = 1;
                }
                lck_mtx_unlock_always(c_list_lock);
        }
                }
                lck_mtx_unlock_always(c_list_lock);
        }
@@ -1746,10 +2394,11 @@ vm_wake_compactor_swapper(void)
 void
 vm_consider_swapping()
 {
 void
 vm_consider_swapping()
 {
-       c_segment_t     c_seg, c_seg_next;
-       clock_sec_t     now;
-       clock_nsec_t    nsec;
+       c_segment_t     c_seg, c_seg_next;
+       clock_sec_t     now;
+       clock_nsec_t    nsec;
 
 
+       assert(VM_CONFIG_SWAP_IS_PRESENT);
 
        lck_mtx_lock_spin_always(c_list_lock);
 
 
        lck_mtx_lock_spin_always(c_list_lock);
 
@@ -1759,7 +2408,7 @@ vm_consider_swapping()
                assert_wait((event_t)&compaction_swapper_running, THREAD_UNINT);
 
                lck_mtx_unlock_always(c_list_lock);
                assert_wait((event_t)&compaction_swapper_running, THREAD_UNINT);
 
                lck_mtx_unlock_always(c_list_lock);
-               
+
                thread_block(THREAD_CONTINUE_NULL);
 
                lck_mtx_lock_spin_always(c_list_lock);
                thread_block(THREAD_CONTINUE_NULL);
 
                lck_mtx_lock_spin_always(c_list_lock);
@@ -1770,22 +2419,20 @@ vm_consider_swapping()
        vm_swapout_ripe_segments = TRUE;
 
        if (!queue_empty(&c_major_list_head)) {
        vm_swapout_ripe_segments = TRUE;
 
        if (!queue_empty(&c_major_list_head)) {
-               
                clock_get_system_nanotime(&now, &nsec);
                clock_get_system_nanotime(&now, &nsec);
-                       
+
                c_seg = (c_segment_t)queue_first(&c_major_list_head);
 
                while (!queue_end(&c_major_list_head, (queue_entry_t)c_seg)) {
                c_seg = (c_segment_t)queue_first(&c_major_list_head);
 
                while (!queue_end(&c_major_list_head, (queue_entry_t)c_seg)) {
-
-                       if (c_overage_swapped_count >= c_overage_swapped_limit)
+                       if (c_overage_swapped_count >= c_overage_swapped_limit) {
                                break;
                                break;
+                       }
 
                        c_seg_next = (c_segment_t) queue_next(&c_seg->c_age_list);
 
                        if ((now - c_seg->c_creation_ts) >= vm_ripe_target_age) {
 
                        c_seg_next = (c_segment_t) queue_next(&c_seg->c_age_list);
 
                        if ((now - c_seg->c_creation_ts) >= vm_ripe_target_age) {
-                       
                                lck_mtx_lock_spin_always(&c_seg->c_lock);
                                lck_mtx_lock_spin_always(&c_seg->c_lock);
-                               
+
                                c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
 
                                lck_mtx_unlock_always(&c_seg->c_lock);
                                c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
 
                                lck_mtx_unlock_always(&c_seg->c_lock);
@@ -1798,21 +2445,25 @@ vm_consider_swapping()
        compaction_swapper_running = 0;
 
        vm_swapout_ripe_segments = FALSE;
        compaction_swapper_running = 0;
 
        vm_swapout_ripe_segments = FALSE;
-       
+
        lck_mtx_unlock_always(c_list_lock);
        lck_mtx_unlock_always(c_list_lock);
+
+       thread_wakeup((event_t)&compaction_swapper_running);
 }
 
 
 void
 vm_consider_waking_compactor_swapper(void)
 {
 }
 
 
 void
 vm_consider_waking_compactor_swapper(void)
 {
-       boolean_t       need_wakeup = FALSE;
+       boolean_t       need_wakeup = FALSE;
 
 
-       if (compaction_swapper_running)
+       if (c_segment_count == 0) {
                return;
                return;
+       }
 
 
-       if (c_segment_count == 0)
+       if (compaction_swapper_running || compaction_swapper_awakened) {
                return;
                return;
+       }
 
        if (!compaction_swapper_inited && !compaction_swapper_init_now) {
                compaction_swapper_init_now = 1;
 
        if (!compaction_swapper_inited && !compaction_swapper_init_now) {
                compaction_swapper_init_now = 1;
@@ -1820,60 +2471,57 @@ vm_consider_waking_compactor_swapper(void)
        }
 
        if (c_minor_count && (COMPRESSOR_NEEDS_TO_MINOR_COMPACT())) {
        }
 
        if (c_minor_count && (COMPRESSOR_NEEDS_TO_MINOR_COMPACT())) {
-
                need_wakeup = TRUE;
                need_wakeup = TRUE;
-
        } else if (compressor_needs_to_swap()) {
        } else if (compressor_needs_to_swap()) {
-
                need_wakeup = TRUE;
                need_wakeup = TRUE;
-
        } else if (c_minor_count) {
        } else if (c_minor_count) {
-               uint64_t        total_bytes;
+               uint64_t        total_bytes;
 
                total_bytes = compressor_object->resident_page_count * PAGE_SIZE_64;
 
 
                total_bytes = compressor_object->resident_page_count * PAGE_SIZE_64;
 
-               if ((total_bytes - compressor_bytes_used) > total_bytes / 10)
+               if ((total_bytes - compressor_bytes_used) > total_bytes / 10) {
                        need_wakeup = TRUE;
                        need_wakeup = TRUE;
+               }
        }
        if (need_wakeup == TRUE) {
        }
        if (need_wakeup == TRUE) {
-                       
                lck_mtx_lock_spin_always(c_list_lock);
 
                fastwake_warmup = FALSE;
 
                lck_mtx_lock_spin_always(c_list_lock);
 
                fastwake_warmup = FALSE;
 
-               if (compaction_swapper_running == 0) {
+               if (compaction_swapper_running == 0 && compaction_swapper_awakened == 0) {
                        memoryshot(VM_WAKEUP_COMPACTOR_SWAPPER, DBG_FUNC_NONE);
 
                        memoryshot(VM_WAKEUP_COMPACTOR_SWAPPER, DBG_FUNC_NONE);
 
+                       compaction_swapper_awakened = 1;
                        thread_wakeup((event_t)&c_compressor_swap_trigger);
                        thread_wakeup((event_t)&c_compressor_swap_trigger);
-
-                       compaction_swapper_running = 1;
                }
                lck_mtx_unlock_always(c_list_lock);
        }
 }
 
 
                }
                lck_mtx_unlock_always(c_list_lock);
        }
 }
 
 
-#define        C_SWAPOUT_LIMIT                 4
-#define        DELAYED_COMPACTIONS_PER_PASS    30
+#define C_SWAPOUT_LIMIT                 4
+#define DELAYED_COMPACTIONS_PER_PASS    30
 
 void
 vm_compressor_do_delayed_compactions(boolean_t flush_all)
 {
 
 void
 vm_compressor_do_delayed_compactions(boolean_t flush_all)
 {
-       c_segment_t     c_seg;
-       int             number_compacted = 0;
-       boolean_t       needs_to_swap = FALSE;
+       c_segment_t     c_seg;
+       int             number_compacted = 0;
+       boolean_t       needs_to_swap = FALSE;
+
 
 
+       VM_DEBUG_CONSTANT_EVENT(vm_compressor_do_delayed_compactions, VM_COMPRESSOR_DO_DELAYED_COMPACTIONS, DBG_FUNC_START, c_minor_count, flush_all, 0, 0);
 
 
-       lck_mtx_assert(c_list_lock, LCK_MTX_ASSERT_OWNED);
+#if XNU_TARGET_OS_OSX
+       LCK_MTX_ASSERT(c_list_lock, LCK_MTX_ASSERT_OWNED);
+#endif /* XNU_TARGET_OS_OSX */
 
        while (!queue_empty(&c_minor_list_head) && needs_to_swap == FALSE) {
 
        while (!queue_empty(&c_minor_list_head) && needs_to_swap == FALSE) {
-               
                c_seg = (c_segment_t)queue_first(&c_minor_list_head);
                c_seg = (c_segment_t)queue_first(&c_minor_list_head);
-               
+
                lck_mtx_lock_spin_always(&c_seg->c_lock);
 
                if (c_seg->c_busy) {
                lck_mtx_lock_spin_always(&c_seg->c_lock);
 
                if (c_seg->c_busy) {
-
                        lck_mtx_unlock_always(c_list_lock);
                        c_seg_wait_on_busy(c_seg);
                        lck_mtx_lock_spin_always(c_list_lock);
                        lck_mtx_unlock_always(c_list_lock);
                        c_seg_wait_on_busy(c_seg);
                        lck_mtx_lock_spin_always(c_list_lock);
@@ -1884,36 +2532,38 @@ vm_compressor_do_delayed_compactions(boolean_t flush_all)
 
                c_seg_do_minor_compaction_and_unlock(c_seg, TRUE, FALSE, TRUE);
 
 
                c_seg_do_minor_compaction_and_unlock(c_seg, TRUE, FALSE, TRUE);
 
-               if (vm_swap_up == TRUE && (number_compacted++ > DELAYED_COMPACTIONS_PER_PASS)) {
-                       
-                       if ((flush_all == TRUE || compressor_needs_to_swap() == TRUE) && c_swapout_count < C_SWAPOUT_LIMIT)
+               if (VM_CONFIG_SWAP_IS_ACTIVE && (number_compacted++ > DELAYED_COMPACTIONS_PER_PASS)) {
+                       if ((flush_all == TRUE || compressor_needs_to_swap() == TRUE) && c_swapout_count < C_SWAPOUT_LIMIT) {
                                needs_to_swap = TRUE;
                                needs_to_swap = TRUE;
+                       }
 
                        number_compacted = 0;
                }
                lck_mtx_lock_spin_always(c_list_lock);
        }
 
                        number_compacted = 0;
                }
                lck_mtx_lock_spin_always(c_list_lock);
        }
+
+       VM_DEBUG_CONSTANT_EVENT(vm_compressor_do_delayed_compactions, VM_COMPRESSOR_DO_DELAYED_COMPACTIONS, DBG_FUNC_END, c_minor_count, number_compacted, needs_to_swap, 0);
 }
 
 
 }
 
 
-#define C_SEGMENT_SWAPPEDIN_AGE_LIMIT  10
+#define C_SEGMENT_SWAPPEDIN_AGE_LIMIT   10
 
 static void
 vm_compressor_age_swapped_in_segments(boolean_t flush_all)
 {
 
 static void
 vm_compressor_age_swapped_in_segments(boolean_t flush_all)
 {
-       c_segment_t     c_seg;
-       clock_sec_t     now;
-       clock_nsec_t    nsec;
+       c_segment_t     c_seg;
+       clock_sec_t     now;
+       clock_nsec_t    nsec;
 
 
-       clock_get_system_nanotime(&now,  &nsec);
-                       
-       while (!queue_empty(&c_swappedin_list_head)) {
+       clock_get_system_nanotime(&now, &nsec);
 
 
+       while (!queue_empty(&c_swappedin_list_head)) {
                c_seg = (c_segment_t)queue_first(&c_swappedin_list_head);
 
                c_seg = (c_segment_t)queue_first(&c_swappedin_list_head);
 
-               if (flush_all == FALSE && (now - c_seg->c_swappedin_ts) < C_SEGMENT_SWAPPEDIN_AGE_LIMIT)
+               if (flush_all == FALSE && (now - c_seg->c_swappedin_ts) < C_SEGMENT_SWAPPEDIN_AGE_LIMIT) {
                        break;
                        break;
-                       
+               }
+
                lck_mtx_lock_spin_always(&c_seg->c_lock);
 
                c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
                lck_mtx_lock_spin_always(&c_seg->c_lock);
 
                c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
@@ -1923,15 +2573,23 @@ vm_compressor_age_swapped_in_segments(boolean_t flush_all)
 }
 
 
 }
 
 
+extern  int     vm_num_swap_files;
+extern  int     vm_num_pinned_swap_files;
+extern  int     vm_swappin_enabled;
+
+extern  unsigned int    vm_swapfile_total_segs_used;
+extern  unsigned int    vm_swapfile_total_segs_alloced;
+
+
 void
 vm_compressor_flush(void)
 {
 void
 vm_compressor_flush(void)
 {
-       uint64_t        vm_swap_put_failures_at_start;
-       wait_result_t   wait_result = 0;
-       AbsoluteTime    startTime, endTime;
-       clock_sec_t     now_sec;
-       clock_nsec_t    now_nsec;
-       uint64_t        nsec;
+       uint64_t        vm_swap_put_failures_at_start;
+       wait_result_t   wait_result = 0;
+       AbsoluteTime    startTime, endTime;
+       clock_sec_t     now_sec;
+       clock_nsec_t    now_nsec;
+       uint64_t        nsec;
 
        HIBLOG("vm_compressor_flush - starting\n");
 
 
        HIBLOG("vm_compressor_flush - starting\n");
 
@@ -1946,7 +2604,7 @@ vm_compressor_flush(void)
                assert_wait((event_t)&compaction_swapper_running, THREAD_UNINT);
 
                lck_mtx_unlock_always(c_list_lock);
                assert_wait((event_t)&compaction_swapper_running, THREAD_UNINT);
 
                lck_mtx_unlock_always(c_list_lock);
-               
+
                thread_block(THREAD_CONTINUE_NULL);
 
                lck_mtx_lock_spin_always(c_list_lock);
                thread_block(THREAD_CONTINUE_NULL);
 
                lck_mtx_lock_spin_always(c_list_lock);
@@ -1966,38 +2624,40 @@ vm_compressor_flush(void)
        vm_compressor_compact_and_swap(TRUE);
 
        while (!queue_empty(&c_swapout_list_head)) {
        vm_compressor_compact_and_swap(TRUE);
 
        while (!queue_empty(&c_swapout_list_head)) {
-
-               assert_wait_timeout((event_t) &compaction_swapper_running, THREAD_INTERRUPTIBLE, 5000, 1000*NSEC_PER_USEC);
+               assert_wait_timeout((event_t) &compaction_swapper_running, THREAD_INTERRUPTIBLE, 5000, 1000 * NSEC_PER_USEC);
 
                lck_mtx_unlock_always(c_list_lock);
 
                lck_mtx_unlock_always(c_list_lock);
-               
+
                wait_result = thread_block(THREAD_CONTINUE_NULL);
 
                lck_mtx_lock_spin_always(c_list_lock);
 
                wait_result = thread_block(THREAD_CONTINUE_NULL);
 
                lck_mtx_lock_spin_always(c_list_lock);
 
-               if (wait_result == THREAD_TIMED_OUT)
+               if (wait_result == THREAD_TIMED_OUT) {
                        break;
                        break;
+               }
        }
        hibernate_flushing = FALSE;
        compaction_swapper_running = 0;
 
        }
        hibernate_flushing = FALSE;
        compaction_swapper_running = 0;
 
-       if (vm_swap_put_failures > vm_swap_put_failures_at_start)
+       if (vm_swap_put_failures > vm_swap_put_failures_at_start) {
                HIBLOG("vm_compressor_flush failed to clean %llu segments - vm_page_compressor_count(%d)\n",
                HIBLOG("vm_compressor_flush failed to clean %llu segments - vm_page_compressor_count(%d)\n",
-                      vm_swap_put_failures - vm_swap_put_failures_at_start, VM_PAGE_COMPRESSOR_COUNT);
-       
+                   vm_swap_put_failures - vm_swap_put_failures_at_start, VM_PAGE_COMPRESSOR_COUNT);
+       }
+
        lck_mtx_unlock_always(c_list_lock);
 
        lck_mtx_unlock_always(c_list_lock);
 
-        clock_get_uptime(&endTime);
-        SUB_ABSOLUTETIME(&endTime, &startTime);
-        absolutetime_to_nanoseconds(endTime, &nsec);
+       thread_wakeup((event_t)&compaction_swapper_running);
+
+       clock_get_uptime(&endTime);
+       SUB_ABSOLUTETIME(&endTime, &startTime);
+       absolutetime_to_nanoseconds(endTime, &nsec);
 
 
-       HIBLOG("vm_compressor_flush completed - took %qd msecs\n", nsec / 1000000ULL);
+       HIBLOG("vm_compressor_flush completed - took %qd msecs - vm_num_swap_files = %d, vm_num_pinned_swap_files = %d, vm_swappin_enabled = %d\n",
+           nsec / 1000000ULL, vm_num_swap_files, vm_num_pinned_swap_files, vm_swappin_enabled);
 }
 
 
 }
 
 
-extern void    vm_swap_file_set_tuneables(void);
-int            compaction_swap_trigger_thread_awakened = 0;
-
+int             compaction_swap_trigger_thread_awakened = 0;
 
 static void
 vm_compressor_swap_trigger_thread(void)
 
 static void
 vm_compressor_swap_trigger_thread(void)
@@ -2006,38 +2666,48 @@ vm_compressor_swap_trigger_thread(void)
 
        /*
         * compaction_swapper_init_now is set when the first call to
 
        /*
         * compaction_swapper_init_now is set when the first call to
-        * vm_consider_waking_compactor_swapper is made from 
-        * vm_pageout_scan... since this function is called upon 
+        * vm_consider_waking_compactor_swapper is made from
+        * vm_pageout_scan... since this function is called upon
         * thread creation, we want to make sure to delay adjusting
         * the tuneables until we are awakened via vm_pageout_scan
         * so that we are at a point where the vm_swapfile_open will
         * be operating on the correct directory (in case the default
         * thread creation, we want to make sure to delay adjusting
         * the tuneables until we are awakened via vm_pageout_scan
         * so that we are at a point where the vm_swapfile_open will
         * be operating on the correct directory (in case the default
-        * of /var/vm/  is overridden by the dymanic_pager 
+        * of using the VM volume is overridden by the dynamic_pager)
         */
         */
-       if (compaction_swapper_init_now && !compaction_swapper_inited) {
-               if (vm_compressor_mode == VM_PAGER_COMPRESSOR_WITH_SWAP)
-                       vm_swap_file_set_tuneables();
+       if (compaction_swapper_init_now) {
+               vm_compaction_swapper_do_init();
 
 
-               if (vm_restricted_to_single_processor == TRUE)
+               if (vm_pageout_state.vm_restricted_to_single_processor == TRUE) {
                        thread_vm_bind_group_add();
                        thread_vm_bind_group_add();
-
-               compaction_swapper_inited = 1;
+               }
+#if CONFIG_THREAD_GROUPS
+               thread_group_vm_add();
+#endif
+               thread_set_thread_name(current_thread(), "VM_cswap_trigger");
+               compaction_swapper_init_now = 0;
        }
        lck_mtx_lock_spin_always(c_list_lock);
 
        compaction_swap_trigger_thread_awakened++;
        }
        lck_mtx_lock_spin_always(c_list_lock);
 
        compaction_swap_trigger_thread_awakened++;
+       compaction_swapper_awakened = 0;
 
 
-       vm_compressor_compact_and_swap(FALSE);
+       if (compaction_swapper_running == 0) {
+               compaction_swapper_running = 1;
 
 
+               vm_compressor_compact_and_swap(FALSE);
+
+               compaction_swapper_running = 0;
+       }
        assert_wait((event_t)&c_compressor_swap_trigger, THREAD_UNINT);
 
        assert_wait((event_t)&c_compressor_swap_trigger, THREAD_UNINT);
 
-       compaction_swapper_running = 0;
-       thread_wakeup((event_t)&compaction_swapper_running);
+       if (compaction_swapper_running == 0) {
+               thread_wakeup((event_t)&compaction_swapper_running);
+       }
 
        lck_mtx_unlock_always(c_list_lock);
 
        lck_mtx_unlock_always(c_list_lock);
-               
+
        thread_block((thread_continue_t)vm_compressor_swap_trigger_thread);
        thread_block((thread_continue_t)vm_compressor_swap_trigger_thread);
-       
+
        /* NOTREACHED */
 }
 
        /* NOTREACHED */
 }
 
@@ -2045,18 +2715,18 @@ vm_compressor_swap_trigger_thread(void)
 void
 vm_compressor_record_warmup_start(void)
 {
 void
 vm_compressor_record_warmup_start(void)
 {
-       c_segment_t     c_seg;
+       c_segment_t     c_seg;
 
        lck_mtx_lock_spin_always(c_list_lock);
 
        if (first_c_segment_to_warm_generation_id == 0) {
                if (!queue_empty(&c_age_list_head)) {
 
        lck_mtx_lock_spin_always(c_list_lock);
 
        if (first_c_segment_to_warm_generation_id == 0) {
                if (!queue_empty(&c_age_list_head)) {
-
                        c_seg = (c_segment_t)queue_last(&c_age_list_head);
 
                        first_c_segment_to_warm_generation_id = c_seg->c_generation_id;
                        c_seg = (c_segment_t)queue_last(&c_age_list_head);
 
                        first_c_segment_to_warm_generation_id = c_seg->c_generation_id;
-               } else
+               } else {
                        first_c_segment_to_warm_generation_id = 0;
                        first_c_segment_to_warm_generation_id = 0;
+               }
 
                fastwake_recording_in_progress = TRUE;
        }
 
                fastwake_recording_in_progress = TRUE;
        }
@@ -2064,22 +2734,21 @@ vm_compressor_record_warmup_start(void)
 }
 
 
 }
 
 
-void 
+void
 vm_compressor_record_warmup_end(void)
 {
 vm_compressor_record_warmup_end(void)
 {
-       c_segment_t     c_seg;
+       c_segment_t     c_seg;
 
        lck_mtx_lock_spin_always(c_list_lock);
 
        if (fastwake_recording_in_progress == TRUE) {
 
        lck_mtx_lock_spin_always(c_list_lock);
 
        if (fastwake_recording_in_progress == TRUE) {
-
                if (!queue_empty(&c_age_list_head)) {
                if (!queue_empty(&c_age_list_head)) {
-
                        c_seg = (c_segment_t)queue_last(&c_age_list_head);
 
                        last_c_segment_to_warm_generation_id = c_seg->c_generation_id;
                        c_seg = (c_segment_t)queue_last(&c_age_list_head);
 
                        last_c_segment_to_warm_generation_id = c_seg->c_generation_id;
-               } else
+               } else {
                        last_c_segment_to_warm_generation_id = first_c_segment_to_warm_generation_id;
                        last_c_segment_to_warm_generation_id = first_c_segment_to_warm_generation_id;
+               }
 
                fastwake_recording_in_progress = FALSE;
 
 
                fastwake_recording_in_progress = FALSE;
 
@@ -2089,13 +2758,13 @@ vm_compressor_record_warmup_end(void)
 }
 
 
 }
 
 
-#define DELAY_TRIM_ON_WAKE_SECS                4
+#define DELAY_TRIM_ON_WAKE_SECS         25
 
 void
 vm_compressor_delay_trim(void)
 {
 
 void
 vm_compressor_delay_trim(void)
 {
-        clock_sec_t    sec;
-       clock_nsec_t    nsec;
+       clock_sec_t     sec;
+       clock_nsec_t    nsec;
 
        clock_get_system_nanotime(&sec, &nsec);
        dont_trim_until_ts = sec + DELAY_TRIM_ON_WAKE_SECS;
 
        clock_get_system_nanotime(&sec, &nsec);
        dont_trim_until_ts = sec + DELAY_TRIM_ON_WAKE_SECS;
@@ -2106,7 +2775,7 @@ void
 vm_compressor_do_warmup(void)
 {
        lck_mtx_lock_spin_always(c_list_lock);
 vm_compressor_do_warmup(void)
 {
        lck_mtx_lock_spin_always(c_list_lock);
-       
+
        if (first_c_segment_to_warm_generation_id == last_c_segment_to_warm_generation_id) {
                first_c_segment_to_warm_generation_id = last_c_segment_to_warm_generation_id = 0;
 
        if (first_c_segment_to_warm_generation_id == last_c_segment_to_warm_generation_id) {
                first_c_segment_to_warm_generation_id = last_c_segment_to_warm_generation_id = 0;
 
@@ -2114,23 +2783,42 @@ vm_compressor_do_warmup(void)
                return;
        }
 
                return;
        }
 
-       if (compaction_swapper_running == 0) {
-
+       if (compaction_swapper_running == 0 && compaction_swapper_awakened == 0) {
                fastwake_warmup = TRUE;
                fastwake_warmup = TRUE;
-               compaction_swapper_running = 1;
+
+               compaction_swapper_awakened = 1;
                thread_wakeup((event_t)&c_compressor_swap_trigger);
        }
        lck_mtx_unlock_always(c_list_lock);
 }
 
                thread_wakeup((event_t)&c_compressor_swap_trigger);
        }
        lck_mtx_unlock_always(c_list_lock);
 }
 
+void
+do_fastwake_warmup_all(void)
+{
+       lck_mtx_lock_spin_always(c_list_lock);
+
+       if (queue_empty(&c_swappedout_list_head) && queue_empty(&c_swappedout_sparse_list_head)) {
+               lck_mtx_unlock_always(c_list_lock);
+               return;
+       }
+
+       fastwake_warmup = TRUE;
+
+       do_fastwake_warmup(&c_swappedout_list_head, TRUE);
+
+       do_fastwake_warmup(&c_swappedout_sparse_list_head, TRUE);
+
+       fastwake_warmup = FALSE;
+
+       lck_mtx_unlock_always(c_list_lock);
+}
 
 void
 
 void
-do_fastwake_warmup(void)
+do_fastwake_warmup(queue_head_t *c_queue, boolean_t consider_all_cseg)
 {
 {
-       uint64_t        my_thread_id;
-       c_segment_t     c_seg = NULL;
-       AbsoluteTime    startTime, endTime;
-       uint64_t        nsec;
+       c_segment_t     c_seg = NULL;
+       AbsoluteTime    startTime, endTime;
+       uint64_t        nsec;
 
 
        HIBLOG("vm_compressor_fastwake_warmup (%qd - %qd) - starting\n", first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id);
 
 
        HIBLOG("vm_compressor_fastwake_warmup (%qd - %qd) - starting\n", first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id);
@@ -2139,36 +2827,38 @@ do_fastwake_warmup(void)
 
        lck_mtx_unlock_always(c_list_lock);
 
 
        lck_mtx_unlock_always(c_list_lock);
 
-       my_thread_id = current_thread()->thread_id;
-       proc_set_task_policy_thread(kernel_task, my_thread_id,
-                                   TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER2);
+       proc_set_thread_policy(current_thread(),
+           TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER2);
 
        PAGE_REPLACEMENT_DISALLOWED(TRUE);
 
        lck_mtx_lock_spin_always(c_list_lock);
 
 
        PAGE_REPLACEMENT_DISALLOWED(TRUE);
 
        lck_mtx_lock_spin_always(c_list_lock);
 
-       while (!queue_empty(&c_swappedout_list_head) && fastwake_warmup == TRUE) {
+       while (!queue_empty(c_queue) && fastwake_warmup == TRUE) {
+               c_seg = (c_segment_t) queue_first(c_queue);
 
 
-               c_seg = (c_segment_t) queue_first(&c_swappedout_list_head);
-
-               if (c_seg->c_generation_id < first_c_segment_to_warm_generation_id || 
-                   c_seg->c_generation_id > last_c_segment_to_warm_generation_id)
-                       break;
+               if (consider_all_cseg == FALSE) {
+                       if (c_seg->c_generation_id < first_c_segment_to_warm_generation_id ||
+                           c_seg->c_generation_id > last_c_segment_to_warm_generation_id) {
+                               break;
+                       }
 
 
-               if (vm_page_free_count < (AVAILABLE_MEMORY / 4))
-                       break;
+                       if (vm_page_free_count < (AVAILABLE_MEMORY / 4)) {
+                               break;
+                       }
+               }
 
                lck_mtx_lock_spin_always(&c_seg->c_lock);
                lck_mtx_unlock_always(c_list_lock);
 
                lck_mtx_lock_spin_always(&c_seg->c_lock);
                lck_mtx_unlock_always(c_list_lock);
-               
+
                if (c_seg->c_busy) {
                        PAGE_REPLACEMENT_DISALLOWED(FALSE);
                        c_seg_wait_on_busy(c_seg);
                        PAGE_REPLACEMENT_DISALLOWED(TRUE);
                } else {
                if (c_seg->c_busy) {
                        PAGE_REPLACEMENT_DISALLOWED(FALSE);
                        c_seg_wait_on_busy(c_seg);
                        PAGE_REPLACEMENT_DISALLOWED(TRUE);
                } else {
-                       c_seg_swapin(c_seg, TRUE);
-
-                       lck_mtx_unlock_always(&c_seg->c_lock);
+                       if (c_seg_swapin(c_seg, TRUE, FALSE) == 0) {
+                               lck_mtx_unlock_always(&c_seg->c_lock);
+                       }
                        c_segment_warmup_count++;
 
                        PAGE_REPLACEMENT_DISALLOWED(FALSE);
                        c_segment_warmup_count++;
 
                        PAGE_REPLACEMENT_DISALLOWED(FALSE);
@@ -2181,38 +2871,47 @@ do_fastwake_warmup(void)
 
        PAGE_REPLACEMENT_DISALLOWED(FALSE);
 
 
        PAGE_REPLACEMENT_DISALLOWED(FALSE);
 
-       proc_set_task_policy_thread(kernel_task, my_thread_id,
-                                   TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER0);
+       proc_set_thread_policy(current_thread(),
+           TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER0);
 
 
-        clock_get_uptime(&endTime);
-        SUB_ABSOLUTETIME(&endTime, &startTime);
-        absolutetime_to_nanoseconds(endTime, &nsec);
+       clock_get_uptime(&endTime);
+       SUB_ABSOLUTETIME(&endTime, &startTime);
+       absolutetime_to_nanoseconds(endTime, &nsec);
 
        HIBLOG("vm_compressor_fastwake_warmup completed - took %qd msecs\n", nsec / 1000000ULL);
 
        lck_mtx_lock_spin_always(c_list_lock);
 
 
        HIBLOG("vm_compressor_fastwake_warmup completed - took %qd msecs\n", nsec / 1000000ULL);
 
        lck_mtx_lock_spin_always(c_list_lock);
 
-       first_c_segment_to_warm_generation_id = last_c_segment_to_warm_generation_id = 0;
+       if (consider_all_cseg == FALSE) {
+               first_c_segment_to_warm_generation_id = last_c_segment_to_warm_generation_id = 0;
+       }
 }
 
 }
 
+int min_csegs_per_major_compaction = DELAYED_COMPACTIONS_PER_PASS;
+extern bool     vm_swapout_thread_running;
+extern boolean_t        compressor_store_stop_compaction;
 
 void
 vm_compressor_compact_and_swap(boolean_t flush_all)
 {
 
 void
 vm_compressor_compact_and_swap(boolean_t flush_all)
 {
-       c_segment_t     c_seg, c_seg_next;
-       boolean_t       keep_compacting;
-       clock_sec_t     now;
-       clock_nsec_t    nsec;
+       c_segment_t     c_seg, c_seg_next;
+       boolean_t       keep_compacting, switch_state;
+       clock_sec_t     now;
+       clock_nsec_t    nsec;
+       mach_timespec_t start_ts, end_ts;
+       unsigned int    number_considered, wanted_cseg_found, yield_after_considered_per_pass, number_yields;
+       uint64_t        bytes_to_free, bytes_freed, delta_usec;
 
 
+       VM_DEBUG_CONSTANT_EVENT(vm_compressor_compact_and_swap, VM_COMPRESSOR_COMPACT_AND_SWAP, DBG_FUNC_START, c_age_count, c_minor_count, c_major_count, vm_page_free_count);
 
        if (fastwake_warmup == TRUE) {
 
        if (fastwake_warmup == TRUE) {
-               uint64_t        starting_warmup_count;
+               uint64_t        starting_warmup_count;
 
                starting_warmup_count = c_segment_warmup_count;
 
                KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 11) | DBG_FUNC_START, c_segment_warmup_count,
 
                starting_warmup_count = c_segment_warmup_count;
 
                KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 11) | DBG_FUNC_START, c_segment_warmup_count,
-                                     first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id, 0, 0);
-               do_fastwake_warmup();
+                   first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id, 0, 0);
+               do_fastwake_warmup(&c_swappedout_list_head, FALSE);
                KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 11) | DBG_FUNC_END, c_segment_warmup_count, c_segment_warmup_count - starting_warmup_count, 0, 0, 0);
 
                fastwake_warmup = FALSE;
                KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 11) | DBG_FUNC_END, c_segment_warmup_count, c_segment_warmup_count - starting_warmup_count, 0, 0, 0);
 
                fastwake_warmup = FALSE;
@@ -2231,16 +2930,25 @@ vm_compressor_compact_and_swap(boolean_t flush_all)
 
        /*
         * we only need to grab the timestamp once per
 
        /*
         * we only need to grab the timestamp once per
-        * invocation of this function since the 
+        * invocation of this function since the
         * timescale we're interested in is measured
         * in days
         */
         * timescale we're interested in is measured
         * in days
         */
-       clock_get_system_nanotime(&now,  &nsec);
-
-       while (!queue_empty(&c_age_list_head) && compaction_swapper_abort == 0) {
-
+       clock_get_system_nanotime(&now, &nsec);
+
+       start_ts.tv_sec = (int) now;
+       start_ts.tv_nsec = nsec;
+       delta_usec = 0;
+       number_considered = 0;
+       wanted_cseg_found = 0;
+       number_yields = 0;
+       bytes_to_free = 0;
+       bytes_freed = 0;
+       yield_after_considered_per_pass = MAX(min_csegs_per_major_compaction, DELAYED_COMPACTIONS_PER_PASS);
+
+       while (!queue_empty(&c_age_list_head) && !compaction_swapper_abort && !compressor_store_stop_compaction) {
                if (hibernate_flushing == TRUE) {
                if (hibernate_flushing == TRUE) {
-                       clock_sec_t     sec;
+                       clock_sec_t     sec;
 
                        if (hibernate_should_abort()) {
                                HIBLOG("vm_compressor_flush - hibernate_should_abort returned TRUE\n");
 
                        if (hibernate_should_abort()) {
                                HIBLOG("vm_compressor_flush - hibernate_should_abort returned TRUE\n");
@@ -2250,19 +2958,33 @@ vm_compressor_compact_and_swap(boolean_t flush_all)
                                HIBLOG("vm_compressor_flush - out of swap space\n");
                                break;
                        }
                                HIBLOG("vm_compressor_flush - out of swap space\n");
                                break;
                        }
+                       if (vm_swap_files_pinned() == FALSE) {
+                               HIBLOG("vm_compressor_flush - unpinned swap files\n");
+                               break;
+                       }
+                       if (hibernate_in_progress_with_pinned_swap == TRUE &&
+                           (vm_swapfile_total_segs_alloced == vm_swapfile_total_segs_used)) {
+                               HIBLOG("vm_compressor_flush - out of pinned swap space\n");
+                               break;
+                       }
                        clock_get_system_nanotime(&sec, &nsec);
                        clock_get_system_nanotime(&sec, &nsec);
-               
+
                        if (sec > hibernate_flushing_deadline) {
                                HIBLOG("vm_compressor_flush - failed to finish before deadline\n");
                                break;
                        }
                }
                        if (sec > hibernate_flushing_deadline) {
                                HIBLOG("vm_compressor_flush - failed to finish before deadline\n");
                                break;
                        }
                }
-               if (c_swapout_count >= C_SWAPOUT_LIMIT) {
+               if (!vm_swap_out_of_space() && c_swapout_count >= C_SWAPOUT_LIMIT) {
+                       assert_wait_timeout((event_t) &compaction_swapper_running, THREAD_INTERRUPTIBLE, 100, 1000 * NSEC_PER_USEC);
 
 
-                       assert_wait_timeout((event_t) &compaction_swapper_running, THREAD_INTERRUPTIBLE, 100, 1000*NSEC_PER_USEC);
+                       if (!vm_swapout_thread_running) {
+                               thread_wakeup((event_t)&c_swapout_list_head);
+                       }
 
                        lck_mtx_unlock_always(c_list_lock);
 
 
                        lck_mtx_unlock_always(c_list_lock);
 
+                       VM_DEBUG_CONSTANT_EVENT(vm_compressor_compact_and_swap, VM_COMPRESSOR_COMPACT_AND_SWAP, DBG_FUNC_NONE, 1, c_swapout_count, 0, 0);
+
                        thread_block(THREAD_CONTINUE_NULL);
 
                        lck_mtx_lock_spin_always(c_list_lock);
                        thread_block(THREAD_CONTINUE_NULL);
 
                        lck_mtx_lock_spin_always(c_list_lock);
@@ -2274,7 +2996,7 @@ vm_compressor_compact_and_swap(boolean_t flush_all)
 
                vm_compressor_age_swapped_in_segments(flush_all);
 
 
                vm_compressor_age_swapped_in_segments(flush_all);
 
-               if (c_swapout_count >= C_SWAPOUT_LIMIT) {
+               if (!vm_swap_out_of_space() && c_swapout_count >= C_SWAPOUT_LIMIT) {
                        /*
                         * we timed out on the above thread_block
                         * let's loop around and try again
                        /*
                         * we timed out on the above thread_block
                         * let's loop around and try again
@@ -2282,6 +3004,8 @@ vm_compressor_compact_and_swap(boolean_t flush_all)
                         * to do minor compactions to make
                         * more memory available
                         */
                         * to do minor compactions to make
                         * more memory available
                         */
+                       VM_DEBUG_CONSTANT_EVENT(vm_compressor_compact_and_swap, VM_COMPRESSOR_COMPACT_AND_SWAP, DBG_FUNC_NONE, 2, c_swapout_count, 0, 0);
+
                        continue;
                }
 
                        continue;
                }
 
@@ -2289,32 +3013,37 @@ vm_compressor_compact_and_swap(boolean_t flush_all)
                 * Swap out segments?
                 */
                if (flush_all == FALSE) {
                 * Swap out segments?
                 */
                if (flush_all == FALSE) {
-                       boolean_t       needs_to_swap;
+                       boolean_t       needs_to_swap;
 
                        lck_mtx_unlock_always(c_list_lock);
 
                        needs_to_swap = compressor_needs_to_swap();
 
 
                        lck_mtx_unlock_always(c_list_lock);
 
                        needs_to_swap = compressor_needs_to_swap();
 
-                       if (needs_to_swap == TRUE && vm_swap_low_on_space())
-                               vm_compressor_take_paging_space_action();
-
                        lck_mtx_lock_spin_always(c_list_lock);
                        lck_mtx_lock_spin_always(c_list_lock);
-                       
-                       if (needs_to_swap == FALSE)
+
+                       VM_DEBUG_CONSTANT_EVENT(vm_compressor_compact_and_swap, VM_COMPRESSOR_COMPACT_AND_SWAP, DBG_FUNC_NONE, 3, needs_to_swap, 0, 0);
+
+                       if (needs_to_swap == FALSE) {
                                break;
                                break;
+                       }
                }
                }
-               if (queue_empty(&c_age_list_head))
+               if (queue_empty(&c_age_list_head)) {
+                       VM_DEBUG_CONSTANT_EVENT(vm_compressor_compact_and_swap, VM_COMPRESSOR_COMPACT_AND_SWAP, DBG_FUNC_NONE, 4, c_age_count, 0, 0);
                        break;
                        break;
+               }
                c_seg = (c_segment_t) queue_first(&c_age_list_head);
 
                assert(c_seg->c_state == C_ON_AGE_Q);
 
                c_seg = (c_segment_t) queue_first(&c_age_list_head);
 
                assert(c_seg->c_state == C_ON_AGE_Q);
 
-               if (flush_all == TRUE && c_seg->c_generation_id > c_generation_id_flush_barrier)
+               if (flush_all == TRUE && c_seg->c_generation_id > c_generation_id_flush_barrier) {
+                       VM_DEBUG_CONSTANT_EVENT(vm_compressor_compact_and_swap, VM_COMPRESSOR_COMPACT_AND_SWAP, DBG_FUNC_NONE, 5, 0, 0, 0);
                        break;
                        break;
-               
+               }
+
                lck_mtx_lock_spin_always(&c_seg->c_lock);
 
                if (c_seg->c_busy) {
                lck_mtx_lock_spin_always(&c_seg->c_lock);
 
                if (c_seg->c_busy) {
+                       VM_DEBUG_CONSTANT_EVENT(vm_compressor_compact_and_swap, VM_COMPRESSOR_COMPACT_AND_SWAP, DBG_FUNC_NONE, 6, (void*) VM_KERNEL_ADDRPERM(c_seg), 0, 0);
 
                        lck_mtx_unlock_always(c_list_lock);
                        c_seg_wait_on_busy(c_seg);
 
                        lck_mtx_unlock_always(c_list_lock);
                        c_seg_wait_on_busy(c_seg);
@@ -2329,35 +3058,56 @@ vm_compressor_compact_and_swap(boolean_t flush_all)
                         * found an empty c_segment and freed it
                         * so go grab the next guy in the queue
                         */
                         * found an empty c_segment and freed it
                         * so go grab the next guy in the queue
                         */
-                       c_seg_major_compact_stats.count_of_freed_segs++;
+                       VM_DEBUG_CONSTANT_EVENT(vm_compressor_compact_and_swap, VM_COMPRESSOR_COMPACT_AND_SWAP, DBG_FUNC_NONE, 7, 0, 0, 0);
+                       c_seg_major_compact_stats[c_seg_major_compact_stats_now].count_of_freed_segs++;
                        continue;
                }
                /*
                 * Major compaction
                 */
                keep_compacting = TRUE;
                        continue;
                }
                /*
                 * Major compaction
                 */
                keep_compacting = TRUE;
+               switch_state = TRUE;
 
                while (keep_compacting == TRUE) {
 
                while (keep_compacting == TRUE) {
-                                       
                        assert(c_seg->c_busy);
 
                        /* look for another segment to consolidate */
 
                        c_seg_next = (c_segment_t) queue_next(&c_seg->c_age_list);
                        assert(c_seg->c_busy);
 
                        /* look for another segment to consolidate */
 
                        c_seg_next = (c_segment_t) queue_next(&c_seg->c_age_list);
-                       
-                       if (queue_end(&c_age_list_head, (queue_entry_t)c_seg_next))
+
+                       if (queue_end(&c_age_list_head, (queue_entry_t)c_seg_next)) {
                                break;
                                break;
+                       }
 
                        assert(c_seg_next->c_state == C_ON_AGE_Q);
 
 
                        assert(c_seg_next->c_state == C_ON_AGE_Q);
 
-                       if (c_seg_major_compact_ok(c_seg, c_seg_next) == FALSE)
+                       number_considered++;
+
+                       if (c_seg_major_compact_ok(c_seg, c_seg_next) == FALSE) {
                                break;
                                break;
+                       }
 
                        lck_mtx_lock_spin_always(&c_seg_next->c_lock);
 
                        if (c_seg_next->c_busy) {
 
                        lck_mtx_lock_spin_always(&c_seg_next->c_lock);
 
                        if (c_seg_next->c_busy) {
+                               /*
+                                * We are going to block for our neighbor.
+                                * If our c_seg is wanted, we should unbusy
+                                * it because we don't know how long we might
+                                * have to block here.
+                                */
+                               if (c_seg->c_wanted) {
+                                       lck_mtx_unlock_always(&c_seg_next->c_lock);
+                                       switch_state = FALSE;
+                                       c_seg_major_compact_stats[c_seg_major_compact_stats_now].bailed_compactions++;
+                                       wanted_cseg_found++;
+                                       break;
+                               }
 
                                lck_mtx_unlock_always(c_list_lock);
 
                                lck_mtx_unlock_always(c_list_lock);
+
+                               VM_DEBUG_CONSTANT_EVENT(vm_compressor_compact_and_swap, VM_COMPRESSOR_COMPACT_AND_SWAP, DBG_FUNC_NONE, 8, (void*) VM_KERNEL_ADDRPERM(c_seg_next), 0, 0);
+
                                c_seg_wait_on_busy(c_seg_next);
                                lck_mtx_lock_spin_always(c_list_lock);
 
                                c_seg_wait_on_busy(c_seg_next);
                                lck_mtx_lock_spin_always(c_list_lock);
 
@@ -2366,12 +3116,14 @@ vm_compressor_compact_and_swap(boolean_t flush_all)
                        /* grab that segment */
                        C_SEG_BUSY(c_seg_next);
 
                        /* grab that segment */
                        C_SEG_BUSY(c_seg_next);
 
+                       bytes_to_free = C_SEG_OFFSET_TO_BYTES(c_seg_next->c_populated_offset);
                        if (c_seg_do_minor_compaction_and_unlock(c_seg_next, FALSE, TRUE, TRUE)) {
                                /*
                                 * found an empty c_segment and freed it
                                 * so we can't continue to use c_seg_next
                                 */
                        if (c_seg_do_minor_compaction_and_unlock(c_seg_next, FALSE, TRUE, TRUE)) {
                                /*
                                 * found an empty c_segment and freed it
                                 * so we can't continue to use c_seg_next
                                 */
-                               c_seg_major_compact_stats.count_of_freed_segs++;
+                               bytes_freed += bytes_to_free;
+                               c_seg_major_compact_stats[c_seg_major_compact_stats_now].count_of_freed_segs++;
                                continue;
                        }
 
                                continue;
                        }
 
@@ -2382,12 +3134,14 @@ vm_compressor_compact_and_swap(boolean_t flush_all)
 
                        keep_compacting = c_seg_major_compact(c_seg, c_seg_next);
 
 
                        keep_compacting = c_seg_major_compact(c_seg, c_seg_next);
 
+                       VM_DEBUG_CONSTANT_EVENT(vm_compressor_compact_and_swap, VM_COMPRESSOR_COMPACT_AND_SWAP, DBG_FUNC_NONE, 9, keep_compacting, 0, 0);
+
                        PAGE_REPLACEMENT_DISALLOWED(TRUE);
 
                        lck_mtx_lock_spin_always(&c_seg_next->c_lock);
                        /*
                         * run a minor compaction on the donor segment
                        PAGE_REPLACEMENT_DISALLOWED(TRUE);
 
                        lck_mtx_lock_spin_always(&c_seg_next->c_lock);
                        /*
                         * run a minor compaction on the donor segment
-                        * since we pulled at least some of it's 
+                        * since we pulled at least some of it's
                         * data into our target...  if we've emptied
                         * it, now is a good time to free it which
                         * c_seg_minor_compaction_and_unlock also takes care of
                         * data into our target...  if we've emptied
                         * it, now is a good time to free it which
                         * c_seg_minor_compaction_and_unlock also takes care of
@@ -2395,32 +3149,52 @@ vm_compressor_compact_and_swap(boolean_t flush_all)
                         * by passing TRUE, we ask for c_busy to be cleared
                         * and c_wanted to be taken care of
                         */
                         * by passing TRUE, we ask for c_busy to be cleared
                         * and c_wanted to be taken care of
                         */
-                       if (c_seg_minor_compaction_and_unlock(c_seg_next, TRUE))
-                               c_seg_major_compact_stats.count_of_freed_segs++;
+                       bytes_to_free = C_SEG_OFFSET_TO_BYTES(c_seg_next->c_populated_offset);
+                       if (c_seg_minor_compaction_and_unlock(c_seg_next, TRUE)) {
+                               bytes_freed += bytes_to_free;
+                               c_seg_major_compact_stats[c_seg_major_compact_stats_now].count_of_freed_segs++;
+                       } else {
+                               bytes_to_free -= C_SEG_OFFSET_TO_BYTES(c_seg_next->c_populated_offset);
+                               bytes_freed += bytes_to_free;
+                       }
 
                        PAGE_REPLACEMENT_DISALLOWED(FALSE);
 
                        /* relock the list */
                        lck_mtx_lock_spin_always(c_list_lock);
 
 
                        PAGE_REPLACEMENT_DISALLOWED(FALSE);
 
                        /* relock the list */
                        lck_mtx_lock_spin_always(c_list_lock);
 
+                       if (c_seg->c_wanted) {
+                               /*
+                                * Our c_seg is in demand. Let's
+                                * unbusy it and wakeup the waiters
+                                * instead of continuing the compaction
+                                * because we could be in this loop
+                                * for a while.
+                                */
+                               switch_state = FALSE;
+                               wanted_cseg_found++;
+                               c_seg_major_compact_stats[c_seg_major_compact_stats_now].bailed_compactions++;
+                               break;
+                       }
                } /* major compaction */
 
                } /* major compaction */
 
+               VM_DEBUG_CONSTANT_EVENT(vm_compressor_compact_and_swap, VM_COMPRESSOR_COMPACT_AND_SWAP, DBG_FUNC_NONE, 10, number_considered, wanted_cseg_found, 0);
+
                lck_mtx_lock_spin_always(&c_seg->c_lock);
 
                assert(c_seg->c_busy);
                assert(!c_seg->c_on_minorcompact_q);
 
                lck_mtx_lock_spin_always(&c_seg->c_lock);
 
                assert(c_seg->c_busy);
                assert(!c_seg->c_on_minorcompact_q);
 
-               if (vm_swap_up == TRUE) {
-                       /*
-                        * This mode of putting a generic c_seg on the swapout list is
-                        * only supported when we have general swap ON i.e.
-                        * we compress pages into c_segs as we process them off
-                        * the paging queues in vm_pageout_scan().
-                        */
-                       if (COMPRESSED_PAGER_IS_SWAPBACKED)
+               if (switch_state) {
+                       if (VM_CONFIG_SWAP_IS_ACTIVE) {
+                               /*
+                                * This mode of putting a generic c_seg on the swapout list is
+                                * only supported when we have general swapping enabled
+                                */
                                c_seg_switch_state(c_seg, C_ON_SWAPOUT_Q, FALSE);
                                c_seg_switch_state(c_seg, C_ON_SWAPOUT_Q, FALSE);
-                       else {
+                       else {
                                if ((vm_swapout_ripe_segments == TRUE && c_overage_swapped_count < c_overage_swapped_limit)) {
                                if ((vm_swapout_ripe_segments == TRUE && c_overage_swapped_count < c_overage_swapped_limit)) {
+                                       assert(VM_CONFIG_SWAP_IS_PRESENT);
                                        /*
                                         * we are running compressor sweeps with swap-behind
                                         * make sure the c_seg has aged enough before swapping it
                                        /*
                                         * we are running compressor sweeps with swap-behind
                                         * make sure the c_seg has aged enough before swapping it
@@ -2433,52 +3207,107 @@ vm_compressor_compact_and_swap(boolean_t flush_all)
                                        }
                                }
                        }
                                        }
                                }
                        }
+                       if (c_seg->c_state == C_ON_AGE_Q) {
+                               /*
+                                * this c_seg didn't get moved to the swapout queue
+                                * so we need to move it out of the way...
+                                * we just did a major compaction on it so put it
+                                * on that queue
+                                */
+                               c_seg_switch_state(c_seg, C_ON_MAJORCOMPACT_Q, FALSE);
+                       } else {
+                               c_seg_major_compact_stats[c_seg_major_compact_stats_now].wasted_space_in_swapouts += C_SEG_BUFSIZE - c_seg->c_bytes_used;
+                               c_seg_major_compact_stats[c_seg_major_compact_stats_now].count_of_swapouts++;
+                       }
                }
                }
-               if (c_seg->c_state == C_ON_AGE_Q) {
-                       /*
-                        * this c_seg didn't get moved to the swapout queue
-                        * so we need to move it out of the way...
-                        * we just did a major compaction on it so put it
-                        * on that queue
-                        */ 
-                       c_seg_switch_state(c_seg, C_ON_MAJORCOMPACT_Q, FALSE);
-               } else {
-                       c_seg_major_compact_stats.wasted_space_in_swapouts += C_SEG_BUFSIZE - c_seg->c_bytes_used;
-                       c_seg_major_compact_stats.count_of_swapouts++;
-               }
+
                C_SEG_WAKEUP_DONE(c_seg);
 
                lck_mtx_unlock_always(&c_seg->c_lock);
 
                if (c_swapout_count) {
                C_SEG_WAKEUP_DONE(c_seg);
 
                lck_mtx_unlock_always(&c_seg->c_lock);
 
                if (c_swapout_count) {
-                       lck_mtx_unlock_always(c_list_lock);
+                       /*
+                        * We don't pause/yield here because we will either
+                        * yield below or at the top of the loop with the
+                        * assert_wait_timeout.
+                        */
+                       if (!vm_swapout_thread_running) {
+                               thread_wakeup((event_t)&c_swapout_list_head);
+                       }
+               }
 
 
-                       thread_wakeup((event_t)&c_swapout_list_head);
-                       
-                       lck_mtx_lock_spin_always(c_list_lock);
+               if (number_considered >= yield_after_considered_per_pass) {
+                       if (wanted_cseg_found) {
+                               /*
+                                * We stopped major compactions on a c_seg
+                                * that is wanted. We don't know the priority
+                                * of the waiter unfortunately but we are at
+                                * a very high priority and so, just in case
+                                * the waiter is a critical system daemon or
+                                * UI thread, let's give up the CPU in case
+                                * the system is running a few CPU intensive
+                                * tasks.
+                                */
+                               lck_mtx_unlock_always(c_list_lock);
+
+                               mutex_pause(2); /* 100us yield */
+
+                               number_yields++;
+
+                               VM_DEBUG_CONSTANT_EVENT(vm_compressor_compact_and_swap, VM_COMPRESSOR_COMPACT_AND_SWAP, DBG_FUNC_NONE, 11, number_considered, number_yields, 0);
+
+                               lck_mtx_lock_spin_always(c_list_lock);
+                       }
+
+                       number_considered = 0;
+                       wanted_cseg_found = 0;
                }
        }
                }
        }
+       clock_get_system_nanotime(&now, &nsec);
+       end_ts.tv_sec = (int) now;
+       end_ts.tv_nsec = nsec;
+
+       SUB_MACH_TIMESPEC(&end_ts, &start_ts);
+
+       delta_usec = (end_ts.tv_sec * USEC_PER_SEC) + (end_ts.tv_nsec / NSEC_PER_USEC) - (number_yields * 100);
+
+       delta_usec = MAX(1, delta_usec); /* we could have 0 usec run if conditions weren't right */
+
+       c_seg_major_compact_stats[c_seg_major_compact_stats_now].bytes_freed_rate_us = (bytes_freed / delta_usec);
+
+       if ((c_seg_major_compact_stats_now + 1) == C_SEG_MAJOR_COMPACT_STATS_MAX) {
+               c_seg_major_compact_stats_now = 0;
+       } else {
+               c_seg_major_compact_stats_now++;
+       }
+
+       assert(c_seg_major_compact_stats_now < C_SEG_MAJOR_COMPACT_STATS_MAX);
+
+       VM_DEBUG_CONSTANT_EVENT(vm_compressor_compact_and_swap, VM_COMPRESSOR_COMPACT_AND_SWAP, DBG_FUNC_END, c_age_count, c_minor_count, c_major_count, vm_page_free_count);
 }
 
 
 static c_segment_t
 c_seg_allocate(c_segment_t *current_chead)
 {
 }
 
 
 static c_segment_t
 c_seg_allocate(c_segment_t *current_chead)
 {
-       c_segment_t     c_seg;
-       int             min_needed;
-       int             size_to_populate;
+       c_segment_t     c_seg;
+       int             min_needed;
+       int             size_to_populate;
 
 
-       if (vm_compressor_low_on_space())
+#if XNU_TARGET_OS_OSX
+       if (vm_compressor_low_on_space()) {
                vm_compressor_take_paging_space_action();
                vm_compressor_take_paging_space_action();
+       }
+#endif /* XNU_TARGET_OS_OSX */
 
 
-       if ( (c_seg = *current_chead) == NULL ) {
-               uint32_t        c_segno;
+       if ((c_seg = *current_chead) == NULL) {
+               uint32_t        c_segno;
 
                lck_mtx_lock_spin_always(c_list_lock);
 
                while (c_segments_busy == TRUE) {
                        assert_wait((event_t) (&c_segments_busy), THREAD_UNINT);
 
                lck_mtx_lock_spin_always(c_list_lock);
 
                while (c_segments_busy == TRUE) {
                        assert_wait((event_t) (&c_segments_busy), THREAD_UNINT);
-       
+
                        lck_mtx_unlock_always(c_list_lock);
 
                        thread_block(THREAD_CONTINUE_NULL);
                        lck_mtx_unlock_always(c_list_lock);
 
                        thread_block(THREAD_CONTINUE_NULL);
@@ -2486,27 +3315,40 @@ c_seg_allocate(c_segment_t *current_chead)
                        lck_mtx_lock_spin_always(c_list_lock);
                }
                if (c_free_segno_head == (uint32_t)-1) {
                        lck_mtx_lock_spin_always(c_list_lock);
                }
                if (c_free_segno_head == (uint32_t)-1) {
-                       uint32_t        c_segments_available_new;
+                       uint32_t        c_segments_available_new;
+                       uint32_t        compressed_pages;
+
+#if CONFIG_FREEZE
+                       if (freezer_incore_cseg_acct) {
+                               compressed_pages = c_segment_pages_compressed_incore;
+                       } else {
+                               compressed_pages = c_segment_pages_compressed;
+                       }
+#else
+                       compressed_pages = c_segment_pages_compressed;
+#endif /* CONFIG_FREEZE */
 
 
-                       if (c_segments_available >= c_segments_limit || c_segment_pages_compressed >= c_segment_pages_compressed_limit) {
+                       if (c_segments_available >= c_segments_limit || compressed_pages >= c_segment_pages_compressed_limit) {
                                lck_mtx_unlock_always(c_list_lock);
 
                                lck_mtx_unlock_always(c_list_lock);
 
-                               return (NULL);
+                               return NULL;
                        }
                        c_segments_busy = TRUE;
                        lck_mtx_unlock_always(c_list_lock);
 
                        }
                        c_segments_busy = TRUE;
                        lck_mtx_unlock_always(c_list_lock);
 
-                       kernel_memory_populate(kernel_map, (vm_offset_t)c_segments_next_page, 
-                                               PAGE_SIZE, KMA_KOBJECT, VM_KERN_MEMORY_COMPRESSOR);
+                       kernel_memory_populate(compressor_map, (vm_offset_t)c_segments_next_page,
+                           PAGE_SIZE, KMA_KOBJECT, VM_KERN_MEMORY_COMPRESSOR);
                        c_segments_next_page += PAGE_SIZE;
 
                        c_segments_available_new = c_segments_available + C_SEGMENTS_PER_PAGE;
 
                        c_segments_next_page += PAGE_SIZE;
 
                        c_segments_available_new = c_segments_available + C_SEGMENTS_PER_PAGE;
 
-                       if (c_segments_available_new > c_segments_limit)
+                       if (c_segments_available_new > c_segments_limit) {
                                c_segments_available_new = c_segments_limit;
                                c_segments_available_new = c_segments_limit;
+                       }
 
 
-                       for (c_segno = c_segments_available + 1; c_segno < c_segments_available_new; c_segno++)
+                       for (c_segno = c_segments_available + 1; c_segno < c_segments_available_new; c_segno++) {
                                c_segments[c_segno - 1].c_segno = c_segno;
                                c_segments[c_segno - 1].c_segno = c_segno;
+                       }
 
                        lck_mtx_lock_spin_always(c_list_lock);
 
 
                        lck_mtx_lock_spin_always(c_list_lock);
 
@@ -2520,7 +3362,7 @@ c_seg_allocate(c_segment_t *current_chead)
                c_segno = c_free_segno_head;
                assert(c_segno >= 0 && c_segno < c_segments_limit);
 
                c_segno = c_free_segno_head;
                assert(c_segno >= 0 && c_segno < c_segments_limit);
 
-               c_free_segno_head = c_segments[c_segno].c_segno;
+               c_free_segno_head = (uint32_t)c_segments[c_segno].c_segno;
 
                /*
                 * do the rest of the bookkeeping now while we're still behind
 
                /*
                 * do the rest of the bookkeeping now while we're still behind
@@ -2528,22 +3370,18 @@ c_seg_allocate(c_segment_t *current_chead)
                 * so that we can install it once we have the c_seg allocated
                 */
                c_segment_count++;
                 * so that we can install it once we have the c_seg allocated
                 */
                c_segment_count++;
-               if (c_segment_count > c_segment_count_max)
+               if (c_segment_count > c_segment_count_max) {
                        c_segment_count_max = c_segment_count;
                        c_segment_count_max = c_segment_count;
+               }
 
                lck_mtx_unlock_always(c_list_lock);
 
 
                lck_mtx_unlock_always(c_list_lock);
 
-               c_seg = (c_segment_t)zalloc(compressor_segment_zone);
-               bzero((char *)c_seg, sizeof(struct c_segment));
+               c_seg = zalloc_flags(compressor_segment_zone, Z_WAITOK | Z_ZERO);
 
                c_seg->c_store.c_buffer = (int32_t *)C_SEG_BUFFER_ADDRESS(c_segno);
 
 
                c_seg->c_store.c_buffer = (int32_t *)C_SEG_BUFFER_ADDRESS(c_segno);
 
-#if __i386__ || __x86_64__
-               lck_mtx_init(&c_seg->c_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
-#else /* __i386__ || __x86_64__ */
-               lck_spin_init(&c_seg->c_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
-#endif /* __i386__ || __x86_64__ */
-       
+               lck_mtx_init(&c_seg->c_lock, &vm_compressor_lck_grp, LCK_ATTR_NULL);
+
                c_seg->c_state = C_IS_EMPTY;
                c_seg->c_firstemptyslot = C_SLOT_MAX_INDEX;
                c_seg->c_mysegno = c_segno;
                c_seg->c_state = C_IS_EMPTY;
                c_seg->c_firstemptyslot = C_SLOT_MAX_INDEX;
                c_seg->c_mysegno = c_segno;
@@ -2552,55 +3390,68 @@ c_seg_allocate(c_segment_t *current_chead)
                c_empty_count++;
                c_seg_switch_state(c_seg, C_IS_FILLING, FALSE);
                c_segments[c_segno].c_seg = c_seg;
                c_empty_count++;
                c_seg_switch_state(c_seg, C_IS_FILLING, FALSE);
                c_segments[c_segno].c_seg = c_seg;
+               assert(c_segments[c_segno].c_segno > c_segments_available);
                lck_mtx_unlock_always(c_list_lock);
 
                *current_chead = c_seg;
                lck_mtx_unlock_always(c_list_lock);
 
                *current_chead = c_seg;
+
+#if DEVELOPMENT || DEBUG
+               C_SEG_MAKE_WRITEABLE(c_seg);
+#endif
        }
        c_seg_alloc_nextslot(c_seg);
 
        size_to_populate = C_SEG_ALLOCSIZE - C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset);
        }
        c_seg_alloc_nextslot(c_seg);
 
        size_to_populate = C_SEG_ALLOCSIZE - C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset);
-       
-       if (size_to_populate) {
 
 
+       if (size_to_populate) {
                min_needed = PAGE_SIZE + (C_SEG_ALLOCSIZE - C_SEG_BUFSIZE);
 
                if (C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset - c_seg->c_nextoffset) < (unsigned) min_needed) {
                min_needed = PAGE_SIZE + (C_SEG_ALLOCSIZE - C_SEG_BUFSIZE);
 
                if (C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset - c_seg->c_nextoffset) < (unsigned) min_needed) {
-
-                       if (size_to_populate > C_SEG_MAX_POPULATE_SIZE)
+                       if (size_to_populate > C_SEG_MAX_POPULATE_SIZE) {
                                size_to_populate = C_SEG_MAX_POPULATE_SIZE;
                                size_to_populate = C_SEG_MAX_POPULATE_SIZE;
+                       }
 
 
-                       kernel_memory_populate(kernel_map,
-                                              (vm_offset_t) &c_seg->c_store.c_buffer[c_seg->c_populated_offset],
-                                              size_to_populate,
-                                              KMA_COMPRESSOR,
-                                              VM_KERN_MEMORY_COMPRESSOR);
-               } else
+                       OSAddAtomic64(size_to_populate / PAGE_SIZE, &vm_pageout_vminfo.vm_compressor_pages_grabbed);
+
+                       kernel_memory_populate(compressor_map,
+                           (vm_offset_t) &c_seg->c_store.c_buffer[c_seg->c_populated_offset],
+                           size_to_populate,
+                           KMA_COMPRESSOR,
+                           VM_KERN_MEMORY_COMPRESSOR);
+               } else {
                        size_to_populate = 0;
                        size_to_populate = 0;
+               }
        }
        PAGE_REPLACEMENT_DISALLOWED(TRUE);
 
        lck_mtx_lock_spin_always(&c_seg->c_lock);
 
        }
        PAGE_REPLACEMENT_DISALLOWED(TRUE);
 
        lck_mtx_lock_spin_always(&c_seg->c_lock);
 
-       if (size_to_populate)
+       if (size_to_populate) {
                c_seg->c_populated_offset += C_SEG_BYTES_TO_OFFSET(size_to_populate);
                c_seg->c_populated_offset += C_SEG_BYTES_TO_OFFSET(size_to_populate);
+       }
 
 
-       return (c_seg);
+       return c_seg;
 }
 
 }
 
+#if DEVELOPMENT || DEBUG
+#if CONFIG_FREEZE
+extern boolean_t memorystatus_freeze_to_memory;
+#endif /* CONFIG_FREEZE */
+#endif /* DEVELOPMENT || DEBUG */
 
 static void
 c_current_seg_filled(c_segment_t c_seg, c_segment_t *current_chead)
 {
 
 static void
 c_current_seg_filled(c_segment_t c_seg, c_segment_t *current_chead)
 {
-       uint32_t        unused_bytes;
-       uint32_t        offset_to_depopulate;
-       int             new_state = C_ON_AGE_Q;
-       clock_sec_t     sec;
-       clock_nsec_t    nsec;
+       uint32_t        unused_bytes;
+       uint32_t        offset_to_depopulate;
+       int             new_state = C_ON_AGE_Q;
+       clock_sec_t     sec;
+       clock_nsec_t    nsec;
+       boolean_t       head_insert = FALSE;
 
        unused_bytes = trunc_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset - c_seg->c_nextoffset));
 
        if (unused_bytes) {
 
        unused_bytes = trunc_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset - c_seg->c_nextoffset));
 
        if (unused_bytes) {
-
                offset_to_depopulate = C_SEG_BYTES_TO_OFFSET(round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_nextoffset)));
 
                /*
                offset_to_depopulate = C_SEG_BYTES_TO_OFFSET(round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_nextoffset)));
 
                /*
@@ -2609,10 +3460,11 @@ c_current_seg_filled(c_segment_t c_seg, c_segment_t *current_chead)
                lck_mtx_unlock_always(&c_seg->c_lock);
 
                kernel_memory_depopulate(
                lck_mtx_unlock_always(&c_seg->c_lock);
 
                kernel_memory_depopulate(
-                       kernel_map,
+                       compressor_map,
                        (vm_offset_t) &c_seg->c_store.c_buffer[offset_to_depopulate],
                        unused_bytes,
                        (vm_offset_t) &c_seg->c_store.c_buffer[offset_to_depopulate],
                        unused_bytes,
-                       KMA_COMPRESSOR);
+                       KMA_COMPRESSOR,
+                       VM_KERN_MEMORY_COMPRESSOR);
 
                lck_mtx_lock_spin_always(&c_seg->c_lock);
 
 
                lck_mtx_lock_spin_always(&c_seg->c_lock);
 
@@ -2620,61 +3472,122 @@ c_current_seg_filled(c_segment_t c_seg, c_segment_t *current_chead)
        }
        assert(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset) <= C_SEG_BUFSIZE);
 
        }
        assert(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset) <= C_SEG_BUFSIZE);
 
+#if DEVELOPMENT || DEBUG
+       {
+               boolean_t       c_seg_was_busy = FALSE;
+
+               if (!c_seg->c_busy) {
+                       C_SEG_BUSY(c_seg);
+               } else {
+                       c_seg_was_busy = TRUE;
+               }
+
+               lck_mtx_unlock_always(&c_seg->c_lock);
+
+               C_SEG_WRITE_PROTECT(c_seg);
+
+               lck_mtx_lock_spin_always(&c_seg->c_lock);
+
+               if (c_seg_was_busy == FALSE) {
+                       C_SEG_WAKEUP_DONE(c_seg);
+               }
+       }
+#endif
+
 #if CONFIG_FREEZE
 #if CONFIG_FREEZE
-       if (current_chead == (c_segment_t*)&freezer_chead && DEFAULT_FREEZER_COMPRESSED_PAGER_IS_SWAPBACKED &&
-           c_freezer_swapout_count < VM_MAX_FREEZER_CSEG_SWAP_COUNT) {
+       if (current_chead == (c_segment_t*) &(freezer_context_global.freezer_ctx_chead) &&
+           VM_CONFIG_SWAP_IS_PRESENT &&
+           VM_CONFIG_FREEZER_SWAP_IS_ACTIVE
+#if DEVELOPMENT || DEBUG
+           && !memorystatus_freeze_to_memory
+#endif /* DEVELOPMENT || DEBUG */
+           ) {
                new_state = C_ON_SWAPOUT_Q;
        }
 #endif /* CONFIG_FREEZE */
 
                new_state = C_ON_SWAPOUT_Q;
        }
 #endif /* CONFIG_FREEZE */
 
+       if (vm_darkwake_mode == TRUE) {
+               new_state = C_ON_SWAPOUT_Q;
+               head_insert = TRUE;
+       }
+
        clock_get_system_nanotime(&sec, &nsec);
        c_seg->c_creation_ts = (uint32_t)sec;
 
        lck_mtx_lock_spin_always(c_list_lock);
 
        clock_get_system_nanotime(&sec, &nsec);
        c_seg->c_creation_ts = (uint32_t)sec;
 
        lck_mtx_lock_spin_always(c_list_lock);
 
+       c_seg->c_generation_id = c_generation_id++;
+       c_seg_switch_state(c_seg, new_state, head_insert);
+
 #if CONFIG_FREEZE
 #if CONFIG_FREEZE
-       if (c_seg->c_state == C_ON_SWAPOUT_Q)
-               c_freezer_swapout_count++;
+       if (c_seg->c_state == C_ON_SWAPOUT_Q) {
+               /*
+                * darkwake and freezer can't co-exist together
+                * We'll need to fix this accounting as a start.
+                */
+               assert(vm_darkwake_mode == FALSE);
+               c_seg_update_task_owner(c_seg, freezer_context_global.freezer_ctx_task);
+               freezer_context_global.freezer_ctx_swapped_bytes += c_seg->c_bytes_used;
+       }
 #endif /* CONFIG_FREEZE */
 
 #endif /* CONFIG_FREEZE */
 
-       c_seg->c_generation_id = c_generation_id++;
-       c_seg_switch_state(c_seg, new_state, FALSE);
+       if (c_seg->c_state == C_ON_AGE_Q && C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE) {
+#if CONFIG_FREEZE
+               assert(c_seg->c_task_owner == NULL);
+#endif /* CONFIG_FREEZE */
+               c_seg_need_delayed_compaction(c_seg, TRUE);
+       }
 
        lck_mtx_unlock_always(c_list_lock);
 
 
        lck_mtx_unlock_always(c_list_lock);
 
-#if CONFIG_FREEZE
-       if (c_seg->c_state == C_ON_SWAPOUT_Q)
+       if (c_seg->c_state == C_ON_SWAPOUT_Q) {
+               /*
+                * Darkwake and Freeze configs always
+                * wake up the swapout thread because
+                * the compactor thread that normally handles
+                * it may not be running as much in these
+                * configs.
+                */
                thread_wakeup((event_t)&c_swapout_list_head);
                thread_wakeup((event_t)&c_swapout_list_head);
-#endif /* CONFIG_FREEZE */
-
-       if (c_seg->c_state == C_ON_AGE_Q && C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE)
-               c_seg_need_delayed_compaction(c_seg);
+       }
 
        *current_chead = NULL;
 }
 
 
        *current_chead = NULL;
 }
 
+
 /*
  * returns with c_seg locked
  */
 void
 /*
  * returns with c_seg locked
  */
 void
-c_seg_swapin_requeue(c_segment_t c_seg, boolean_t has_data)
+c_seg_swapin_requeue(c_segment_t c_seg, boolean_t has_data, boolean_t minor_compact_ok, boolean_t age_on_swapin_q)
 {
 {
-        clock_sec_t    sec;
-        clock_nsec_t   nsec;
+       clock_sec_t     sec;
+       clock_nsec_t    nsec;
 
        clock_get_system_nanotime(&sec, &nsec);
 
        lck_mtx_lock_spin_always(c_list_lock);
        lck_mtx_lock_spin_always(&c_seg->c_lock);
 
 
        clock_get_system_nanotime(&sec, &nsec);
 
        lck_mtx_lock_spin_always(c_list_lock);
        lck_mtx_lock_spin_always(&c_seg->c_lock);
 
+       assert(c_seg->c_busy_swapping);
+       assert(c_seg->c_busy);
+
        c_seg->c_busy_swapping = 0;
 
        if (c_seg->c_overage_swap == TRUE) {
                c_overage_swapped_count--;
                c_seg->c_overage_swap = FALSE;
        c_seg->c_busy_swapping = 0;
 
        if (c_seg->c_overage_swap == TRUE) {
                c_overage_swapped_count--;
                c_seg->c_overage_swap = FALSE;
-       }       
+       }
        if (has_data == TRUE) {
        if (has_data == TRUE) {
-               c_seg_switch_state(c_seg, C_ON_SWAPPEDIN_Q, FALSE);
+               if (age_on_swapin_q == TRUE) {
+                       c_seg_switch_state(c_seg, C_ON_SWAPPEDIN_Q, FALSE);
+               } else {
+                       c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE);
+               }
+
+               if (minor_compact_ok == TRUE && !c_seg->c_on_minorcompact_q && C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE) {
+                       c_seg_need_delayed_compaction(c_seg, TRUE);
+               }
        } else {
                c_seg->c_store.c_buffer = (int32_t*) NULL;
                c_seg->c_populated_offset = C_SEG_BYTES_TO_OFFSET(0);
        } else {
                c_seg->c_store.c_buffer = (int32_t*) NULL;
                c_seg->c_populated_offset = C_SEG_BYTES_TO_OFFSET(0);
@@ -2689,19 +3602,20 @@ c_seg_swapin_requeue(c_segment_t c_seg, boolean_t has_data)
 
 
 /*
 
 
 /*
- * c_seg has to be locked and is returned locked.
+ * c_seg has to be locked and is returned locked if the c_seg isn't freed
  * PAGE_REPLACMENT_DISALLOWED has to be TRUE on entry and is returned TRUE
  * PAGE_REPLACMENT_DISALLOWED has to be TRUE on entry and is returned TRUE
+ * c_seg_swapin returns 1 if the c_seg was freed, 0 otherwise
  */
 
  */
 
-void
-c_seg_swapin(c_segment_t c_seg, boolean_t force_minor_compaction)
+int
+c_seg_swapin(c_segment_t c_seg, boolean_t force_minor_compaction, boolean_t age_on_swapin_q)
 {
 {
-       vm_offset_t     addr = 0;
-       uint32_t        io_size = 0;
-       uint64_t        f_offset;
+       vm_offset_t     addr = 0;
+       uint32_t        io_size = 0;
+       uint64_t        f_offset;
 
        assert(C_SEG_IS_ONDISK(c_seg));
 
        assert(C_SEG_IS_ONDISK(c_seg));
-       
+
 #if !CHECKSUM_THE_SWAP
        c_seg_trim_tail(c_seg);
 #endif
 #if !CHECKSUM_THE_SWAP
        c_seg_trim_tail(c_seg);
 #endif
@@ -2710,47 +3624,97 @@ c_seg_swapin(c_segment_t c_seg, boolean_t force_minor_compaction)
 
        C_SEG_BUSY(c_seg);
        c_seg->c_busy_swapping = 1;
 
        C_SEG_BUSY(c_seg);
        c_seg->c_busy_swapping = 1;
+
+       /*
+        * This thread is likely going to block for I/O.
+        * Make sure it is ready to run when the I/O completes because
+        * it needs to clear the busy bit on the c_seg so that other
+        * waiting threads can make progress too. To do that, boost
+        * the rwlock_count so that the priority is boosted.
+        */
+       set_thread_rwlock_boost();
        lck_mtx_unlock_always(&c_seg->c_lock);
 
        PAGE_REPLACEMENT_DISALLOWED(FALSE);
 
        addr = (vm_offset_t)C_SEG_BUFFER_ADDRESS(c_seg->c_mysegno);
        lck_mtx_unlock_always(&c_seg->c_lock);
 
        PAGE_REPLACEMENT_DISALLOWED(FALSE);
 
        addr = (vm_offset_t)C_SEG_BUFFER_ADDRESS(c_seg->c_mysegno);
+       c_seg->c_store.c_buffer = (int32_t*) addr;
 
 
-       kernel_memory_populate(kernel_map, addr, io_size, KMA_COMPRESSOR, VM_KERN_MEMORY_COMPRESSOR);
+       kernel_memory_populate(compressor_map, addr, io_size, KMA_COMPRESSOR, VM_KERN_MEMORY_COMPRESSOR);
 
 
-       if (vm_swap_get(addr, f_offset, io_size) != KERN_SUCCESS) {
+       if (vm_swap_get(c_seg, f_offset, io_size) != KERN_SUCCESS) {
                PAGE_REPLACEMENT_DISALLOWED(TRUE);
 
                PAGE_REPLACEMENT_DISALLOWED(TRUE);
 
-               kernel_memory_depopulate(kernel_map, addr, io_size, KMA_COMPRESSOR);
+               kernel_memory_depopulate(compressor_map, addr, io_size, KMA_COMPRESSOR, VM_KERN_MEMORY_COMPRESSOR);
 
 
-               c_seg_swapin_requeue(c_seg, FALSE);
+               c_seg_swapin_requeue(c_seg, FALSE, TRUE, age_on_swapin_q);
        } else {
        } else {
-               c_seg->c_store.c_buffer = (int32_t*) addr;
 #if ENCRYPTED_SWAP
                vm_swap_decrypt(c_seg);
 #endif /* ENCRYPTED_SWAP */
 
 #if CHECKSUM_THE_SWAP
 #if ENCRYPTED_SWAP
                vm_swap_decrypt(c_seg);
 #endif /* ENCRYPTED_SWAP */
 
 #if CHECKSUM_THE_SWAP
-               if (c_seg->cseg_swap_size != io_size)
+               if (c_seg->cseg_swap_size != io_size) {
                        panic("swapin size doesn't match swapout size");
                        panic("swapin size doesn't match swapout size");
+               }
 
 
-               if (c_seg->cseg_hash != hash_string((char*) c_seg->c_store.c_buffer, (int)io_size)) {
+               if (c_seg->cseg_hash != vmc_hash((char*) c_seg->c_store.c_buffer, (int)io_size)) {
                        panic("c_seg_swapin - Swap hash mismatch\n");
                }
 #endif /* CHECKSUM_THE_SWAP */
 
                PAGE_REPLACEMENT_DISALLOWED(TRUE);
 
                        panic("c_seg_swapin - Swap hash mismatch\n");
                }
 #endif /* CHECKSUM_THE_SWAP */
 
                PAGE_REPLACEMENT_DISALLOWED(TRUE);
 
-               if (force_minor_compaction == TRUE) {
+               c_seg_swapin_requeue(c_seg, TRUE, force_minor_compaction == TRUE ? FALSE : TRUE, age_on_swapin_q);
+
+#if CONFIG_FREEZE
+               /*
+                * c_seg_swapin_requeue() returns with the c_seg lock held.
+                */
+               if (!lck_mtx_try_lock_spin_always(c_list_lock)) {
+                       assert(c_seg->c_busy);
+
+                       lck_mtx_unlock_always(&c_seg->c_lock);
+                       lck_mtx_lock_spin_always(c_list_lock);
                        lck_mtx_lock_spin_always(&c_seg->c_lock);
                        lck_mtx_lock_spin_always(&c_seg->c_lock);
-                       
-                       c_seg_minor_compaction_and_unlock(c_seg, FALSE);
                }
                }
+
+               if (c_seg->c_task_owner) {
+                       c_seg_update_task_owner(c_seg, NULL);
+               }
+
+               lck_mtx_unlock_always(c_list_lock);
+
+               OSAddAtomic(c_seg->c_slots_used, &c_segment_pages_compressed_incore);
+#endif /* CONFIG_FREEZE */
+
                OSAddAtomic64(c_seg->c_bytes_used, &compressor_bytes_used);
 
                OSAddAtomic64(c_seg->c_bytes_used, &compressor_bytes_used);
 
-               c_seg_swapin_requeue(c_seg, TRUE);
+               if (force_minor_compaction == TRUE) {
+                       if (c_seg_minor_compaction_and_unlock(c_seg, FALSE)) {
+                               /*
+                                * c_seg was completely empty so it was freed,
+                                * so be careful not to reference it again
+                                *
+                                * Drop the rwlock_count so that the thread priority
+                                * is returned back to where it is supposed to be.
+                                */
+                               clear_thread_rwlock_boost();
+                               return 1;
+                       }
+
+                       lck_mtx_lock_spin_always(&c_seg->c_lock);
+               }
        }
        C_SEG_WAKEUP_DONE(c_seg);
        }
        C_SEG_WAKEUP_DONE(c_seg);
+
+       /*
+        * Drop the rwlock_count so that the thread priority
+        * is returned back to where it is supposed to be.
+        */
+       clear_thread_rwlock_boost();
+
+       return 0;
 }
 
 
 }
 
 
@@ -2760,15 +3724,15 @@ c_segment_sv_hash_drop_ref(int hash_indx)
        struct c_sv_hash_entry o_sv_he, n_sv_he;
 
        while (1) {
        struct c_sv_hash_entry o_sv_he, n_sv_he;
 
        while (1) {
-
                o_sv_he.he_record = c_segment_sv_hash_table[hash_indx].he_record;
 
                n_sv_he.he_ref = o_sv_he.he_ref - 1;
                n_sv_he.he_data = o_sv_he.he_data;
 
                if (OSCompareAndSwap64((UInt64)o_sv_he.he_record, (UInt64)n_sv_he.he_record, (UInt64 *) &c_segment_sv_hash_table[hash_indx].he_record) == TRUE) {
                o_sv_he.he_record = c_segment_sv_hash_table[hash_indx].he_record;
 
                n_sv_he.he_ref = o_sv_he.he_ref - 1;
                n_sv_he.he_data = o_sv_he.he_data;
 
                if (OSCompareAndSwap64((UInt64)o_sv_he.he_record, (UInt64)n_sv_he.he_record, (UInt64 *) &c_segment_sv_hash_table[hash_indx].he_record) == TRUE) {
-                       if (n_sv_he.he_ref == 0)
+                       if (n_sv_he.he_ref == 0) {
                                OSAddAtomic(-1, &c_segment_svp_in_hash);
                                OSAddAtomic(-1, &c_segment_svp_in_hash);
+                       }
                        break;
                }
        }
                        break;
                }
        }
@@ -2778,20 +3742,20 @@ c_segment_sv_hash_drop_ref(int hash_indx)
 static int
 c_segment_sv_hash_insert(uint32_t data)
 {
 static int
 c_segment_sv_hash_insert(uint32_t data)
 {
-       int             hash_sindx;
-       int             misses;
+       int             hash_sindx;
+       int             misses;
        struct c_sv_hash_entry o_sv_he, n_sv_he;
        struct c_sv_hash_entry o_sv_he, n_sv_he;
-       boolean_t       got_ref = FALSE;
+       boolean_t       got_ref = FALSE;
 
 
-       if (data == 0)
+       if (data == 0) {
                OSAddAtomic(1, &c_segment_svp_zero_compressions);
                OSAddAtomic(1, &c_segment_svp_zero_compressions);
-       else
+       } else {
                OSAddAtomic(1, &c_segment_svp_nonzero_compressions);
                OSAddAtomic(1, &c_segment_svp_nonzero_compressions);
+       }
 
        hash_sindx = data & C_SV_HASH_MASK;
 
        hash_sindx = data & C_SV_HASH_MASK;
-       
-       for (misses = 0; misses < C_SV_HASH_MAX_MISS; misses++)
-       {
+
+       for (misses = 0; misses < C_SV_HASH_MAX_MISS; misses++) {
                o_sv_he.he_record = c_segment_sv_hash_table[hash_sindx].he_record;
 
                while (o_sv_he.he_data == data || o_sv_he.he_ref == 0) {
                o_sv_he.he_record = c_segment_sv_hash_table[hash_sindx].he_record;
 
                while (o_sv_he.he_data == data || o_sv_he.he_ref == 0) {
@@ -2799,24 +3763,28 @@ c_segment_sv_hash_insert(uint32_t data)
                        n_sv_he.he_data = data;
 
                        if (OSCompareAndSwap64((UInt64)o_sv_he.he_record, (UInt64)n_sv_he.he_record, (UInt64 *) &c_segment_sv_hash_table[hash_sindx].he_record) == TRUE) {
                        n_sv_he.he_data = data;
 
                        if (OSCompareAndSwap64((UInt64)o_sv_he.he_record, (UInt64)n_sv_he.he_record, (UInt64 *) &c_segment_sv_hash_table[hash_sindx].he_record) == TRUE) {
-                               if (n_sv_he.he_ref == 1)
+                               if (n_sv_he.he_ref == 1) {
                                        OSAddAtomic(1, &c_segment_svp_in_hash);
                                        OSAddAtomic(1, &c_segment_svp_in_hash);
+                               }
                                got_ref = TRUE;
                                break;
                        }
                        o_sv_he.he_record = c_segment_sv_hash_table[hash_sindx].he_record;
                }
                                got_ref = TRUE;
                                break;
                        }
                        o_sv_he.he_record = c_segment_sv_hash_table[hash_sindx].he_record;
                }
-               if (got_ref == TRUE)
+               if (got_ref == TRUE) {
                        break;
                        break;
+               }
                hash_sindx++;
 
                hash_sindx++;
 
-               if (hash_sindx == C_SV_HASH_SIZE)
+               if (hash_sindx == C_SV_HASH_SIZE) {
                        hash_sindx = 0;
                        hash_sindx = 0;
+               }
+       }
+       if (got_ref == FALSE) {
+               return -1;
        }
        }
-       if (got_ref == FALSE)
-               return(-1);
 
 
-       return (hash_sindx);
+       return hash_sindx;
 }
 
 
 }
 
 
@@ -2825,8 +3793,9 @@ c_segment_sv_hash_insert(uint32_t data)
 static void
 c_compressed_record_data(char *src, int c_size)
 {
 static void
 c_compressed_record_data(char *src, int c_size)
 {
-       if ((c_compressed_record_cptr + c_size + 4) >= c_compressed_record_ebuf)
+       if ((c_compressed_record_cptr + c_size + 4) >= c_compressed_record_ebuf) {
                panic("c_compressed_record_cptr >= c_compressed_record_ebuf");
                panic("c_compressed_record_cptr >= c_compressed_record_ebuf");
+       }
 
        *(int *)((void *)c_compressed_record_cptr) = c_size;
 
 
        *(int *)((void *)c_compressed_record_cptr) = c_size;
 
@@ -2841,16 +3810,17 @@ c_compressed_record_data(char *src, int c_size)
 static int
 c_compress_page(char *src, c_slot_mapping_t slot_ptr, c_segment_t *current_chead, char *scratch_buf)
 {
 static int
 c_compress_page(char *src, c_slot_mapping_t slot_ptr, c_segment_t *current_chead, char *scratch_buf)
 {
-       int             c_size;
-       int             c_rounded_size = 0;
-       int             max_csize;
-       c_slot_t        cs;
-       c_segment_t     c_seg;
+       int             c_size;
+       int             c_rounded_size = 0;
+       int             max_csize;
+       c_slot_t        cs;
+       c_segment_t     c_seg;
 
        KERNEL_DEBUG(0xe0400000 | DBG_FUNC_START, *current_chead, 0, 0, 0, 0);
 retry:
 
        KERNEL_DEBUG(0xe0400000 | DBG_FUNC_START, *current_chead, 0, 0, 0, 0);
 retry:
-       if ((c_seg = c_seg_allocate(current_chead)) == NULL)
-               return (1);
+       if ((c_seg = c_seg_allocate(current_chead)) == NULL) {
+               return 1;
+       }
        /*
         * returns with c_seg lock held
         * and PAGE_REPLACEMENT_DISALLOWED(TRUE)...
        /*
         * returns with c_seg lock held
         * and PAGE_REPLACEMENT_DISALLOWED(TRUE)...
@@ -2861,43 +3831,93 @@ retry:
 
        cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_seg->c_nextslot);
 
 
        cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_seg->c_nextslot);
 
+       C_SLOT_ASSERT_PACKABLE(slot_ptr);
        cs->c_packed_ptr = C_SLOT_PACK_PTR(slot_ptr);
        cs->c_packed_ptr = C_SLOT_PACK_PTR(slot_ptr);
-       assert(slot_ptr == (c_slot_mapping_t)C_SLOT_UNPACK_PTR(cs));
 
        cs->c_offset = c_seg->c_nextoffset;
 
        max_csize = C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES((int32_t)cs->c_offset);
 
 
        cs->c_offset = c_seg->c_nextoffset;
 
        max_csize = C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES((int32_t)cs->c_offset);
 
-       if (max_csize > PAGE_SIZE)
+       if (max_csize > PAGE_SIZE) {
                max_csize = PAGE_SIZE;
                max_csize = PAGE_SIZE;
+       }
 
 #if CHECKSUM_THE_DATA
 
 #if CHECKSUM_THE_DATA
-       cs->c_hash_data = hash_string(src, PAGE_SIZE);
+       cs->c_hash_data = vmc_hash(src, PAGE_SIZE);
+#endif
+       boolean_t incomp_copy = FALSE;
+       int max_csize_adj = (max_csize - 4);
+
+       if (vm_compressor_algorithm() != VM_COMPRESSOR_DEFAULT_CODEC) {
+#if defined(__arm__) || defined(__arm64__)
+               uint16_t ccodec = CINVALID;
+               uint32_t inline_popcount;
+               if (max_csize >= C_SEG_OFFSET_ALIGNMENT_BOUNDARY) {
+                       c_size = metacompressor((const uint8_t *) src,
+                           (uint8_t *) &c_seg->c_store.c_buffer[cs->c_offset],
+                           max_csize_adj, &ccodec,
+                           scratch_buf, &incomp_copy, &inline_popcount);
+#if __ARM_WKDM_POPCNT__
+                       cs->c_inline_popcount = inline_popcount;
+#else
+                       assert(inline_popcount == C_SLOT_NO_POPCOUNT);
 #endif
 
 #endif
 
-       c_size = WKdm_compress_new((const WK_word *)(uintptr_t)src, (WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
-                                 (WK_word *)(uintptr_t)scratch_buf, max_csize - 4);
-       assert(c_size <= (max_csize - 4) && c_size >= -1);
+#if C_SEG_OFFSET_ALIGNMENT_BOUNDARY > 4
+                       if (c_size > max_csize_adj) {
+                               c_size = -1;
+                       }
+#endif
+               } else {
+                       c_size = -1;
+               }
+               assert(ccodec == CCWK || ccodec == CCLZ4);
+               cs->c_codec = ccodec;
+#endif
+       } else {
+#if defined(__arm__) || defined(__arm64__)
+               cs->c_codec = CCWK;
+#endif
+#if defined(__arm64__)
+               __unreachable_ok_push
+               if (PAGE_SIZE == 4096) {
+                       c_size = WKdm_compress_4k((WK_word *)(uintptr_t)src, (WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
+                           (WK_word *)(uintptr_t)scratch_buf, max_csize_adj);
+               } else {
+                       c_size = WKdm_compress_16k((WK_word *)(uintptr_t)src, (WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
+                           (WK_word *)(uintptr_t)scratch_buf, max_csize_adj);
+               }
+               __unreachable_ok_pop
+#else
+               c_size = WKdm_compress_new((const WK_word *)(uintptr_t)src, (WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
+                   (WK_word *)(uintptr_t)scratch_buf, max_csize_adj);
+#endif
+       }
+       assertf(((c_size <= max_csize_adj) && (c_size >= -1)),
+           "c_size invalid (%d, %d), cur compressions: %d", c_size, max_csize_adj, c_segment_pages_compressed);
 
        if (c_size == -1) {
 
        if (c_size == -1) {
-
                if (max_csize < PAGE_SIZE) {
                        c_current_seg_filled(c_seg, current_chead);
                        assert(*current_chead == NULL);
 
                        lck_mtx_unlock_always(&c_seg->c_lock);
                if (max_csize < PAGE_SIZE) {
                        c_current_seg_filled(c_seg, current_chead);
                        assert(*current_chead == NULL);
 
                        lck_mtx_unlock_always(&c_seg->c_lock);
-
+                       /* TODO: it may be worth requiring codecs to distinguish
+                        * between incompressible inputs and failures due to
+                        * budget exhaustion.
+                        */
                        PAGE_REPLACEMENT_DISALLOWED(FALSE);
                        goto retry;
                }
                c_size = PAGE_SIZE;
 
                        PAGE_REPLACEMENT_DISALLOWED(FALSE);
                        goto retry;
                }
                c_size = PAGE_SIZE;
 
-               memcpy(&c_seg->c_store.c_buffer[cs->c_offset], src, c_size);
+               if (incomp_copy == FALSE) {
+                       memcpy(&c_seg->c_store.c_buffer[cs->c_offset], src, c_size);
+               }
 
                OSAddAtomic(1, &c_segment_noncompressible_pages);
 
                OSAddAtomic(1, &c_segment_noncompressible_pages);
-
        } else if (c_size == 0) {
        } else if (c_size == 0) {
-               int             hash_index;
+               int             hash_index;
 
                /*
                 * special case - this is a page completely full of a single 32 bit value
 
                /*
                 * special case - this is a page completely full of a single 32 bit value
@@ -2915,7 +3935,7 @@ retry:
                        goto sv_compression;
                }
                c_size = 4;
                        goto sv_compression;
                }
                c_size = 4;
-               
+
                memcpy(&c_seg->c_store.c_buffer[cs->c_offset], src, c_size);
 
                OSAddAtomic(1, &c_segment_svp_hash_failed);
                memcpy(&c_seg->c_store.c_buffer[cs->c_offset], src, c_size);
 
                OSAddAtomic(1, &c_segment_svp_hash_failed);
@@ -2924,19 +3944,22 @@ retry:
 #if RECORD_THE_COMPRESSED_DATA
        c_compressed_record_data((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size);
 #endif
 #if RECORD_THE_COMPRESSED_DATA
        c_compressed_record_data((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size);
 #endif
-
 #if CHECKSUM_THE_COMPRESSED_DATA
 #if CHECKSUM_THE_COMPRESSED_DATA
-       cs->c_hash_compressed_data = hash_string((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size);
+       cs->c_hash_compressed_data = vmc_hash((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size);
+#endif
+#if POPCOUNT_THE_COMPRESSED_DATA
+       cs->c_pop_cdata = vmc_pop((uintptr_t) &c_seg->c_store.c_buffer[cs->c_offset], c_size);
 #endif
        c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
 
        PACK_C_SIZE(cs, c_size);
        c_seg->c_bytes_used += c_rounded_size;
        c_seg->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
 #endif
        c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
 
        PACK_C_SIZE(cs, c_size);
        c_seg->c_bytes_used += c_rounded_size;
        c_seg->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
+       c_seg->c_slots_used++;
 
        slot_ptr->s_cindx = c_seg->c_nextslot++;
        /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
 
        slot_ptr->s_cindx = c_seg->c_nextslot++;
        /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
-       slot_ptr->s_cseg = c_seg->c_mysegno + 1; 
+       slot_ptr->s_cseg = c_seg->c_mysegno + 1;
 
 sv_compression:
        if (c_seg->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg->c_nextslot >= C_SLOT_MAX_INDEX) {
 
 sv_compression:
        if (c_seg->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg->c_nextslot >= C_SLOT_MAX_INDEX) {
@@ -2960,48 +3983,99 @@ sv_compression:
        OSAddAtomic64(PAGE_SIZE, &c_segment_input_bytes);
 
        OSAddAtomic(1, &c_segment_pages_compressed);
        OSAddAtomic64(PAGE_SIZE, &c_segment_input_bytes);
 
        OSAddAtomic(1, &c_segment_pages_compressed);
+#if CONFIG_FREEZE
+       OSAddAtomic(1, &c_segment_pages_compressed_incore);
+#endif /* CONFIG_FREEZE */
        OSAddAtomic(1, &sample_period_compression_count);
 
        KERNEL_DEBUG(0xe0400000 | DBG_FUNC_END, *current_chead, c_size, c_segment_input_bytes, c_segment_compressed_bytes, 0);
 
        OSAddAtomic(1, &sample_period_compression_count);
 
        KERNEL_DEBUG(0xe0400000 | DBG_FUNC_END, *current_chead, c_size, c_segment_input_bytes, c_segment_compressed_bytes, 0);
 
-       return (0);
+       return 0;
 }
 
 }
 
+static inline void
+sv_decompress(int32_t *ddst, int32_t pattern)
+{
+//     assert(__builtin_constant_p(PAGE_SIZE) != 0);
+#if defined(__x86_64__)
+       memset_word(ddst, pattern, PAGE_SIZE / sizeof(int32_t));
+#elif defined(__arm64__)
+       assert((PAGE_SIZE % 128) == 0);
+       if (pattern == 0) {
+               fill32_dczva((addr64_t)ddst, PAGE_SIZE);
+       } else {
+               fill32_nt((addr64_t)ddst, PAGE_SIZE, pattern);
+       }
+#else
+       size_t          i;
+
+       /* Unroll the pattern fill loop 4x to encourage the
+        * compiler to emit NEON stores, cf.
+        * <rdar://problem/25839866> Loop autovectorization
+        * anomalies.
+        */
+       /* * We use separate loops for each PAGE_SIZE
+        * to allow the autovectorizer to engage, as PAGE_SIZE
+        * may not be a constant.
+        */
+
+       __unreachable_ok_push
+       if (PAGE_SIZE == 4096) {
+               for (i = 0; i < (4096U / sizeof(int32_t)); i += 4) {
+                       *ddst++ = pattern;
+                       *ddst++ = pattern;
+                       *ddst++ = pattern;
+                       *ddst++ = pattern;
+               }
+       } else {
+               assert(PAGE_SIZE == 16384);
+               for (i = 0; i < (int)(16384U / sizeof(int32_t)); i += 4) {
+                       *ddst++ = pattern;
+                       *ddst++ = pattern;
+                       *ddst++ = pattern;
+                       *ddst++ = pattern;
+               }
+       }
+       __unreachable_ok_pop
+#endif
+}
 
 static int
 c_decompress_page(char *dst, volatile c_slot_mapping_t slot_ptr, int flags, int *zeroslot)
 {
 
 static int
 c_decompress_page(char *dst, volatile c_slot_mapping_t slot_ptr, int flags, int *zeroslot)
 {
-       c_slot_t        cs;
-       c_segment_t     c_seg;
-       int             c_indx;
-       int             c_rounded_size;
-       uint32_t        c_size;
-       int             retval = 0;
-       boolean_t       need_unlock = TRUE;
-       boolean_t       consider_defragmenting = FALSE;
-       boolean_t       kdp_mode = FALSE;
-
-       if (flags & C_KDP) {
+       c_slot_t        cs;
+       c_segment_t     c_seg;
+       uint32_t        c_segno;
+       uint16_t        c_indx;
+       int             c_rounded_size;
+       uint32_t        c_size;
+       int             retval = 0;
+       boolean_t       need_unlock = TRUE;
+       boolean_t       consider_defragmenting = FALSE;
+       boolean_t       kdp_mode = FALSE;
+
+       if (__improbable(flags & C_KDP)) {
                if (not_in_kdp) {
                        panic("C_KDP passed to decompress page from outside of debugger context");
                }
 
                if (not_in_kdp) {
                        panic("C_KDP passed to decompress page from outside of debugger context");
                }
 
-               assert((flags & C_KEEP) ==  C_KEEP);
+               assert((flags & C_KEEP) == C_KEEP);
                assert((flags & C_DONT_BLOCK) == C_DONT_BLOCK);
 
                if ((flags & (C_DONT_BLOCK | C_KEEP)) != (C_DONT_BLOCK | C_KEEP)) {
                assert((flags & C_DONT_BLOCK) == C_DONT_BLOCK);
 
                if ((flags & (C_DONT_BLOCK | C_KEEP)) != (C_DONT_BLOCK | C_KEEP)) {
-                       return (-2);
+                       return -2;
                }
 
                kdp_mode = TRUE;
                }
 
                kdp_mode = TRUE;
+               *zeroslot = 0;
        }
 
 ReTry:
        }
 
 ReTry:
-       if (!kdp_mode) {
+       if (__probable(!kdp_mode)) {
                PAGE_REPLACEMENT_DISALLOWED(TRUE);
        } else {
                if (kdp_lck_rw_lock_is_acquired_exclusive(&c_master_lock)) {
                PAGE_REPLACEMENT_DISALLOWED(TRUE);
        } else {
                if (kdp_lck_rw_lock_is_acquired_exclusive(&c_master_lock)) {
-                       return (-2);
+                       return -2;
                }
        }
 
                }
        }
 
@@ -3015,15 +4089,14 @@ ReTry:
         * to disk... in this state we allow freeing of compressed
         * pages and must honor the C_DONT_BLOCK case
         */
         * to disk... in this state we allow freeing of compressed
         * pages and must honor the C_DONT_BLOCK case
         */
-       if (dst && decompressions_blocked == TRUE) {
+       if (__improbable(dst && decompressions_blocked == TRUE)) {
                if (flags & C_DONT_BLOCK) {
                if (flags & C_DONT_BLOCK) {
-
-                       if (!kdp_mode) {
+                       if (__probable(!kdp_mode)) {
                                PAGE_REPLACEMENT_DISALLOWED(FALSE);
                        }
 
                        *zeroslot = 0;
                                PAGE_REPLACEMENT_DISALLOWED(FALSE);
                        }
 
                        *zeroslot = 0;
-                       return (-2);
+                       return -2;
                }
                /*
                 * it's safe to atomically assert and block behind the
                }
                /*
                 * it's safe to atomically assert and block behind the
@@ -3041,18 +4114,35 @@ ReTry:
        }
 #endif
        /* s_cseg is actually "segno+1" */
        }
 #endif
        /* s_cseg is actually "segno+1" */
-       c_seg = c_segments[slot_ptr->s_cseg - 1].c_seg;
+       c_segno = slot_ptr->s_cseg - 1;
+
+       if (__improbable(c_segno >= c_segments_available)) {
+               panic("c_decompress_page: c_segno %d >= c_segments_available %d, slot_ptr(%p), slot_data(%x)",
+                   c_segno, c_segments_available, slot_ptr, *(int *)((void *)slot_ptr));
+       }
 
 
-       if (!kdp_mode) {
+       if (__improbable(c_segments[c_segno].c_segno < c_segments_available)) {
+               panic("c_decompress_page: c_segno %d is free, slot_ptr(%p), slot_data(%x)",
+                   c_segno, slot_ptr, *(int *)((void *)slot_ptr));
+       }
+
+       c_seg = c_segments[c_segno].c_seg;
+
+       if (__probable(!kdp_mode)) {
                lck_mtx_lock_spin_always(&c_seg->c_lock);
        } else {
                if (kdp_lck_mtx_lock_spin_is_acquired(&c_seg->c_lock)) {
                lck_mtx_lock_spin_always(&c_seg->c_lock);
        } else {
                if (kdp_lck_mtx_lock_spin_is_acquired(&c_seg->c_lock)) {
-                       return (-2);
+                       return -2;
                }
        }
 
        assert(c_seg->c_state != C_IS_EMPTY && c_seg->c_state != C_IS_FREE);
 
                }
        }
 
        assert(c_seg->c_state != C_IS_EMPTY && c_seg->c_state != C_IS_FREE);
 
+       if (dst == NULL && c_seg->c_busy_swapping) {
+               assert(c_seg->c_busy);
+
+               goto bypass_busy_check;
+       }
        if (flags & C_DONT_BLOCK) {
                if (c_seg->c_busy || (C_SEG_IS_ONDISK(c_seg) && dst)) {
                        *zeroslot = 0;
        if (flags & C_DONT_BLOCK) {
                if (c_seg->c_busy || (C_SEG_IS_ONDISK(c_seg) && dst)) {
                        *zeroslot = 0;
@@ -3062,41 +4152,87 @@ ReTry:
                }
        }
        if (c_seg->c_busy) {
                }
        }
        if (c_seg->c_busy) {
-
                PAGE_REPLACEMENT_DISALLOWED(FALSE);
 
                c_seg_wait_on_busy(c_seg);
 
                goto ReTry;
        }
                PAGE_REPLACEMENT_DISALLOWED(FALSE);
 
                c_seg_wait_on_busy(c_seg);
 
                goto ReTry;
        }
+bypass_busy_check:
+
        c_indx = slot_ptr->s_cindx;
 
        c_indx = slot_ptr->s_cindx;
 
+       if (__improbable(c_indx >= c_seg->c_nextslot)) {
+               panic("c_decompress_page: c_indx %d >= c_nextslot %d, c_seg(%p), slot_ptr(%p), slot_data(%x)",
+                   c_indx, c_seg->c_nextslot, c_seg, slot_ptr, *(int *)((void *)slot_ptr));
+       }
+
        cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
 
        c_size = UNPACK_C_SIZE(cs);
 
        cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
 
        c_size = UNPACK_C_SIZE(cs);
 
+       if (__improbable(c_size == 0)) {
+               panic("c_decompress_page: c_size == 0, c_seg(%p), slot_ptr(%p), slot_data(%x)",
+                   c_seg, slot_ptr, *(int *)((void *)slot_ptr));
+       }
+
        c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
 
        if (dst) {
        c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
 
        if (dst) {
-               uint32_t        age_of_cseg;
-               clock_sec_t     cur_ts_sec;
-               clock_nsec_t    cur_ts_nsec;
+               uint32_t        age_of_cseg;
+               clock_sec_t     cur_ts_sec;
+               clock_nsec_t    cur_ts_nsec;
 
                if (C_SEG_IS_ONDISK(c_seg)) {
 
                if (C_SEG_IS_ONDISK(c_seg)) {
+#if CONFIG_FREEZE
+                       if (freezer_incore_cseg_acct) {
+                               if ((c_seg->c_slots_used + c_segment_pages_compressed_incore) >= c_segment_pages_compressed_nearing_limit) {
+                                       PAGE_REPLACEMENT_DISALLOWED(FALSE);
+                                       lck_mtx_unlock_always(&c_seg->c_lock);
+
+                                       memorystatus_kill_on_VM_compressor_space_shortage(FALSE /* async */);
+
+                                       goto ReTry;
+                               }
+
+                               uint32_t incore_seg_count = c_segment_count - c_swappedout_count - c_swappedout_sparse_count;
+                               if ((incore_seg_count + 1) >= c_segments_nearing_limit) {
+                                       PAGE_REPLACEMENT_DISALLOWED(FALSE);
+                                       lck_mtx_unlock_always(&c_seg->c_lock);
+
+                                       memorystatus_kill_on_VM_compressor_space_shortage(FALSE /* async */);
+
+                                       goto ReTry;
+                               }
+                       }
+#endif /* CONFIG_FREEZE */
                        assert(kdp_mode == FALSE);
                        assert(kdp_mode == FALSE);
-                       c_seg_swapin(c_seg, FALSE);
+                       retval = c_seg_swapin(c_seg, FALSE, TRUE);
+                       assert(retval == 0);
 
                        retval = 1;
 
                        retval = 1;
-               }               
+               }
                if (c_seg->c_state == C_ON_BAD_Q) {
                        assert(c_seg->c_store.c_buffer == NULL);
                if (c_seg->c_state == C_ON_BAD_Q) {
                        assert(c_seg->c_store.c_buffer == NULL);
+                       *zeroslot = 0;
 
                        retval = -1;
 
                        retval = -1;
-                       goto c_seg_invalid_data;
+                       goto done;
+               }
+
+#if POPCOUNT_THE_COMPRESSED_DATA
+               unsigned csvpop;
+               uintptr_t csvaddr = (uintptr_t) &c_seg->c_store.c_buffer[cs->c_offset];
+               if (cs->c_pop_cdata != (csvpop = vmc_pop(csvaddr, c_size))) {
+                       panic("Compressed data popcount doesn't match original, bit distance: %d %p (phys: %p) %p %p 0x%x 0x%x 0x%x 0x%x", (csvpop - cs->c_pop_cdata), (void *)csvaddr, (void *) kvtophys(csvaddr), c_seg, cs, cs->c_offset, c_size, csvpop, cs->c_pop_cdata);
                }
                }
+#endif
+
 #if CHECKSUM_THE_COMPRESSED_DATA
 #if CHECKSUM_THE_COMPRESSED_DATA
-               if (cs->c_hash_compressed_data != hash_string((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size))
-                       panic("compressed data doesn't match original");
+               unsigned csvhash;
+               if (cs->c_hash_compressed_data != (csvhash = vmc_hash((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size))) {
+                       panic("Compressed data doesn't match original %p %p %u %u %u", c_seg, cs, c_size, cs->c_hash_compressed_data, csvhash);
+               }
 #endif
                if (c_rounded_size == PAGE_SIZE) {
                        /*
 #endif
                if (c_rounded_size == PAGE_SIZE) {
                        /*
@@ -3104,8 +4240,8 @@ ReTry:
                         */
                        memcpy(dst, &c_seg->c_store.c_buffer[cs->c_offset], PAGE_SIZE);
                } else if (c_size == 4) {
                         */
                        memcpy(dst, &c_seg->c_store.c_buffer[cs->c_offset], PAGE_SIZE);
                } else if (c_size == 4) {
-                       int32_t         data;
-                       int32_t         *dptr;
+                       int32_t         data;
+                       int32_t         *dptr;
 
                        /*
                         * page was populated with a single value
 
                        /*
                         * page was populated with a single value
@@ -3115,21 +4251,12 @@ ReTry:
                         */
                        dptr = (int32_t *)(uintptr_t)dst;
                        data = *(int32_t *)(&c_seg->c_store.c_buffer[cs->c_offset]);
                         */
                        dptr = (int32_t *)(uintptr_t)dst;
                        data = *(int32_t *)(&c_seg->c_store.c_buffer[cs->c_offset]);
-#if __x86_64__
-                       memset_word(dptr, data, PAGE_SIZE / sizeof(int32_t));
-#else
-                       {
-                       int             i;
-
-                       for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++)
-                               *dptr++ = data;
-                       }
-#endif
+                       sv_decompress(dptr, data);
                } else {
                } else {
-                       uint32_t        my_cpu_no;
-                       char            *scratch_buf;
+                       uint32_t        my_cpu_no;
+                       char            *scratch_buf;
 
 
-                       if (!kdp_mode) {
+                       if (__probable(!kdp_mode)) {
                                /*
                                 * we're behind the c_seg lock held in spin mode
                                 * which means pre-emption is disabled... therefore
                                /*
                                 * we're behind the c_seg lock held in spin mode
                                 * which means pre-emption is disabled... therefore
@@ -3139,48 +4266,130 @@ ReTry:
 
                                assert(my_cpu_no < compressor_cpus);
 
 
                                assert(my_cpu_no < compressor_cpus);
 
-                               scratch_buf = &compressor_scratch_bufs[my_cpu_no * WKdm_SCRATCH_BUF_SIZE];
+                               scratch_buf = &compressor_scratch_bufs[my_cpu_no * vm_compressor_get_decode_scratch_size()];
                        } else {
                                scratch_buf = kdp_compressor_scratch_buf;
                        }
                        } else {
                                scratch_buf = kdp_compressor_scratch_buf;
                        }
-                       WKdm_decompress_new((WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
+
+                       if (vm_compressor_algorithm() != VM_COMPRESSOR_DEFAULT_CODEC) {
+#if defined(__arm__) || defined(__arm64__)
+                               uint16_t c_codec = cs->c_codec;
+                               uint32_t inline_popcount;
+                               if (!metadecompressor((const uint8_t *) &c_seg->c_store.c_buffer[cs->c_offset],
+                                   (uint8_t *)dst, c_size, c_codec, (void *)scratch_buf, &inline_popcount)) {
+                                       retval = -1;
+                               } else {
+#if __ARM_WKDM_POPCNT__
+                                       if (inline_popcount != cs->c_inline_popcount) {
+                                               /*
+                                                * The codec choice in compression and
+                                                * decompression must agree, so there
+                                                * should never be a disagreement in
+                                                * whether an inline population count
+                                                * was performed.
+                                                */
+                                               assert(inline_popcount != C_SLOT_NO_POPCOUNT);
+                                               assert(cs->c_inline_popcount != C_SLOT_NO_POPCOUNT);
+                                               printf("decompression failure from physical region %llx+%05x: popcount mismatch (%d != %d)\n",
+                                                   (unsigned long long)kvtophys((uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset]), c_size,
+                                                   inline_popcount,
+                                                   cs->c_inline_popcount);
+                                               retval = -1;
+                                       }
+#else
+                                       assert(inline_popcount == C_SLOT_NO_POPCOUNT);
+#endif /* __ARM_WKDM_POPCNT__ */
+                               }
+#endif
+                       } else {
+#if defined(__arm64__)
+                               __unreachable_ok_push
+                               if (PAGE_SIZE == 4096) {
+                                       WKdm_decompress_4k((WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
                                            (WK_word *)(uintptr_t)dst, (WK_word *)(uintptr_t)scratch_buf, c_size);
                                            (WK_word *)(uintptr_t)dst, (WK_word *)(uintptr_t)scratch_buf, c_size);
+                               } else {
+                                       WKdm_decompress_16k((WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
+                                           (WK_word *)(uintptr_t)dst, (WK_word *)(uintptr_t)scratch_buf, c_size);
+                               }
+                               __unreachable_ok_pop
+#else
+                               WKdm_decompress_new((WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
+                                   (WK_word *)(uintptr_t)dst, (WK_word *)(uintptr_t)scratch_buf, c_size);
+#endif
+                       }
                }
 
 #if CHECKSUM_THE_DATA
                }
 
 #if CHECKSUM_THE_DATA
-               if (cs->c_hash_data != hash_string(dst, PAGE_SIZE))
-                       panic("decompressed data doesn't match original");
+               if (cs->c_hash_data != vmc_hash(dst, PAGE_SIZE)) {
+#if     defined(__arm__) || defined(__arm64__)
+                       int32_t *dinput = &c_seg->c_store.c_buffer[cs->c_offset];
+                       panic("decompressed data doesn't match original cs: %p, hash: 0x%x, offset: %d, c_size: %d, c_rounded_size: %d, codec: %d, header: 0x%x 0x%x 0x%x", cs, cs->c_hash_data, cs->c_offset, c_size, c_rounded_size, cs->c_codec, *dinput, *(dinput + 1), *(dinput + 2));
+#else
+                       panic("decompressed data doesn't match original cs: %p, hash: %d, offset: 0x%x, c_size: %d", cs, cs->c_hash_data, cs->c_offset, c_size);
+#endif
+               }
 #endif
                if (c_seg->c_swappedin_ts == 0 && !kdp_mode) {
 #endif
                if (c_seg->c_swappedin_ts == 0 && !kdp_mode) {
-
                        clock_get_system_nanotime(&cur_ts_sec, &cur_ts_nsec);
 
                        age_of_cseg = (uint32_t)cur_ts_sec - c_seg->c_creation_ts;
                        clock_get_system_nanotime(&cur_ts_sec, &cur_ts_nsec);
 
                        age_of_cseg = (uint32_t)cur_ts_sec - c_seg->c_creation_ts;
-
-                       if (age_of_cseg < DECOMPRESSION_SAMPLE_MAX_AGE)
+                       if (age_of_cseg < DECOMPRESSION_SAMPLE_MAX_AGE) {
                                OSAddAtomic(1, &age_of_decompressions_during_sample_period[age_of_cseg]);
                                OSAddAtomic(1, &age_of_decompressions_during_sample_period[age_of_cseg]);
-                       else
+                       } else {
                                OSAddAtomic(1, &overage_decompressions_during_sample_period);
                                OSAddAtomic(1, &overage_decompressions_during_sample_period);
+                       }
 
                        OSAddAtomic(1, &sample_period_decompression_count);
                }
        }
 
                        OSAddAtomic(1, &sample_period_decompression_count);
                }
        }
-c_seg_invalid_data:
+#if CONFIG_FREEZE
+       else {
+               /*
+                * We are freeing an uncompressed page from this c_seg and so balance the ledgers.
+                */
+               if (C_SEG_IS_ONDISK(c_seg)) {
+                       /*
+                        * The compression sweep feature will push out anonymous pages to disk
+                        * without going through the freezer path and so those c_segs, while
+                        * swapped out, won't have an owner.
+                        */
+                       if (c_seg->c_task_owner) {
+                               task_update_frozen_to_swap_acct(c_seg->c_task_owner, PAGE_SIZE_64, DEBIT_FROM_SWAP);
+                       }
+
+                       /*
+                        * We are freeing a page in swap without swapping it in. We bump the in-core
+                        * count here to simulate a swapin of a page so that we can accurately
+                        * decrement it below.
+                        */
+                       OSAddAtomic(1, &c_segment_pages_compressed_incore);
+               }
+       }
+#endif /* CONFIG_FREEZE */
 
        if (flags & C_KEEP) {
                *zeroslot = 0;
                goto done;
        }
 
        if (flags & C_KEEP) {
                *zeroslot = 0;
                goto done;
        }
-
        assert(kdp_mode == FALSE);
        assert(kdp_mode == FALSE);
+
        c_seg->c_bytes_unused += c_rounded_size;
        c_seg->c_bytes_used -= c_rounded_size;
        c_seg->c_bytes_unused += c_rounded_size;
        c_seg->c_bytes_used -= c_rounded_size;
+
+       assert(c_seg->c_slots_used);
+       c_seg->c_slots_used--;
+
        PACK_C_SIZE(cs, 0);
 
        PACK_C_SIZE(cs, 0);
 
-       if (c_indx < c_seg->c_firstemptyslot)
+       if (c_indx < c_seg->c_firstemptyslot) {
                c_seg->c_firstemptyslot = c_indx;
                c_seg->c_firstemptyslot = c_indx;
+       }
 
        OSAddAtomic(-1, &c_segment_pages_compressed);
 
        OSAddAtomic(-1, &c_segment_pages_compressed);
+#if CONFIG_FREEZE
+       OSAddAtomic(-1, &c_segment_pages_compressed_incore);
+       assertf(c_segment_pages_compressed_incore >= 0, "-ve incore count %p 0x%x", c_seg, c_segment_pages_compressed_incore);
+#endif /* CONFIG_FREEZE */
 
        if (c_seg->c_state != C_ON_BAD_Q && !(C_SEG_IS_ONDISK(c_seg))) {
                /*
 
        if (c_seg->c_state != C_ON_BAD_Q && !(C_SEG_IS_ONDISK(c_seg))) {
                /*
@@ -3189,66 +4398,88 @@ c_seg_invalid_data:
                 */
                OSAddAtomic64(-c_rounded_size, &compressor_bytes_used);
        }
                 */
                OSAddAtomic64(-c_rounded_size, &compressor_bytes_used);
        }
+       if (c_seg->c_busy_swapping) {
+               /*
+                * bypass case for c_busy_swapping...
+                * let the swapin/swapout paths deal with putting
+                * the c_seg on the minor compaction queue if needed
+                */
+               assert(c_seg->c_busy);
+               goto done;
+       }
+       assert(!c_seg->c_busy);
+
        if (c_seg->c_state != C_IS_FILLING) {
                if (c_seg->c_bytes_used == 0) {
        if (c_seg->c_state != C_IS_FILLING) {
                if (c_seg->c_bytes_used == 0) {
-                       if ( !(C_SEG_IS_ONDISK(c_seg))) {
-                               int     pages_populated;
+                       if (!(C_SEG_IS_ONDISK(c_seg))) {
+                               int     pages_populated;
 
                                pages_populated = (round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE;
                                c_seg->c_populated_offset = C_SEG_BYTES_TO_OFFSET(0);
 
                                if (pages_populated) {
 
                                pages_populated = (round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE;
                                c_seg->c_populated_offset = C_SEG_BYTES_TO_OFFSET(0);
 
                                if (pages_populated) {
-
                                        assert(c_seg->c_state != C_ON_BAD_Q);
                                        assert(c_seg->c_store.c_buffer != NULL);
 
                                        C_SEG_BUSY(c_seg);
                                        lck_mtx_unlock_always(&c_seg->c_lock);
 
                                        assert(c_seg->c_state != C_ON_BAD_Q);
                                        assert(c_seg->c_store.c_buffer != NULL);
 
                                        C_SEG_BUSY(c_seg);
                                        lck_mtx_unlock_always(&c_seg->c_lock);
 
-                                       kernel_memory_depopulate(kernel_map, (vm_offset_t) c_seg->c_store.c_buffer, pages_populated * PAGE_SIZE, KMA_COMPRESSOR);
+                                       kernel_memory_depopulate(compressor_map,
+                                           (vm_offset_t) c_seg->c_store.c_buffer,
+                                           pages_populated * PAGE_SIZE, KMA_COMPRESSOR, VM_KERN_MEMORY_COMPRESSOR);
 
                                        lck_mtx_lock_spin_always(&c_seg->c_lock);
                                        C_SEG_WAKEUP_DONE(c_seg);
                                }
 
                                        lck_mtx_lock_spin_always(&c_seg->c_lock);
                                        C_SEG_WAKEUP_DONE(c_seg);
                                }
-                               if (!c_seg->c_on_minorcompact_q)
-                                       c_seg_need_delayed_compaction(c_seg);
-                       } else
-                               assert(c_seg->c_state == C_ON_SWAPPEDOUTSPARSE_Q);
-
+                               if (!c_seg->c_on_minorcompact_q && c_seg->c_state != C_ON_SWAPOUT_Q && c_seg->c_state != C_ON_SWAPIO_Q) {
+                                       c_seg_need_delayed_compaction(c_seg, FALSE);
+                               }
+                       } else {
+                               if (c_seg->c_state != C_ON_SWAPPEDOUTSPARSE_Q) {
+                                       c_seg_move_to_sparse_list(c_seg);
+                                       consider_defragmenting = TRUE;
+                               }
+                       }
                } else if (c_seg->c_on_minorcompact_q) {
                } else if (c_seg->c_on_minorcompact_q) {
-
                        assert(c_seg->c_state != C_ON_BAD_Q);
                        assert(c_seg->c_state != C_ON_BAD_Q);
+                       assert(!C_SEG_IS_ON_DISK_OR_SOQ(c_seg));
 
 
-                       if (C_SEG_SHOULD_MINORCOMPACT(c_seg)) {
+                       if (C_SEG_SHOULD_MINORCOMPACT_NOW(c_seg)) {
                                c_seg_try_minor_compaction_and_unlock(c_seg);
                                need_unlock = FALSE;
                        }
                                c_seg_try_minor_compaction_and_unlock(c_seg);
                                need_unlock = FALSE;
                        }
-               } else if ( !(C_SEG_IS_ONDISK(c_seg))) {
-
-                       if (c_seg->c_state != C_ON_BAD_Q && c_seg->c_state != C_ON_SWAPOUT_Q && C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE) {
-                               c_seg_need_delayed_compaction(c_seg);
+               } else if (!(C_SEG_IS_ONDISK(c_seg))) {
+                       if (c_seg->c_state != C_ON_BAD_Q && c_seg->c_state != C_ON_SWAPOUT_Q && c_seg->c_state != C_ON_SWAPIO_Q &&
+                           C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE) {
+                               c_seg_need_delayed_compaction(c_seg, FALSE);
                        }
                } else if (c_seg->c_state != C_ON_SWAPPEDOUTSPARSE_Q && C_SEG_ONDISK_IS_SPARSE(c_seg)) {
                        }
                } else if (c_seg->c_state != C_ON_SWAPPEDOUTSPARSE_Q && C_SEG_ONDISK_IS_SPARSE(c_seg)) {
-
                        c_seg_move_to_sparse_list(c_seg);
                        consider_defragmenting = TRUE;
                }
        }
 done:
                        c_seg_move_to_sparse_list(c_seg);
                        consider_defragmenting = TRUE;
                }
        }
 done:
-       if (kdp_mode) {
+       if (__improbable(kdp_mode)) {
                return retval;
        }
 
                return retval;
        }
 
-       if (need_unlock == TRUE)
+       if (need_unlock == TRUE) {
                lck_mtx_unlock_always(&c_seg->c_lock);
                lck_mtx_unlock_always(&c_seg->c_lock);
+       }
 
        PAGE_REPLACEMENT_DISALLOWED(FALSE);
 
 
        PAGE_REPLACEMENT_DISALLOWED(FALSE);
 
-       if (consider_defragmenting == TRUE)
-               vm_swap_consider_defragmenting();
+       if (consider_defragmenting == TRUE) {
+               vm_swap_consider_defragmenting(VM_SWAP_FLAGS_NONE);
+       }
 
 
+#if !XNU_TARGET_OS_OSX
+       if ((c_minor_count && COMPRESSOR_NEEDS_TO_MINOR_COMPACT()) || vm_compressor_needs_to_major_compact()) {
+               vm_wake_compactor_swapper();
+       }
+#endif /* !XNU_TARGET_OS_OSX */
 
 
-       return (retval);
+       return retval;
 }
 
 
 }
 
 
@@ -3256,20 +4487,18 @@ int
 vm_compressor_get(ppnum_t pn, int *slot, int flags)
 {
        c_slot_mapping_t  slot_ptr;
 vm_compressor_get(ppnum_t pn, int *slot, int flags)
 {
        c_slot_mapping_t  slot_ptr;
-       char    *dst;
-       int     zeroslot = 1;
-       int     retval;
+       char    *dst;
+       int     zeroslot = 1;
+       int     retval;
 
 
-#if __x86_64__
-       dst = PHYSMAP_PTOV((uint64_t)pn << (uint64_t)PAGE_SHIFT);
-#else
-#error "unsupported architecture"
-#endif
+       dst = pmap_map_compressor_page(pn);
        slot_ptr = (c_slot_mapping_t)slot;
 
        slot_ptr = (c_slot_mapping_t)slot;
 
+       assert(dst != NULL);
+
        if (slot_ptr->s_cseg == C_SV_CSEG_ID) {
        if (slot_ptr->s_cseg == C_SV_CSEG_ID) {
-               int32_t         data;
-               int32_t         *dptr;
+               int32_t         data;
+               int32_t         *dptr;
 
                /*
                 * page was populated with a single value
 
                /*
                 * page was populated with a single value
@@ -3279,28 +4508,25 @@ vm_compressor_get(ppnum_t pn, int *slot, int flags)
                 */
                dptr = (int32_t *)(uintptr_t)dst;
                data = c_segment_sv_hash_table[slot_ptr->s_cindx].he_data;
                 */
                dptr = (int32_t *)(uintptr_t)dst;
                data = c_segment_sv_hash_table[slot_ptr->s_cindx].he_data;
-#if __x86_64__
-               memset_word(dptr, data, PAGE_SIZE / sizeof(int32_t));
-#else
-               {
-               int             i;
-
-               for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++)
-                       *dptr++ = data;
-               }
-#endif
-               c_segment_sv_hash_drop_ref(slot_ptr->s_cindx);
+               sv_decompress(dptr, data);
+               if (!(flags & C_KEEP)) {
+                       c_segment_sv_hash_drop_ref(slot_ptr->s_cindx);
 
 
-               if ( !(flags & C_KEEP)) {
                        OSAddAtomic(-1, &c_segment_pages_compressed);
                        OSAddAtomic(-1, &c_segment_pages_compressed);
+#if CONFIG_FREEZE
+                       OSAddAtomic(-1, &c_segment_pages_compressed_incore);
+                       assertf(c_segment_pages_compressed_incore >= 0, "-ve incore count 0x%x", c_segment_pages_compressed_incore);
+#endif /* CONFIG_FREEZE */
                        *slot = 0;
                }
                        *slot = 0;
                }
-               if (data)
+               if (data) {
                        OSAddAtomic(1, &c_segment_svp_nonzero_decompressions);
                        OSAddAtomic(1, &c_segment_svp_nonzero_decompressions);
-               else
+               } else {
                        OSAddAtomic(1, &c_segment_svp_zero_decompressions);
                        OSAddAtomic(1, &c_segment_svp_zero_decompressions);
+               }
 
 
-               return (0);
+               pmap_unmap_compressor_page(pn, dst);
+               return 0;
        }
 
        retval = c_decompress_page(dst, slot_ptr, flags, &zeroslot);
        }
 
        retval = c_decompress_page(dst, slot_ptr, flags, &zeroslot);
@@ -3312,34 +4538,129 @@ vm_compressor_get(ppnum_t pn, int *slot, int flags)
        if (zeroslot) {
                *slot = 0;
        }
        if (zeroslot) {
                *slot = 0;
        }
+
+       pmap_unmap_compressor_page(pn, dst);
+
        /*
         * returns 0 if we successfully decompressed a page from a segment already in memory
         * returns 1 if we had to first swap in the segment, before successfully decompressing the page
         * returns -1 if we encountered an error swapping in the segment - decompression failed
         * returns -2 if (flags & C_DONT_BLOCK) and we found 'c_busy' or 'C_SEG_IS_ONDISK' to be true
         */
        /*
         * returns 0 if we successfully decompressed a page from a segment already in memory
         * returns 1 if we had to first swap in the segment, before successfully decompressing the page
         * returns -1 if we encountered an error swapping in the segment - decompression failed
         * returns -2 if (flags & C_DONT_BLOCK) and we found 'c_busy' or 'C_SEG_IS_ONDISK' to be true
         */
-       return (retval);
+       return retval;
+}
+
+#if DEVELOPMENT || DEBUG
+
+void
+vm_compressor_inject_error(int *slot)
+{
+       c_slot_mapping_t slot_ptr = (c_slot_mapping_t)slot;
+
+       /* No error detection for single-value compression. */
+       if (slot_ptr->s_cseg == C_SV_CSEG_ID) {
+               printf("%s(): cannot inject errors in SV-compressed pages\n", __func__ );
+               return;
+       }
+
+       /* s_cseg is actually "segno+1" */
+       const uint32_t c_segno = slot_ptr->s_cseg - 1;
+
+       assert(c_segno < c_segments_available);
+       assert(c_segments[c_segno].c_segno >= c_segments_available);
+
+       const c_segment_t c_seg = c_segments[c_segno].c_seg;
+
+       PAGE_REPLACEMENT_DISALLOWED(TRUE);
+
+       lck_mtx_lock_spin_always(&c_seg->c_lock);
+       assert(c_seg->c_state != C_IS_EMPTY && c_seg->c_state != C_IS_FREE);
+
+       const uint16_t c_indx = slot_ptr->s_cindx;
+       assert(c_indx < c_seg->c_nextslot);
+
+       /*
+        * To safely make this segment temporarily writable, we need to mark
+        * the segment busy, which allows us to release the segment lock.
+        */
+       while (c_seg->c_busy) {
+               c_seg_wait_on_busy(c_seg);
+               lck_mtx_lock_spin_always(&c_seg->c_lock);
+       }
+       C_SEG_BUSY(c_seg);
+
+       bool already_writable = (c_seg->c_state == C_IS_FILLING);
+       if (!already_writable) {
+               /*
+                * Protection update must be performed preemptibly, so temporarily drop
+                * the lock. Having set c_busy will prevent most other concurrent
+                * operations.
+                */
+               lck_mtx_unlock_always(&c_seg->c_lock);
+               C_SEG_MAKE_WRITEABLE(c_seg);
+               lck_mtx_lock_spin_always(&c_seg->c_lock);
+       }
+
+       /*
+        * Once we've released the lock following our c_state == C_IS_FILLING check,
+        * c_current_seg_filled() can (re-)write-protect the segment. However, it
+        * will transition from C_IS_FILLING before releasing the c_seg lock, so we
+        * can detect this by re-checking after we've reobtained the lock.
+        */
+       if (already_writable && c_seg->c_state != C_IS_FILLING) {
+               lck_mtx_unlock_always(&c_seg->c_lock);
+               C_SEG_MAKE_WRITEABLE(c_seg);
+               lck_mtx_lock_spin_always(&c_seg->c_lock);
+               already_writable = false;
+               /* Segment can't be freed while c_busy is set. */
+               assert(c_seg->c_state != C_IS_FILLING);
+       }
+
+       c_slot_t cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
+       int32_t *data = &c_seg->c_store.c_buffer[cs->c_offset];
+       /* assume that the compressed data holds at least one int32_t */
+       assert(UNPACK_C_SIZE(cs) > sizeof(*data));
+       /*
+        * This bit is known to be in the payload of a MISS packet resulting from
+        * the pattern used in the test pattern from decompression_failure.c.
+        * Flipping it should result in many corrupted bits in the test page.
+        */
+       data[0] ^= 0x00000100;
+       if (!already_writable) {
+               lck_mtx_unlock_always(&c_seg->c_lock);
+               C_SEG_WRITE_PROTECT(c_seg);
+               lck_mtx_lock_spin_always(&c_seg->c_lock);
+       }
+
+       C_SEG_WAKEUP_DONE(c_seg);
+       lck_mtx_unlock_always(&c_seg->c_lock);
+
+       PAGE_REPLACEMENT_DISALLOWED(FALSE);
 }
 
 }
 
+#endif /* DEVELOPMENT || DEBUG */
 
 int
 vm_compressor_free(int *slot, int flags)
 {
        c_slot_mapping_t  slot_ptr;
 
 int
 vm_compressor_free(int *slot, int flags)
 {
        c_slot_mapping_t  slot_ptr;
-       int     zeroslot = 1;
-       int     retval;
+       int     zeroslot = 1;
+       int     retval;
 
        assert(flags == 0 || flags == C_DONT_BLOCK);
 
        slot_ptr = (c_slot_mapping_t)slot;
 
        if (slot_ptr->s_cseg == C_SV_CSEG_ID) {
 
        assert(flags == 0 || flags == C_DONT_BLOCK);
 
        slot_ptr = (c_slot_mapping_t)slot;
 
        if (slot_ptr->s_cseg == C_SV_CSEG_ID) {
-
                c_segment_sv_hash_drop_ref(slot_ptr->s_cindx);
                OSAddAtomic(-1, &c_segment_pages_compressed);
                c_segment_sv_hash_drop_ref(slot_ptr->s_cindx);
                OSAddAtomic(-1, &c_segment_pages_compressed);
+#if CONFIG_FREEZE
+               OSAddAtomic(-1, &c_segment_pages_compressed_incore);
+               assertf(c_segment_pages_compressed_incore >= 0, "-ve incore count 0x%x", c_segment_pages_compressed_incore);
+#endif /* CONFIG_FREEZE */
 
                *slot = 0;
 
                *slot = 0;
-               return (0);
+               return 0;
        }
        retval = c_decompress_page(NULL, slot_ptr, flags, &zeroslot);
        /*
        }
        retval = c_decompress_page(NULL, slot_ptr, flags, &zeroslot);
        /*
@@ -3347,40 +4668,40 @@ vm_compressor_free(int *slot, int flags)
         * returns -2 if (flags & C_DONT_BLOCK) and we found 'c_busy' set
         */
 
         * returns -2 if (flags & C_DONT_BLOCK) and we found 'c_busy' set
         */
 
-       if (retval == 0)
+       if (retval == 0) {
                *slot = 0;
                *slot = 0;
-       else
+       } else {
                assert(retval == -2);
                assert(retval == -2);
+       }
 
 
-       return (retval);
+       return retval;
 }
 
 
 int
 vm_compressor_put(ppnum_t pn, int *slot, void  **current_chead, char *scratch_buf)
 {
 }
 
 
 int
 vm_compressor_put(ppnum_t pn, int *slot, void  **current_chead, char *scratch_buf)
 {
-       char    *src;
-       int     retval;
+       char    *src;
+       int     retval;
+
+       src = pmap_map_compressor_page(pn);
+       assert(src != NULL);
 
 
-#if __x86_64__
-       src = PHYSMAP_PTOV((uint64_t)pn << (uint64_t)PAGE_SHIFT);
-#else
-#error "unsupported architecture"
-#endif
        retval = c_compress_page(src, (c_slot_mapping_t)slot, (c_segment_t *)current_chead, scratch_buf);
        retval = c_compress_page(src, (c_slot_mapping_t)slot, (c_segment_t *)current_chead, scratch_buf);
+       pmap_unmap_compressor_page(pn, src);
 
 
-       return (retval);
+       return retval;
 }
 
 void
 vm_compressor_transfer(
 }
 
 void
 vm_compressor_transfer(
-       int     *dst_slot_p,
-       int     *src_slot_p)
+       int     *dst_slot_p,
+       int     *src_slot_p)
 {
 {
-       c_slot_mapping_t        dst_slot, src_slot;
-       c_segment_t             c_seg;
-       int                     c_indx;
-       c_slot_t                cs;
+       c_slot_mapping_t        dst_slot, src_slot;
+       c_segment_t             c_seg;
+       uint16_t                c_indx;
+       c_slot_t                cs;
 
        src_slot = (c_slot_mapping_t) src_slot_p;
 
 
        src_slot = (c_slot_mapping_t) src_slot_p;
 
@@ -3393,7 +4714,7 @@ vm_compressor_transfer(
 Retry:
        PAGE_REPLACEMENT_DISALLOWED(TRUE);
        /* get segment for src_slot */
 Retry:
        PAGE_REPLACEMENT_DISALLOWED(TRUE);
        /* get segment for src_slot */
-       c_seg = c_segments[src_slot->s_cseg -1].c_seg;
+       c_seg = c_segments[src_slot->s_cseg - 1].c_seg;
        /* lock segment */
        lck_mtx_lock_spin_always(&c_seg->c_lock);
        /* wait if it's busy */
        /* lock segment */
        lck_mtx_lock_spin_always(&c_seg->c_lock);
        /* wait if it's busy */
@@ -3406,6 +4727,7 @@ Retry:
        c_indx = src_slot->s_cindx;
        cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
        /* point the c_slot back to dst_slot instead of src_slot */
        c_indx = src_slot->s_cindx;
        cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
        /* point the c_slot back to dst_slot instead of src_slot */
+       C_SLOT_ASSERT_PACKABLE(dst_slot);
        cs->c_packed_ptr = C_SLOT_PACK_PTR(dst_slot);
        /* transfer */
        *dst_slot_p = *src_slot_p;
        cs->c_packed_ptr = C_SLOT_PACK_PTR(dst_slot);
        /* transfer */
        *dst_slot_p = *src_slot_p;
@@ -3416,19 +4738,20 @@ Retry:
 
 #if CONFIG_FREEZE
 
 
 #if CONFIG_FREEZE
 
-int    freezer_finished_filling = 0;
+int     freezer_finished_filling = 0;
 
 void
 vm_compressor_finished_filling(
 
 void
 vm_compressor_finished_filling(
-       void    **current_chead)
+       void    **current_chead)
 {
 {
-       c_segment_t     c_seg;
+       c_segment_t     c_seg;
 
 
-       if ((c_seg = *(c_segment_t *)current_chead) == NULL)
+       if ((c_seg = *(c_segment_t *)current_chead) == NULL) {
                return;
                return;
+       }
 
        assert(c_seg->c_state == C_IS_FILLING);
 
        assert(c_seg->c_state == C_IS_FILLING);
-       
+
        lck_mtx_lock_spin_always(&c_seg->c_lock);
 
        c_current_seg_filled(c_seg, (c_segment_t *)current_chead);
        lck_mtx_lock_spin_always(&c_seg->c_lock);
 
        c_current_seg_filled(c_seg, (c_segment_t *)current_chead);
@@ -3446,27 +4769,27 @@ vm_compressor_finished_filling(
  *
  * Currently, this routine is only used by the "freezer backed by
  * compressor with swap" mode to create a series of c_segs that
  *
  * Currently, this routine is only used by the "freezer backed by
  * compressor with swap" mode to create a series of c_segs that
- * only contain compressed data belonging to one task. So, we 
+ * only contain compressed data belonging to one task. So, we
  * move a task's previously compressed data into a set of new
  * c_segs which will also hold the task's yet to be compressed data.
  */
 
 kern_return_t
 vm_compressor_relocate(
  * move a task's previously compressed data into a set of new
  * c_segs which will also hold the task's yet to be compressed data.
  */
 
 kern_return_t
 vm_compressor_relocate(
-       void            **current_chead,
-       int             *slot_p)
-{
-       c_slot_mapping_t        slot_ptr;
-       c_slot_mapping_t        src_slot;
-       uint32_t                c_rounded_size;
-       uint32_t                c_size;
-       uint16_t                dst_slot;
-       c_slot_t                c_dst;
-       c_slot_t                c_src;
-       int                     c_indx;
-       c_segment_t             c_seg_dst = NULL;
-       c_segment_t             c_seg_src = NULL;
-       kern_return_t           kr = KERN_SUCCESS;
+       void            **current_chead,
+       int             *slot_p)
+{
+       c_slot_mapping_t        slot_ptr;
+       c_slot_mapping_t        src_slot;
+       uint32_t                c_rounded_size;
+       uint32_t                c_size;
+       uint16_t                dst_slot;
+       c_slot_t                c_dst;
+       c_slot_t                c_src;
+       uint16_t                c_indx;
+       c_segment_t             c_seg_dst = NULL;
+       c_segment_t             c_seg_src = NULL;
+       kern_return_t           kr = KERN_SUCCESS;
 
 
        src_slot = (c_slot_mapping_t) slot_p;
 
 
        src_slot = (c_slot_mapping_t) slot_p;
@@ -3477,7 +4800,7 @@ vm_compressor_relocate(
                 * value which is hashed to a single entry not contained
                 * in a c_segment_t
                 */
                 * value which is hashed to a single entry not contained
                 * in a c_segment_t
                 */
-               return (kr);
+               return kr;
        }
 
 Relookup_dst:
        }
 
 Relookup_dst:
@@ -3501,7 +4824,7 @@ Relookup_dst:
        C_SEG_BUSY(c_seg_dst);
 
        dst_slot = c_seg_dst->c_nextslot;
        C_SEG_BUSY(c_seg_dst);
 
        dst_slot = c_seg_dst->c_nextslot;
-       
+
        lck_mtx_unlock_always(&c_seg_dst->c_lock);
 
 Relookup_src:
        lck_mtx_unlock_always(&c_seg_dst->c_lock);
 
 Relookup_src:
@@ -3511,20 +4834,23 @@ Relookup_src:
 
        lck_mtx_lock_spin_always(&c_seg_src->c_lock);
 
 
        lck_mtx_lock_spin_always(&c_seg_src->c_lock);
 
-       if (C_SEG_IS_ONDISK(c_seg_src)) {
-       
+       if (C_SEG_IS_ON_DISK_OR_SOQ(c_seg_src) ||
+           c_seg_src->c_state == C_IS_FILLING) {
                /*
                /*
-                * A "thaw" can mark a process as eligible for
+                * Skip this page if :-
+                * a) the src c_seg is already on-disk (or on its way there)
+                *    A "thaw" can mark a process as eligible for
                 * another freeze cycle without bringing any of
                 * its swapped out c_segs back from disk (because
                 * that is done on-demand).
                 * another freeze cycle without bringing any of
                 * its swapped out c_segs back from disk (because
                 * that is done on-demand).
+                *    Or, this page may be mapped elsewhere in the task's map,
+                * and we may have marked it for swap already.
                 *
                 *
-                * If the src c_seg we find for our pre-compressed
-                * data is already on-disk, then we are dealing
-                * with an app's data that is already packed and
-                * swapped out. Don't do anything.
+                * b) Or, the src c_seg is being filled by the compressor
+                * thread. We don't want the added latency of waiting for
+                * this c_seg in the freeze path and so we skip it.
                 */
                 */
-               
+
                PAGE_REPLACEMENT_DISALLOWED(FALSE);
 
                lck_mtx_unlock_always(&c_seg_src->c_lock);
                PAGE_REPLACEMENT_DISALLOWED(FALSE);
 
                lck_mtx_unlock_always(&c_seg_src->c_lock);
@@ -3535,10 +4861,9 @@ Relookup_src:
        }
 
        if (c_seg_src->c_busy) {
        }
 
        if (c_seg_src->c_busy) {
-
                PAGE_REPLACEMENT_DISALLOWED(FALSE);
                c_seg_wait_on_busy(c_seg_src);
                PAGE_REPLACEMENT_DISALLOWED(FALSE);
                c_seg_wait_on_busy(c_seg_src);
-                       
+
                c_seg_src = NULL;
 
                PAGE_REPLACEMENT_DISALLOWED(TRUE);
                c_seg_src = NULL;
 
                PAGE_REPLACEMENT_DISALLOWED(TRUE);
@@ -3549,7 +4874,7 @@ Relookup_src:
        C_SEG_BUSY(c_seg_src);
 
        lck_mtx_unlock_always(&c_seg_src->c_lock);
        C_SEG_BUSY(c_seg_src);
 
        lck_mtx_unlock_always(&c_seg_src->c_lock);
-       
+
        PAGE_REPLACEMENT_DISALLOWED(FALSE);
 
        /* find the c_slot */
        PAGE_REPLACEMENT_DISALLOWED(FALSE);
 
        /* find the c_slot */
@@ -3567,7 +4892,7 @@ Relookup_src:
                 */
 
                PAGE_REPLACEMENT_DISALLOWED(TRUE);
                 */
 
                PAGE_REPLACEMENT_DISALLOWED(TRUE);
-       
+
                lck_mtx_lock_spin_always(&c_seg_src->c_lock);
                C_SEG_WAKEUP_DONE(c_seg_src);
                lck_mtx_unlock_always(&c_seg_src->c_lock);
                lck_mtx_lock_spin_always(&c_seg_src->c_lock);
                C_SEG_WAKEUP_DONE(c_seg_src);
                lck_mtx_unlock_always(&c_seg_src->c_lock);
@@ -3582,9 +4907,9 @@ Relookup_src:
 
                c_current_seg_filled(c_seg_dst, (c_segment_t *)current_chead);
                assert(*current_chead == NULL);
 
                c_current_seg_filled(c_seg_dst, (c_segment_t *)current_chead);
                assert(*current_chead == NULL);
-       
+
                C_SEG_WAKEUP_DONE(c_seg_dst);
                C_SEG_WAKEUP_DONE(c_seg_dst);
-       
+
                lck_mtx_unlock_always(&c_seg_dst->c_lock);
 
                c_seg_dst = NULL;
                lck_mtx_unlock_always(&c_seg_dst->c_lock);
 
                c_seg_dst = NULL;
@@ -3597,41 +4922,40 @@ Relookup_src:
        c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, c_seg_dst->c_nextslot);
 
        memcpy(&c_seg_dst->c_store.c_buffer[c_seg_dst->c_nextoffset], &c_seg_src->c_store.c_buffer[c_src->c_offset], c_size);
        c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, c_seg_dst->c_nextslot);
 
        memcpy(&c_seg_dst->c_store.c_buffer[c_seg_dst->c_nextoffset], &c_seg_src->c_store.c_buffer[c_src->c_offset], c_size);
-
+       /*
+        * Is platform alignment actually necessary since wkdm aligns its output?
+        */
        c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
 
        c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
 
-#if CHECKSUM_THE_DATA
-       c_dst->c_hash_data = c_src->c_hash_data;
-#endif
-#if CHECKSUM_THE_COMPRESSED_DATA
-       c_dst->c_hash_compressed_data = c_src->c_hash_compressed_data;
-#endif
-
-       c_dst->c_size = c_src->c_size;
-       c_dst->c_packed_ptr = c_src->c_packed_ptr;
+       cslot_copy(c_dst, c_src);
        c_dst->c_offset = c_seg_dst->c_nextoffset;
 
        c_dst->c_offset = c_seg_dst->c_nextoffset;
 
-       if (c_seg_dst->c_firstemptyslot == c_seg_dst->c_nextslot)
+       if (c_seg_dst->c_firstemptyslot == c_seg_dst->c_nextslot) {
                c_seg_dst->c_firstemptyslot++;
                c_seg_dst->c_firstemptyslot++;
+       }
 
 
+       c_seg_dst->c_slots_used++;
        c_seg_dst->c_nextslot++;
        c_seg_dst->c_bytes_used += c_rounded_size;
        c_seg_dst->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
        c_seg_dst->c_nextslot++;
        c_seg_dst->c_bytes_used += c_rounded_size;
        c_seg_dst->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
-               
+
 
        PACK_C_SIZE(c_src, 0);
 
        c_seg_src->c_bytes_used -= c_rounded_size;
        c_seg_src->c_bytes_unused += c_rounded_size;
 
        PACK_C_SIZE(c_src, 0);
 
        c_seg_src->c_bytes_used -= c_rounded_size;
        c_seg_src->c_bytes_unused += c_rounded_size;
-       
+
+       assert(c_seg_src->c_slots_used);
+       c_seg_src->c_slots_used--;
+
        if (c_indx < c_seg_src->c_firstemptyslot) {
                c_seg_src->c_firstemptyslot = c_indx;
        }
 
        c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, dst_slot);
        if (c_indx < c_seg_src->c_firstemptyslot) {
                c_seg_src->c_firstemptyslot = c_indx;
        }
 
        c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, dst_slot);
-               
+
        PAGE_REPLACEMENT_ALLOWED(TRUE);
        PAGE_REPLACEMENT_ALLOWED(TRUE);
-       slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
+       slot_ptr = C_SLOT_UNPACK_PTR(c_dst);
        /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
        slot_ptr->s_cseg = c_seg_dst->c_mysegno + 1;
        slot_ptr->s_cindx = dst_slot;
        /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
        slot_ptr->s_cseg = c_seg_dst->c_mysegno + 1;
        slot_ptr->s_cindx = dst_slot;
@@ -3640,21 +4964,20 @@ Relookup_src:
 
 out:
        if (c_seg_src) {
 
 out:
        if (c_seg_src) {
-
                lck_mtx_lock_spin_always(&c_seg_src->c_lock);
 
                C_SEG_WAKEUP_DONE(c_seg_src);
 
                if (c_seg_src->c_bytes_used == 0 && c_seg_src->c_state != C_IS_FILLING) {
                lck_mtx_lock_spin_always(&c_seg_src->c_lock);
 
                C_SEG_WAKEUP_DONE(c_seg_src);
 
                if (c_seg_src->c_bytes_used == 0 && c_seg_src->c_state != C_IS_FILLING) {
-                       if (!c_seg_src->c_on_minorcompact_q)
-                               c_seg_need_delayed_compaction(c_seg_src);
+                       if (!c_seg_src->c_on_minorcompact_q) {
+                               c_seg_need_delayed_compaction(c_seg_src, FALSE);
+                       }
                }
 
                lck_mtx_unlock_always(&c_seg_src->c_lock);
        }
                }
 
                lck_mtx_unlock_always(&c_seg_src->c_lock);
        }
-       
-       if (c_seg_dst) {
 
 
+       if (c_seg_dst) {
                PAGE_REPLACEMENT_DISALLOWED(TRUE);
 
                lck_mtx_lock_spin_always(&c_seg_dst->c_lock);
                PAGE_REPLACEMENT_DISALLOWED(TRUE);
 
                lck_mtx_lock_spin_always(&c_seg_dst->c_lock);
@@ -3669,8 +4992,8 @@ out:
 
                        c_current_seg_filled(c_seg_dst, (c_segment_t *)current_chead);
                        assert(*current_chead == NULL);
 
                        c_current_seg_filled(c_seg_dst, (c_segment_t *)current_chead);
                        assert(*current_chead == NULL);
-               }  
-               
+               }
+
                C_SEG_WAKEUP_DONE(c_seg_dst);
 
                lck_mtx_unlock_always(&c_seg_dst->c_lock);
                C_SEG_WAKEUP_DONE(c_seg_dst);
 
                lck_mtx_unlock_always(&c_seg_dst->c_lock);