]> git.saurik.com Git - apple/xnu.git/blobdiff - osfmk/vm/vm_compressor.c
xnu-7195.101.1.tar.gz
[apple/xnu.git] / osfmk / vm / vm_compressor.c
index c79a03e57b471e7190b20946d361eeba8749229c..315c8b429173900981277393e75256c411f3be3e 100644 (file)
@@ -1,5 +1,5 @@
 /*
 /*
- * Copyright (c) 2000-2013 Apple Inc. All rights reserved.
+ * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *
 #include <IOKit/IOHibernatePrivate.h>
 
 extern boolean_t vm_darkwake_mode;
 #include <IOKit/IOHibernatePrivate.h>
 
 extern boolean_t vm_darkwake_mode;
+extern zone_t vm_page_zone;
+
+#if DEVELOPMENT || DEBUG
+/* sysctl defined in bsd/dev/arm64/sysctl.c */
+int do_cseg_wedge_thread(void);
+int do_cseg_unwedge_thread(void);
+static event_t debug_cseg_wait_event = NULL;
+#endif /* DEVELOPMENT || DEBUG */
+
+#if CONFIG_FREEZE
+bool freezer_incore_cseg_acct = TRUE; /* Only count incore compressed memory for jetsams. */
+void task_disown_frozen_csegs(task_t owner_task);
+#endif /* CONFIG_FREEZE */
 
 #if POPCOUNT_THE_COMPRESSED_DATA
 boolean_t popcount_c_segs = TRUE;
 
 #if POPCOUNT_THE_COMPRESSED_DATA
 boolean_t popcount_c_segs = TRUE;
@@ -105,32 +118,25 @@ boolean_t validate_c_segs = TRUE;
  * the boot-arg & device-tree code.
  */
 
  * the boot-arg & device-tree code.
  */
 
-#if CONFIG_EMBEDDED
+#if !XNU_TARGET_OS_OSX
 
 #if CONFIG_FREEZE
 
 #if CONFIG_FREEZE
-int             vm_compressor_mode = VM_PAGER_FREEZER_DEFAULT;
-
-void            *freezer_chead; /* The chead used to track c_segs allocated for the exclusive use of holding just one task's compressed memory.*/
-char            *freezer_compressor_scratch_buf = NULL;
-
-extern int      c_freezer_swapout_page_count;      /* This count keeps track of the # of compressed pages holding just one task's compressed memory on the swapout queue. This count is used during each freeze i.e. on a per-task basis.*/
-
+int     vm_compressor_mode = VM_PAGER_FREEZER_DEFAULT;
+struct  freezer_context freezer_context_global;
 #else /* CONFIG_FREEZE */
 #else /* CONFIG_FREEZE */
-int             vm_compressor_mode = VM_PAGER_NOT_CONFIGURED;
+int     vm_compressor_mode = VM_PAGER_NOT_CONFIGURED;
 #endif /* CONFIG_FREEZE */
 
 #endif /* CONFIG_FREEZE */
 
-int             vm_scale = 1;
-
-#else /* CONFIG_EMBEDDED */
+#else /* !XNU_TARGET_OS_OSX */
 int             vm_compressor_mode = VM_PAGER_COMPRESSOR_WITH_SWAP;
 int             vm_compressor_mode = VM_PAGER_COMPRESSOR_WITH_SWAP;
-int             vm_scale = 16;
 
 
-#endif /* CONFIG_EMBEDDED */
+#endif /* !XNU_TARGET_OS_OSX */
 
 
+TUNABLE(uint32_t, vm_compression_limit, "vm_compression_limit", 0);
 int             vm_compressor_is_active = 0;
 int             vm_compressor_is_active = 0;
-int             vm_compression_limit = 0;
 int             vm_compressor_available = 0;
 
 int             vm_compressor_available = 0;
 
+extern uint64_t vm_swap_get_max_configured_space(void);
 extern void     vm_pageout_io_throttle(void);
 
 #if CHECKSUM_THE_DATA || CHECKSUM_THE_SWAP || CHECKSUM_THE_COMPRESSED_DATA
 extern void     vm_pageout_io_throttle(void);
 
 #if CHECKSUM_THE_DATA || CHECKSUM_THE_SWAP || CHECKSUM_THE_COMPRESSED_DATA
@@ -177,11 +183,18 @@ union c_segu {
        uintptr_t       c_segno;
 };
 
        uintptr_t       c_segno;
 };
 
+#define C_SLOT_ASSERT_PACKABLE(ptr) \
+       VM_ASSERT_POINTER_PACKABLE((vm_offset_t)(ptr), C_SLOT_PACKED_PTR);
 
 
+#define C_SLOT_PACK_PTR(ptr) \
+       VM_PACK_POINTER((vm_offset_t)(ptr), C_SLOT_PACKED_PTR)
 
 
-#define C_SLOT_PACK_PTR(ptr)            (((uintptr_t)ptr - (uintptr_t) KERNEL_PMAP_HEAP_RANGE_START) >> 2)
-#define C_SLOT_UNPACK_PTR(cslot)        ((uintptr_t)(cslot->c_packed_ptr << 2) + (uintptr_t) KERNEL_PMAP_HEAP_RANGE_START)
+#define C_SLOT_UNPACK_PTR(cslot) \
+       (c_slot_mapping_t)VM_UNPACK_POINTER((cslot)->c_packed_ptr, C_SLOT_PACKED_PTR)
 
 
+/* for debugging purposes */
+SECURITY_READ_ONLY_EARLY(vm_packing_params_t) c_slot_packing_params =
+    VM_PACKING_PARAMS(C_SLOT_PACKED_PTR);
 
 uint32_t        c_segment_count = 0;
 uint32_t        c_segment_count_max = 0;
 
 uint32_t        c_segment_count = 0;
 uint32_t        c_segment_count_max = 0;
@@ -251,7 +264,12 @@ uint32_t        c_segment_svp_nonzero_decompressions;
 
 uint32_t        c_segment_noncompressible_pages;
 
 
 uint32_t        c_segment_noncompressible_pages;
 
-uint32_t        c_segment_pages_compressed;
+uint32_t        c_segment_pages_compressed = 0; /* Tracks # of uncompressed pages fed into the compressor */
+#if CONFIG_FREEZE
+int32_t        c_segment_pages_compressed_incore = 0; /* Tracks # of uncompressed pages fed into the compressor that are in memory */
+uint32_t        c_segments_incore_limit = 0; /* Tracks # of segments allowed to be in-core. Based on compressor pool size */
+#endif /* CONFIG_FREEZE */
+
 uint32_t        c_segment_pages_compressed_limit;
 uint32_t        c_segment_pages_compressed_nearing_limit;
 uint32_t        c_free_segno_head = (uint32_t)-1;
 uint32_t        c_segment_pages_compressed_limit;
 uint32_t        c_segment_pages_compressed_nearing_limit;
 uint32_t        c_free_segno_head = (uint32_t)-1;
@@ -268,12 +286,10 @@ uint32_t        vm_compressor_catchup_threshold_divisor_overridden = 0;
 
 #define         C_SEGMENTS_PER_PAGE     (PAGE_SIZE / sizeof(union c_segu))
 
 
 #define         C_SEGMENTS_PER_PAGE     (PAGE_SIZE / sizeof(union c_segu))
 
+LCK_GRP_DECLARE(vm_compressor_lck_grp, "vm_compressor");
+LCK_RW_DECLARE(c_master_lock, &vm_compressor_lck_grp);
+LCK_MTX_DECLARE(c_list_lock_storage, &vm_compressor_lck_grp);
 
 
-lck_grp_attr_t  vm_compressor_lck_grp_attr;
-lck_attr_t      vm_compressor_lck_attr;
-lck_grp_t       vm_compressor_lck_grp;
-lck_mtx_t       *c_list_lock;
-lck_rw_t        c_master_lock;
 boolean_t       decompressions_blocked = FALSE;
 
 zone_t          compressor_segment_zone;
 boolean_t       decompressions_blocked = FALSE;
 
 zone_t          compressor_segment_zone;
@@ -328,9 +344,9 @@ static void vm_compressor_do_delayed_compactions(boolean_t);
 static void vm_compressor_compact_and_swap(boolean_t);
 static void vm_compressor_age_swapped_in_segments(boolean_t);
 
 static void vm_compressor_compact_and_swap(boolean_t);
 static void vm_compressor_age_swapped_in_segments(boolean_t);
 
-#if !CONFIG_EMBEDDED
+#if XNU_TARGET_OS_OSX
 static void vm_compressor_take_paging_space_action(void);
 static void vm_compressor_take_paging_space_action(void);
-#endif
+#endif /* XNU_TARGET_OS_OSX */
 
 void compute_swapout_target_age(void);
 
 
 void compute_swapout_target_age(void);
 
@@ -387,11 +403,27 @@ vm_compressor_pages_compressed(void)
 boolean_t
 vm_compressor_low_on_space(void)
 {
 boolean_t
 vm_compressor_low_on_space(void)
 {
+#if CONFIG_FREEZE
+       uint64_t incore_seg_count;
+       uint32_t incore_compressed_pages;
+       if (freezer_incore_cseg_acct) {
+               incore_seg_count = c_segment_count - c_swappedout_count - c_swappedout_sparse_count;
+               incore_compressed_pages = c_segment_pages_compressed_incore;
+       } else {
+               incore_seg_count = c_segment_count;
+               incore_compressed_pages = c_segment_pages_compressed;
+       }
+
+       if ((incore_compressed_pages > c_segment_pages_compressed_nearing_limit) ||
+           (incore_seg_count > c_segments_nearing_limit)) {
+               return TRUE;
+       }
+#else /* CONFIG_FREEZE */
        if ((c_segment_pages_compressed > c_segment_pages_compressed_nearing_limit) ||
            (c_segment_count > c_segments_nearing_limit)) {
                return TRUE;
        }
        if ((c_segment_pages_compressed > c_segment_pages_compressed_nearing_limit) ||
            (c_segment_count > c_segments_nearing_limit)) {
                return TRUE;
        }
-
+#endif /* CONFIG_FREEZE */
        return FALSE;
 }
 
        return FALSE;
 }
 
@@ -399,11 +431,27 @@ vm_compressor_low_on_space(void)
 boolean_t
 vm_compressor_out_of_space(void)
 {
 boolean_t
 vm_compressor_out_of_space(void)
 {
+#if CONFIG_FREEZE
+       uint64_t incore_seg_count;
+       uint32_t incore_compressed_pages;
+       if (freezer_incore_cseg_acct) {
+               incore_seg_count = c_segment_count - c_swappedout_count - c_swappedout_sparse_count;
+               incore_compressed_pages = c_segment_pages_compressed_incore;
+       } else {
+               incore_seg_count = c_segment_count;
+               incore_compressed_pages = c_segment_pages_compressed;
+       }
+
+       if ((incore_compressed_pages >= c_segment_pages_compressed_limit) ||
+           (incore_seg_count > c_segments_incore_limit)) {
+               return TRUE;
+       }
+#else /* CONFIG_FREEZE */
        if ((c_segment_pages_compressed >= c_segment_pages_compressed_limit) ||
            (c_segment_count >= c_segments_limit)) {
                return TRUE;
        }
        if ((c_segment_pages_compressed >= c_segment_pages_compressed_limit) ||
            (c_segment_count >= c_segments_limit)) {
                return TRUE;
        }
-
+#endif /* CONFIG_FREEZE */
        return FALSE;
 }
 
        return FALSE;
 }
 
@@ -426,11 +474,14 @@ vm_wants_task_throttled(task_t task)
 
 
 #if DEVELOPMENT || DEBUG
 
 
 #if DEVELOPMENT || DEBUG
-boolean_t kill_on_no_paging_space = FALSE; /* On compressor/swap exhaustion, kill the largest process regardless of
-                                            * its chosen process policy. Controlled by a boot-arg of the same name. */
+/*
+ * On compressor/swap exhaustion, kill the largest process regardless of
+ * its chosen process policy.
+ */
+TUNABLE(bool, kill_on_no_paging_space, "-kill_on_no_paging_space", false);
 #endif /* DEVELOPMENT || DEBUG */
 
 #endif /* DEVELOPMENT || DEBUG */
 
-#if !CONFIG_EMBEDDED
+#if XNU_TARGET_OS_OSX
 
 static uint32_t no_paging_space_action_in_progress = 0;
 extern void memorystatus_send_low_swap_note(void);
 
 static uint32_t no_paging_space_action_in_progress = 0;
 extern void memorystatus_send_low_swap_note(void);
@@ -442,7 +493,7 @@ vm_compressor_take_paging_space_action(void)
                if (OSCompareAndSwap(0, 1, (UInt32 *)&no_paging_space_action_in_progress)) {
                        if (no_paging_space_action()) {
 #if DEVELOPMENT || DEBUG
                if (OSCompareAndSwap(0, 1, (UInt32 *)&no_paging_space_action_in_progress)) {
                        if (no_paging_space_action()) {
 #if DEVELOPMENT || DEBUG
-                               if (kill_on_no_paging_space == TRUE) {
+                               if (kill_on_no_paging_space) {
                                        /*
                                         * Since we are choosing to always kill a process, we don't need the
                                         * "out of application memory" dialog box in this mode. And, hence we won't
                                        /*
                                         * Since we are choosing to always kill a process, we don't need the
                                         * "out of application memory" dialog box in this mode. And, hence we won't
@@ -459,18 +510,7 @@ vm_compressor_take_paging_space_action(void)
                }
        }
 }
                }
        }
 }
-#endif /* !CONFIG_EMBEDDED */
-
-
-void
-vm_compressor_init_locks(void)
-{
-       lck_grp_attr_setdefault(&vm_compressor_lck_grp_attr);
-       lck_grp_init(&vm_compressor_lck_grp, "vm_compressor", &vm_compressor_lck_grp_attr);
-       lck_attr_setdefault(&vm_compressor_lck_attr);
-
-       lck_rw_init(&c_master_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
-}
+#endif /* XNU_TARGET_OS_OSX */
 
 
 void
 
 
 void
@@ -512,6 +552,9 @@ cslot_copy(c_slot_t cdst, c_slot_t csrc)
 #if defined(__arm__) || defined(__arm64__)
        cdst->c_codec = csrc->c_codec;
 #endif
 #if defined(__arm__) || defined(__arm64__)
        cdst->c_codec = csrc->c_codec;
 #endif
+#if __ARM_WKDM_POPCNT__
+       cdst->c_inline_popcount = csrc->c_inline_popcount;
+#endif
 }
 
 vm_map_t compressor_map;
 }
 
 vm_map_t compressor_map;
@@ -535,10 +578,6 @@ void
 vm_compressor_init(void)
 {
        thread_t        thread;
 vm_compressor_init(void)
 {
        thread_t        thread;
-       struct c_slot   cs_dummy;
-       c_slot_t cs  = &cs_dummy;
-       int             c_segment_min_size;
-       int             c_segment_padded_size;
        int             attempts = 1;
        kern_return_t   retval = KERN_SUCCESS;
        vm_offset_t     start_addr = 0;
        int             attempts = 1;
        kern_return_t   retval = KERN_SUCCESS;
        vm_offset_t     start_addr = 0;
@@ -548,11 +587,11 @@ vm_compressor_init(void)
        vm_size_t       c_compressed_record_sbuf_size = 0;
 #endif /* RECORD_THE_COMPRESSED_DATA */
 
        vm_size_t       c_compressed_record_sbuf_size = 0;
 #endif /* RECORD_THE_COMPRESSED_DATA */
 
-#if DEVELOPMENT || DEBUG
+#if DEVELOPMENT || DEBUG || CONFIG_FREEZE
        char bootarg_name[32];
        char bootarg_name[32];
-       if (PE_parse_boot_argn("-kill_on_no_paging_space", bootarg_name, sizeof(bootarg_name))) {
-               kill_on_no_paging_space = TRUE;
-       }
+#endif /* DEVELOPMENT || DEBUG || CONFIG_FREEZE */
+
+#if DEVELOPMENT || DEBUG
        if (PE_parse_boot_argn("-disable_cseg_write_protection", bootarg_name, sizeof(bootarg_name))) {
                write_protect_c_segs = FALSE;
        }
        if (PE_parse_boot_argn("-disable_cseg_write_protection", bootarg_name, sizeof(bootarg_name))) {
                write_protect_c_segs = FALSE;
        }
@@ -576,33 +615,20 @@ vm_compressor_init(void)
        }
 #endif /* DEVELOPMENT || DEBUG */
 
        }
 #endif /* DEVELOPMENT || DEBUG */
 
-       /*
-        * ensure that any pointer that gets created from
-        * the vm_page zone can be packed properly
-        */
-       cs->c_packed_ptr = C_SLOT_PACK_PTR(zone_map_min_address);
-
-       if (C_SLOT_UNPACK_PTR(cs) != (uintptr_t)zone_map_min_address) {
-               panic("C_SLOT_UNPACK_PTR failed on zone_map_min_address - %p", (void *)zone_map_min_address);
-       }
-
-       cs->c_packed_ptr = C_SLOT_PACK_PTR(zone_map_max_address);
-
-       if (C_SLOT_UNPACK_PTR(cs) != (uintptr_t)zone_map_max_address) {
-               panic("C_SLOT_UNPACK_PTR failed on zone_map_max_address - %p", (void *)zone_map_max_address);
+#if CONFIG_FREEZE
+       if (PE_parse_boot_argn("-disable_freezer_cseg_acct", bootarg_name, sizeof(bootarg_name))) {
+               freezer_incore_cseg_acct = FALSE;
        }
        }
-
+#endif /* CONFIG_FREEZE */
 
        assert((C_SEGMENTS_PER_PAGE * sizeof(union c_segu)) == PAGE_SIZE);
 
 
        assert((C_SEGMENTS_PER_PAGE * sizeof(union c_segu)) == PAGE_SIZE);
 
-       PE_parse_boot_argn("vm_compression_limit", &vm_compression_limit, sizeof(vm_compression_limit));
-
-#ifdef CONFIG_EMBEDDED
+#if !XNU_TARGET_OS_OSX
        vm_compressor_minorcompact_threshold_divisor = 20;
        vm_compressor_majorcompact_threshold_divisor = 30;
        vm_compressor_unthrottle_threshold_divisor = 40;
        vm_compressor_catchup_threshold_divisor = 60;
        vm_compressor_minorcompact_threshold_divisor = 20;
        vm_compressor_majorcompact_threshold_divisor = 30;
        vm_compressor_unthrottle_threshold_divisor = 40;
        vm_compressor_catchup_threshold_divisor = 60;
-#else
+#else /* !XNU_TARGET_OS_OSX */
        if (max_mem <= (3ULL * 1024ULL * 1024ULL * 1024ULL)) {
                vm_compressor_minorcompact_threshold_divisor = 11;
                vm_compressor_majorcompact_threshold_divisor = 13;
        if (max_mem <= (3ULL * 1024ULL * 1024ULL * 1024ULL)) {
                vm_compressor_minorcompact_threshold_divisor = 11;
                vm_compressor_majorcompact_threshold_divisor = 13;
@@ -614,14 +640,7 @@ vm_compressor_init(void)
                vm_compressor_unthrottle_threshold_divisor = 35;
                vm_compressor_catchup_threshold_divisor = 50;
        }
                vm_compressor_unthrottle_threshold_divisor = 35;
                vm_compressor_catchup_threshold_divisor = 50;
        }
-#endif
-       /*
-        * vm_page_init_lck_grp is now responsible for calling vm_compressor_init_locks
-        * c_master_lock needs to be available early so that "vm_page_find_contiguous" can
-        * use PAGE_REPLACEMENT_ALLOWED to coordinate with the compressor.
-        */
-
-       c_list_lock = lck_mtx_alloc_init(&vm_compressor_lck_grp, &vm_compressor_lck_attr);
+#endif /* !XNU_TARGET_OS_OSX */
 
        queue_init(&c_bad_list_head);
        queue_init(&c_age_list_head);
 
        queue_init(&c_bad_list_head);
        queue_init(&c_age_list_head);
@@ -638,13 +657,13 @@ vm_compressor_init(void)
        c_segments_available = 0;
 
        if (vm_compression_limit) {
        c_segments_available = 0;
 
        if (vm_compression_limit) {
-               compressor_pool_size = (uint64_t)vm_compression_limit * PAGE_SIZE_64;
+               compressor_pool_size = ptoa_64(vm_compression_limit);
        }
 
        compressor_pool_max_size = C_SEG_MAX_LIMIT;
        compressor_pool_max_size *= C_SEG_BUFSIZE;
 
        }
 
        compressor_pool_max_size = C_SEG_MAX_LIMIT;
        compressor_pool_max_size *= C_SEG_BUFSIZE;
 
-#if !CONFIG_EMBEDDED
+#if XNU_TARGET_OS_OSX
 
        if (vm_compression_limit == 0) {
                if (max_mem <= (4ULL * 1024ULL * 1024ULL * 1024ULL)) {
 
        if (vm_compression_limit == 0) {
                if (max_mem <= (4ULL * 1024ULL * 1024ULL * 1024ULL)) {
@@ -678,7 +697,29 @@ vm_compressor_init(void)
                compressor_pool_size = ((kernel_map->max_offset - kernel_map->min_offset) - kernel_map->size) - VM_RESERVE_SIZE;
        }
        compressor_pool_multiplier = 1;
                compressor_pool_size = ((kernel_map->max_offset - kernel_map->min_offset) - kernel_map->size) - VM_RESERVE_SIZE;
        }
        compressor_pool_multiplier = 1;
+
+#elif defined(__arm64__) && defined(XNU_TARGET_OS_WATCH)
+
+       /*
+        * On M9 watches the compressor can become big and can lead to
+        * churn in workingset resulting in audio drops. Setting a cap
+        * on the compressor size favors reclaiming unused memory
+        * sitting in idle band via jetsams
+        */
+
+#define COMPRESSOR_CAP_PERCENTAGE        37ULL
+
+       if (compressor_pool_max_size > max_mem) {
+               compressor_pool_max_size = max_mem;
+       }
+
+       if (vm_compression_limit == 0) {
+               compressor_pool_size = (max_mem * COMPRESSOR_CAP_PERCENTAGE) / 100ULL;
+       }
+       compressor_pool_multiplier = 1;
+
 #else
 #else
+
        if (compressor_pool_max_size > max_mem) {
                compressor_pool_max_size = max_mem;
        }
        if (compressor_pool_max_size > max_mem) {
                compressor_pool_max_size = max_mem;
        }
@@ -699,11 +740,28 @@ try_again:
        c_segment_pages_compressed_limit = (c_segments_limit * (C_SEG_BUFSIZE / PAGE_SIZE) * compressor_pool_multiplier);
 
        if (c_segment_pages_compressed_limit < (uint32_t)(max_mem / PAGE_SIZE)) {
        c_segment_pages_compressed_limit = (c_segments_limit * (C_SEG_BUFSIZE / PAGE_SIZE) * compressor_pool_multiplier);
 
        if (c_segment_pages_compressed_limit < (uint32_t)(max_mem / PAGE_SIZE)) {
-               c_segment_pages_compressed_limit = (uint32_t)(max_mem / PAGE_SIZE);
+               if (!vm_compression_limit) {
+                       c_segment_pages_compressed_limit = (uint32_t)(max_mem / PAGE_SIZE);
+               }
        }
 
        c_segment_pages_compressed_nearing_limit = (uint32_t)(((uint64_t)c_segment_pages_compressed_limit * 98ULL) / 100ULL);
 
        }
 
        c_segment_pages_compressed_nearing_limit = (uint32_t)(((uint64_t)c_segment_pages_compressed_limit * 98ULL) / 100ULL);
 
+#if CONFIG_FREEZE
+       /*
+        * Our in-core limits are based on the size of the compressor pool.
+        * The c_segments_nearing_limit is also based on the compressor pool
+        * size and calculated above.
+        */
+       c_segments_incore_limit = c_segments_limit;
+
+       if (freezer_incore_cseg_acct) {
+               /*
+                * Add enough segments to track all frozen c_segs that can be stored in swap.
+                */
+               c_segments_limit += (uint32_t)(vm_swap_get_max_configured_space() / (vm_size_t)(C_SEG_ALLOCSIZE));
+       }
+#endif
        /*
         * Submap needs space for:
         * - c_segments
        /*
         * Submap needs space for:
         * - c_segments
@@ -736,25 +794,39 @@ try_again:
                kprintf("retrying creation of the compressor submap at 0x%llx bytes\n", compressor_pool_size);
                goto try_again;
        }
                kprintf("retrying creation of the compressor submap at 0x%llx bytes\n", compressor_pool_size);
                goto try_again;
        }
-       if (kernel_memory_allocate(compressor_map, (vm_offset_t *)(&c_segments), (sizeof(union c_segu) * c_segments_limit), 0, KMA_KOBJECT | KMA_VAONLY | KMA_PERMANENT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS) {
+       if (kernel_memory_allocate(compressor_map, (vm_offset_t *)(&c_segments),
+           (sizeof(union c_segu) * c_segments_limit), 0,
+           KMA_KOBJECT | KMA_VAONLY | KMA_PERMANENT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS) {
                panic("vm_compressor_init: kernel_memory_allocate failed - c_segments\n");
        }
                panic("vm_compressor_init: kernel_memory_allocate failed - c_segments\n");
        }
-       if (kernel_memory_allocate(compressor_map, &c_buffers, c_buffers_size, 0, KMA_COMPRESSOR | KMA_VAONLY | KMA_PERMANENT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS) {
+       if (kernel_memory_allocate(compressor_map, &c_buffers, c_buffers_size, 0,
+           KMA_COMPRESSOR | KMA_VAONLY | KMA_PERMANENT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS) {
                panic("vm_compressor_init: kernel_memory_allocate failed - c_buffers\n");
        }
 
 
                panic("vm_compressor_init: kernel_memory_allocate failed - c_buffers\n");
        }
 
 
-       c_segment_min_size = sizeof(struct c_segment) + (C_SEG_SLOT_VAR_ARRAY_MIN_LEN * sizeof(struct c_slot));
+       /*
+        * Pick a good size that will minimize fragmentation in zalloc
+        * by minimizing the fragmentation in a 16k run.
+        *
+        * C_SEG_SLOT_VAR_ARRAY_MIN_LEN is larger on 4k systems than 16k ones,
+        * making the fragmentation in a 4k page terrible. Using 16k for all
+        * systems matches zalloc() and will minimize fragmentation.
+        */
+       uint32_t c_segment_size = sizeof(struct c_segment) + (C_SEG_SLOT_VAR_ARRAY_MIN_LEN * sizeof(struct c_slot));
+       uint32_t cnt  = (16 << 10) / c_segment_size;
+       uint32_t frag = (16 << 10) % c_segment_size;
 
 
-       for (c_segment_padded_size = 128; c_segment_padded_size < c_segment_min_size; c_segment_padded_size = c_segment_padded_size << 1) {
-               ;
-       }
+       c_seg_fixed_array_len = C_SEG_SLOT_VAR_ARRAY_MIN_LEN;
 
 
-       compressor_segment_zone = zinit(c_segment_padded_size, c_segments_limit * c_segment_padded_size, PAGE_SIZE, "compressor_segment");
-       zone_change(compressor_segment_zone, Z_CALLERACCT, FALSE);
-       zone_change(compressor_segment_zone, Z_NOENCRYPT, TRUE);
+       while (cnt * sizeof(struct c_slot) < frag) {
+               c_segment_size += sizeof(struct c_slot);
+               c_seg_fixed_array_len++;
+               frag -= cnt * sizeof(struct c_slot);
+       }
 
 
-       c_seg_fixed_array_len = (c_segment_padded_size - sizeof(struct c_segment)) / sizeof(struct c_slot);
+       compressor_segment_zone = zone_create("compressor_segment",
+           c_segment_size, ZC_NOENCRYPT | ZC_ZFREE_CLEARMEM);
 
        c_segments_busy = FALSE;
 
 
        c_segments_busy = FALSE;
 
@@ -764,41 +836,62 @@ try_again:
        {
                host_basic_info_data_t hinfo;
                mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
        {
                host_basic_info_data_t hinfo;
                mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
+               size_t bufsize;
+               char *buf;
 
 #define BSD_HOST 1
                host_info((host_t)BSD_HOST, HOST_BASIC_INFO, (host_info_t)&hinfo, &count);
 
                compressor_cpus = hinfo.max_cpus;
 
 #define BSD_HOST 1
                host_info((host_t)BSD_HOST, HOST_BASIC_INFO, (host_info_t)&hinfo, &count);
 
                compressor_cpus = hinfo.max_cpus;
-               compressor_scratch_bufs = kalloc_tag(compressor_cpus * vm_compressor_get_decode_scratch_size(), VM_KERN_MEMORY_COMPRESSOR);
 
 
-               kdp_compressor_scratch_buf = kalloc_tag(vm_compressor_get_decode_scratch_size(), VM_KERN_MEMORY_COMPRESSOR);
+               bufsize = PAGE_SIZE;
+               bufsize += compressor_cpus * vm_compressor_get_decode_scratch_size();
+               bufsize += vm_compressor_get_decode_scratch_size();
+#if CONFIG_FREEZE
+               bufsize += vm_compressor_get_encode_scratch_size();
+#endif
+#if RECORD_THE_COMPRESSED_DATA
+               bufsize += c_compressed_record_sbuf_size;
+#endif
+
+               if (kernel_memory_allocate(kernel_map, (vm_offset_t *)&buf, bufsize,
+                   PAGE_MASK, KMA_KOBJECT | KMA_PERMANENT, VM_KERN_MEMORY_COMPRESSOR)) {
+                       panic("vm_compressor_init: Unable to allocate %zd bytes", bufsize);
+               }
 
                /*
                 * kdp_compressor_decompressed_page must be page aligned because we access
 
                /*
                 * kdp_compressor_decompressed_page must be page aligned because we access
-                * it through the physical apperture by page number. kalloc() does not
-                * guarantee alignment.
+                * it through the physical apperture by page number.
                 */
                 */
-               vm_offset_t addr;
-               if (kernel_memory_allocate(kernel_map, &addr, PAGE_SIZE, 0, KMA_KOBJECT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS) {
-                       panic("vm_compressor_init: kernel_memory_allocate failed - kdp_compressor_decompressed_page\n");
-               }
-               assert((addr & PAGE_MASK) == 0);
-               kdp_compressor_decompressed_page = (void *)addr;
+               kdp_compressor_decompressed_page = buf;
                kdp_compressor_decompressed_page_paddr = kvtophys((vm_offset_t)kdp_compressor_decompressed_page);
                kdp_compressor_decompressed_page_ppnum = (ppnum_t) atop(kdp_compressor_decompressed_page_paddr);
                kdp_compressor_decompressed_page_paddr = kvtophys((vm_offset_t)kdp_compressor_decompressed_page);
                kdp_compressor_decompressed_page_ppnum = (ppnum_t) atop(kdp_compressor_decompressed_page_paddr);
-       }
+               buf += PAGE_SIZE;
+               bufsize -= PAGE_SIZE;
+
+               compressor_scratch_bufs = buf;
+               buf += compressor_cpus * vm_compressor_get_decode_scratch_size();
+               bufsize -= compressor_cpus * vm_compressor_get_decode_scratch_size();
+
+               kdp_compressor_scratch_buf = buf;
+               buf += vm_compressor_get_decode_scratch_size();
+               bufsize -= vm_compressor_get_decode_scratch_size();
+
 #if CONFIG_FREEZE
 #if CONFIG_FREEZE
-       freezer_compressor_scratch_buf = kalloc_tag(vm_compressor_get_encode_scratch_size(), VM_KERN_MEMORY_COMPRESSOR);
+               freezer_context_global.freezer_ctx_compressor_scratch_buf = buf;
+               buf += vm_compressor_get_encode_scratch_size();
+               bufsize -= vm_compressor_get_encode_scratch_size();
 #endif
 
 #if RECORD_THE_COMPRESSED_DATA
 #endif
 
 #if RECORD_THE_COMPRESSED_DATA
-       if (kernel_memory_allocate(compressor_map, (vm_offset_t *)&c_compressed_record_sbuf, c_compressed_record_sbuf_size, 0, KMA_KOBJECT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS) {
-               panic("vm_compressor_init: kernel_memory_allocate failed - c_compressed_record_sbuf\n");
-       }
-
-       c_compressed_record_cptr = c_compressed_record_sbuf;
-       c_compressed_record_ebuf = c_compressed_record_sbuf + c_compressed_record_sbuf_size;
+               c_compressed_record_sbuf = buf;
+               c_compressed_record_cptr = buf;
+               c_compressed_record_ebuf = c_compressed_record_sbuf + c_compressed_record_sbuf_size;
+               buf += c_compressed_record_sbuf_size;
+               bufsize -= c_compressed_record_sbuf_size;
 #endif
 #endif
+               assert(bufsize == 0);
+       }
 
        if (kernel_thread_start_priority((thread_continue_t)vm_compressor_swap_trigger_thread, NULL,
            BASEPRI_VM, &thread) != KERN_SUCCESS) {
 
        if (kernel_thread_start_priority((thread_continue_t)vm_compressor_swap_trigger_thread, NULL,
            BASEPRI_VM, &thread) != KERN_SUCCESS) {
@@ -832,7 +925,7 @@ try_again:
 static void
 c_seg_validate(c_segment_t c_seg, boolean_t must_be_compact)
 {
 static void
 c_seg_validate(c_segment_t c_seg, boolean_t must_be_compact)
 {
-       int             c_indx;
+       uint16_t        c_indx;
        int32_t         bytes_used;
        uint32_t        c_rounded_size;
        uint32_t        c_size;
        int32_t         bytes_used;
        uint32_t        c_rounded_size;
        uint32_t        c_size;
@@ -1074,31 +1167,156 @@ c_seg_do_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy, bo
        return c_seg_freed;
 }
 
        return c_seg_freed;
 }
 
+void
+kdp_compressor_busy_find_owner(event64_t wait_event, thread_waitinfo_t *waitinfo)
+{
+       c_segment_t c_seg = (c_segment_t) wait_event;
+
+       waitinfo->owner = thread_tid(c_seg->c_busy_for_thread);
+       waitinfo->context = VM_KERNEL_UNSLIDE_OR_PERM(c_seg);
+}
+
+#if DEVELOPMENT || DEBUG
+int
+do_cseg_wedge_thread(void)
+{
+       struct c_segment c_seg;
+       c_seg.c_busy_for_thread = current_thread();
+
+       debug_cseg_wait_event = (event_t) &c_seg;
+
+       thread_set_pending_block_hint(current_thread(), kThreadWaitCompressor);
+       assert_wait((event_t) (&c_seg), THREAD_INTERRUPTIBLE);
+
+       thread_block(THREAD_CONTINUE_NULL);
+
+       return 0;
+}
+
+int
+do_cseg_unwedge_thread(void)
+{
+       thread_wakeup(debug_cseg_wait_event);
+       debug_cseg_wait_event = NULL;
+
+       return 0;
+}
+#endif /* DEVELOPMENT || DEBUG */
 
 void
 c_seg_wait_on_busy(c_segment_t c_seg)
 {
        c_seg->c_wanted = 1;
 
 void
 c_seg_wait_on_busy(c_segment_t c_seg)
 {
        c_seg->c_wanted = 1;
+
+       thread_set_pending_block_hint(current_thread(), kThreadWaitCompressor);
        assert_wait((event_t) (c_seg), THREAD_UNINT);
 
        lck_mtx_unlock_always(&c_seg->c_lock);
        thread_block(THREAD_CONTINUE_NULL);
 }
 
        assert_wait((event_t) (c_seg), THREAD_UNINT);
 
        lck_mtx_unlock_always(&c_seg->c_lock);
        thread_block(THREAD_CONTINUE_NULL);
 }
 
+#if CONFIG_FREEZE
+/*
+ * We don't have the task lock held while updating the task's
+ * c_seg queues. We can do that because of the following restrictions:
+ *
+ * - SINGLE FREEZER CONTEXT:
+ *   We 'insert' c_segs into the task list on the task_freeze path.
+ *   There can only be one such freeze in progress and the task
+ *   isn't disappearing because we have the VM map lock held throughout
+ *   and we have a reference on the proc too.
+ *
+ * - SINGLE TASK DISOWN CONTEXT:
+ *   We 'disown' c_segs of a task ONLY from the task_terminate context. So
+ *   we don't need the task lock but we need the c_list_lock and the
+ *   compressor master lock (shared). We also hold the individual
+ *   c_seg locks (exclusive).
+ *
+ *   If we either:
+ *   - can't get the c_seg lock on a try, then we start again because maybe
+ *   the c_seg is part of a compaction and might get freed. So we can't trust
+ *   that linkage and need to restart our queue traversal.
+ *   - OR, we run into a busy c_seg (say being swapped in or free-ing) we
+ *   drop all locks again and wait and restart our queue traversal.
+ *
+ * - The new_owner_task below is currently only the kernel or NULL.
+ *
+ */
+void
+c_seg_update_task_owner(c_segment_t c_seg, task_t new_owner_task)
+{
+       task_t          owner_task = c_seg->c_task_owner;
+       uint64_t        uncompressed_bytes = ((c_seg->c_slots_used) * PAGE_SIZE_64);
+
+       LCK_MTX_ASSERT(c_list_lock, LCK_MTX_ASSERT_OWNED);
+       LCK_MTX_ASSERT(&c_seg->c_lock, LCK_MTX_ASSERT_OWNED);
+
+       if (owner_task) {
+               task_update_frozen_to_swap_acct(owner_task, uncompressed_bytes, DEBIT_FROM_SWAP);
+               queue_remove(&owner_task->task_frozen_cseg_q, c_seg,
+                   c_segment_t, c_task_list_next_cseg);
+       }
+
+       if (new_owner_task) {
+               queue_enter(&new_owner_task->task_frozen_cseg_q, c_seg,
+                   c_segment_t, c_task_list_next_cseg);
+               task_update_frozen_to_swap_acct(new_owner_task, uncompressed_bytes, CREDIT_TO_SWAP);
+       }
+
+       c_seg->c_task_owner = new_owner_task;
+}
+
+void
+task_disown_frozen_csegs(task_t owner_task)
+{
+       c_segment_t c_seg = NULL, next_cseg = NULL;
+
+again:
+       PAGE_REPLACEMENT_DISALLOWED(TRUE);
+       lck_mtx_lock_spin_always(c_list_lock);
+
+       for (c_seg = (c_segment_t) queue_first(&owner_task->task_frozen_cseg_q);
+           !queue_end(&owner_task->task_frozen_cseg_q, (queue_entry_t) c_seg);
+           c_seg = next_cseg) {
+               next_cseg = (c_segment_t) queue_next(&c_seg->c_task_list_next_cseg);;
+
+               if (!lck_mtx_try_lock_spin_always(&c_seg->c_lock)) {
+                       lck_mtx_unlock(c_list_lock);
+                       PAGE_REPLACEMENT_DISALLOWED(FALSE);
+                       goto again;
+               }
+
+               if (c_seg->c_busy) {
+                       lck_mtx_unlock(c_list_lock);
+                       PAGE_REPLACEMENT_DISALLOWED(FALSE);
+
+                       c_seg_wait_on_busy(c_seg);
+
+                       goto again;
+               }
+               assert(c_seg->c_task_owner == owner_task);
+               c_seg_update_task_owner(c_seg, kernel_task);
+               lck_mtx_unlock_always(&c_seg->c_lock);
+       }
+
+       lck_mtx_unlock(c_list_lock);
+       PAGE_REPLACEMENT_DISALLOWED(FALSE);
+}
+#endif /* CONFIG_FREEZE */
 
 void
 c_seg_switch_state(c_segment_t c_seg, int new_state, boolean_t insert_head)
 {
        int     old_state = c_seg->c_state;
 
 
 void
 c_seg_switch_state(c_segment_t c_seg, int new_state, boolean_t insert_head)
 {
        int     old_state = c_seg->c_state;
 
-#if !CONFIG_EMBEDDED
+#if XNU_TARGET_OS_OSX
 #if     DEVELOPMENT || DEBUG
        if (new_state != C_IS_FILLING) {
                LCK_MTX_ASSERT(&c_seg->c_lock, LCK_MTX_ASSERT_OWNED);
        }
        LCK_MTX_ASSERT(c_list_lock, LCK_MTX_ASSERT_OWNED);
 #endif
 #if     DEVELOPMENT || DEBUG
        if (new_state != C_IS_FILLING) {
                LCK_MTX_ASSERT(&c_seg->c_lock, LCK_MTX_ASSERT_OWNED);
        }
        LCK_MTX_ASSERT(c_list_lock, LCK_MTX_ASSERT_OWNED);
 #endif
-#endif /* !CONFIG_EMBEDDED */
+#endif /* XNU_TARGET_OS_OSX */
        switch (old_state) {
        case C_IS_EMPTY:
                assert(new_state == C_IS_FILLING || new_state == C_IS_FREE);
        switch (old_state) {
        case C_IS_EMPTY:
                assert(new_state == C_IS_FILLING || new_state == C_IS_FREE);
@@ -1131,6 +1349,12 @@ c_seg_switch_state(c_segment_t c_seg, int new_state, boolean_t insert_head)
        case C_ON_SWAPOUT_Q:
                assert(new_state == C_ON_AGE_Q || new_state == C_IS_FREE || new_state == C_IS_EMPTY || new_state == C_ON_SWAPIO_Q);
 
        case C_ON_SWAPOUT_Q:
                assert(new_state == C_ON_AGE_Q || new_state == C_IS_FREE || new_state == C_IS_EMPTY || new_state == C_ON_SWAPIO_Q);
 
+#if CONFIG_FREEZE
+               if (c_seg->c_task_owner && (new_state != C_ON_SWAPIO_Q)) {
+                       c_seg_update_task_owner(c_seg, NULL);
+               }
+#endif /* CONFIG_FREEZE */
+
                queue_remove(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
                thread_wakeup((event_t)&compaction_swapper_running);
                c_swapout_count--;
                queue_remove(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
                thread_wakeup((event_t)&compaction_swapper_running);
                c_swapout_count--;
@@ -1340,20 +1564,25 @@ c_seg_free_locked(c_segment_t c_seg)
 
        c_seg_switch_state(c_seg, C_IS_FREE, FALSE);
 
 
        c_seg_switch_state(c_seg, C_IS_FREE, FALSE);
 
-       lck_mtx_unlock_always(c_list_lock);
-
        if (c_buffer) {
                pages_populated = (round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE;
                c_seg->c_store.c_buffer = NULL;
        } else {
        if (c_buffer) {
                pages_populated = (round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE;
                c_seg->c_store.c_buffer = NULL;
        } else {
+#if CONFIG_FREEZE
+               c_seg_update_task_owner(c_seg, NULL);
+#endif /* CONFIG_FREEZE */
+
                c_seg->c_store.c_swap_handle = (uint64_t)-1;
        }
 
        lck_mtx_unlock_always(&c_seg->c_lock);
 
                c_seg->c_store.c_swap_handle = (uint64_t)-1;
        }
 
        lck_mtx_unlock_always(&c_seg->c_lock);
 
+       lck_mtx_unlock_always(c_list_lock);
+
        if (c_buffer) {
                if (pages_populated) {
        if (c_buffer) {
                if (pages_populated) {
-                       kernel_memory_depopulate(compressor_map, (vm_offset_t) c_buffer, pages_populated * PAGE_SIZE, KMA_COMPRESSOR);
+                       kernel_memory_depopulate(compressor_map, (vm_offset_t)c_buffer,
+                           pages_populated * PAGE_SIZE, KMA_COMPRESSOR, VM_KERN_MEMORY_COMPRESSOR);
                }
        } else if (c_swap_handle) {
                /*
                }
        } else if (c_swap_handle) {
                /*
@@ -1388,7 +1617,8 @@ c_seg_free_locked(c_segment_t c_seg)
        lck_mtx_destroy(&c_seg->c_lock, &vm_compressor_lck_grp);
 
        if (c_seg->c_slot_var_array_len) {
        lck_mtx_destroy(&c_seg->c_lock, &vm_compressor_lck_grp);
 
        if (c_seg->c_slot_var_array_len) {
-               kfree(c_seg->c_slot_var_array, sizeof(struct c_slot) * c_seg->c_slot_var_array_len);
+               kheap_free(KHEAP_DATA_BUFFERS, c_seg->c_slot_var_array,
+                   sizeof(struct c_slot) * c_seg->c_slot_var_array_len);
        }
 
        zfree(compressor_segment_zone, c_seg);
        }
 
        zfree(compressor_segment_zone, c_seg);
@@ -1453,7 +1683,7 @@ c_seg_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy)
        uint32_t        old_populated_offset;
        uint32_t        c_rounded_size;
        uint32_t        c_size;
        uint32_t        old_populated_offset;
        uint32_t        c_rounded_size;
        uint32_t        c_size;
-       int             c_indx = 0;
+       uint16_t        c_indx = 0;
        int             i;
        c_slot_t        c_dst;
        c_slot_t        c_src;
        int             i;
        c_slot_t        c_dst;
        c_slot_t        c_src;
@@ -1504,7 +1734,7 @@ c_seg_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy)
                cslot_copy(c_dst, c_src);
                c_dst->c_offset = c_offset;
 
                cslot_copy(c_dst, c_src);
                c_dst->c_offset = c_offset;
 
-               slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
+               slot_ptr = C_SLOT_UNPACK_PTR(c_dst);
                slot_ptr->s_cindx = c_indx;
 
                c_offset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
                slot_ptr->s_cindx = c_indx;
 
                c_offset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
@@ -1529,7 +1759,8 @@ c_seg_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy)
                gc_size = C_SEG_OFFSET_TO_BYTES(old_populated_offset - c_seg->c_populated_offset);
                gc_ptr = &c_seg->c_store.c_buffer[c_seg->c_populated_offset];
 
                gc_size = C_SEG_OFFSET_TO_BYTES(old_populated_offset - c_seg->c_populated_offset);
                gc_ptr = &c_seg->c_store.c_buffer[c_seg->c_populated_offset];
 
-               kernel_memory_depopulate(compressor_map, (vm_offset_t)gc_ptr, gc_size, KMA_COMPRESSOR);
+               kernel_memory_depopulate(compressor_map, (vm_offset_t)gc_ptr, gc_size,
+                   KMA_COMPRESSOR, VM_KERN_MEMORY_COMPRESSOR);
        }
 
 #if DEVELOPMENT || DEBUG
        }
 
 #if DEVELOPMENT || DEBUG
@@ -1568,12 +1799,14 @@ c_seg_alloc_nextslot(c_segment_t c_seg)
                        newlen = oldlen * 2;
                }
 
                        newlen = oldlen * 2;
                }
 
-               new_slot_array = (struct c_slot *)kalloc(sizeof(struct c_slot) * newlen);
+               new_slot_array = kheap_alloc(KHEAP_DATA_BUFFERS,
+                   sizeof(struct c_slot) * newlen, Z_WAITOK);
 
                lck_mtx_lock_spin_always(&c_seg->c_lock);
 
                if (old_slot_array) {
 
                lck_mtx_lock_spin_always(&c_seg->c_lock);
 
                if (old_slot_array) {
-                       memcpy((char *)new_slot_array, (char *)old_slot_array, sizeof(struct c_slot) * oldlen);
+                       memcpy(new_slot_array, old_slot_array,
+                           sizeof(struct c_slot) * oldlen);
                }
 
                c_seg->c_slot_var_array_len = newlen;
                }
 
                c_seg->c_slot_var_array_len = newlen;
@@ -1582,12 +1815,14 @@ c_seg_alloc_nextslot(c_segment_t c_seg)
                lck_mtx_unlock_always(&c_seg->c_lock);
 
                if (old_slot_array) {
                lck_mtx_unlock_always(&c_seg->c_lock);
 
                if (old_slot_array) {
-                       kfree(old_slot_array, sizeof(struct c_slot) * oldlen);
+                       kheap_free(KHEAP_DATA_BUFFERS, old_slot_array,
+                           sizeof(struct c_slot) * oldlen);
                }
        }
 }
 
 
                }
        }
 }
 
 
+#define C_SEG_MAJOR_COMPACT_STATS_MAX   (30)
 
 struct {
        uint64_t asked_permission;
 
 struct {
        uint64_t asked_permission;
@@ -1597,7 +1832,11 @@ struct {
        uint64_t wasted_space_in_swapouts;
        uint64_t count_of_swapouts;
        uint64_t count_of_freed_segs;
        uint64_t wasted_space_in_swapouts;
        uint64_t count_of_swapouts;
        uint64_t count_of_freed_segs;
-} c_seg_major_compact_stats;
+       uint64_t bailed_compactions;
+       uint64_t bytes_freed_rate_us;
+} c_seg_major_compact_stats[C_SEG_MAJOR_COMPACT_STATS_MAX];
+
+int c_seg_major_compact_stats_now = 0;
 
 
 #define C_MAJOR_COMPACTION_SIZE_APPROPRIATE     ((C_SEG_BUFSIZE * 90) / 100)
 
 
 #define C_MAJOR_COMPACTION_SIZE_APPROPRIATE     ((C_SEG_BUFSIZE * 90) / 100)
@@ -1608,7 +1847,7 @@ c_seg_major_compact_ok(
        c_segment_t c_seg_dst,
        c_segment_t c_seg_src)
 {
        c_segment_t c_seg_dst,
        c_segment_t c_seg_src)
 {
-       c_seg_major_compact_stats.asked_permission++;
+       c_seg_major_compact_stats[c_seg_major_compact_stats_now].asked_permission++;
 
        if (c_seg_src->c_bytes_used >= C_MAJOR_COMPACTION_SIZE_APPROPRIATE &&
            c_seg_dst->c_bytes_used >= C_MAJOR_COMPACTION_SIZE_APPROPRIATE) {
 
        if (c_seg_src->c_bytes_used >= C_MAJOR_COMPACTION_SIZE_APPROPRIATE &&
            c_seg_dst->c_bytes_used >= C_MAJOR_COMPACTION_SIZE_APPROPRIATE) {
@@ -1655,7 +1894,7 @@ c_seg_major_compact(
        c_seg_dst->c_was_major_compacted++;
        c_seg_src->c_was_major_donor++;
 #endif
        c_seg_dst->c_was_major_compacted++;
        c_seg_src->c_was_major_donor++;
 #endif
-       c_seg_major_compact_stats.compactions++;
+       c_seg_major_compact_stats[c_seg_major_compact_stats_now].compactions++;
 
        dst_slot = c_seg_dst->c_nextslot;
 
 
        dst_slot = c_seg_dst->c_nextslot;
 
@@ -1701,8 +1940,8 @@ c_seg_major_compact(
 
                c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
 
 
                c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
 
-               c_seg_major_compact_stats.moved_slots++;
-               c_seg_major_compact_stats.moved_bytes += c_size;
+               c_seg_major_compact_stats[c_seg_major_compact_stats_now].moved_slots++;
+               c_seg_major_compact_stats[c_seg_major_compact_stats_now].moved_bytes += c_size;
 
                cslot_copy(c_dst, c_src);
                c_dst->c_offset = c_seg_dst->c_nextoffset;
 
                cslot_copy(c_dst, c_src);
                c_dst->c_offset = c_seg_dst->c_nextoffset;
@@ -1745,7 +1984,7 @@ c_seg_major_compact(
                while (dst_slot < c_seg_dst->c_nextslot) {
                        c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, dst_slot);
 
                while (dst_slot < c_seg_dst->c_nextslot) {
                        c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, dst_slot);
 
-                       slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
+                       slot_ptr = C_SLOT_UNPACK_PTR(c_dst);
                        /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
                        slot_ptr->s_cseg = c_seg_dst->c_mysegno + 1;
                        slot_ptr->s_cindx = dst_slot++;
                        /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
                        slot_ptr->s_cseg = c_seg_dst->c_mysegno + 1;
                        slot_ptr->s_cindx = dst_slot++;
@@ -1938,20 +2177,31 @@ compressor_needs_to_swap(void)
                lck_mtx_unlock_always(c_list_lock);
 
                if (age >= vm_ripe_target_age) {
                lck_mtx_unlock_always(c_list_lock);
 
                if (age >= vm_ripe_target_age) {
-                       return TRUE;
+                       should_swap = TRUE;
+                       goto check_if_low_space;
                }
        }
        if (VM_CONFIG_SWAP_IS_ACTIVE) {
                if (COMPRESSOR_NEEDS_TO_SWAP()) {
                }
        }
        if (VM_CONFIG_SWAP_IS_ACTIVE) {
                if (COMPRESSOR_NEEDS_TO_SWAP()) {
-                       return TRUE;
+                       should_swap = TRUE;
+                       goto check_if_low_space;
                }
                if (VM_PAGE_Q_THROTTLED(&vm_pageout_queue_external) && vm_page_anonymous_count < (vm_page_inactive_count / 20)) {
                }
                if (VM_PAGE_Q_THROTTLED(&vm_pageout_queue_external) && vm_page_anonymous_count < (vm_page_inactive_count / 20)) {
-                       return TRUE;
+                       should_swap = TRUE;
+                       goto check_if_low_space;
                }
                if (vm_page_free_count < (vm_page_free_reserved - (COMPRESSOR_FREE_RESERVED_LIMIT * 2))) {
                }
                if (vm_page_free_count < (vm_page_free_reserved - (COMPRESSOR_FREE_RESERVED_LIMIT * 2))) {
-                       return TRUE;
+                       should_swap = TRUE;
+                       goto check_if_low_space;
                }
        }
                }
        }
+
+#if (XNU_TARGET_OS_OSX && __arm64__)
+       /*
+        * Thrashing detection disabled.
+        */
+#else /* (XNU_TARGET_OS_OSX && __arm64__) */
+
        compute_swapout_target_age();
 
        if (swapout_target_age) {
        compute_swapout_target_age();
 
        if (swapout_target_age) {
@@ -1976,20 +2226,22 @@ compressor_needs_to_swap(void)
        if (swapout_target_age) {
                should_swap = TRUE;
        }
        if (swapout_target_age) {
                should_swap = TRUE;
        }
+#endif /* (XNU_TARGET_OS_OSX && __arm64__) */
+
+check_if_low_space:
 
 #if CONFIG_JETSAM
        if (should_swap || vm_compressor_low_on_space() == TRUE) {
                if (vm_compressor_thrashing_detected == FALSE) {
                        vm_compressor_thrashing_detected = TRUE;
 
 
 #if CONFIG_JETSAM
        if (should_swap || vm_compressor_low_on_space() == TRUE) {
                if (vm_compressor_thrashing_detected == FALSE) {
                        vm_compressor_thrashing_detected = TRUE;
 
-                       if (swapout_target_age || vm_compressor_low_on_space() == TRUE) {
-                               if (swapout_target_age) {
-                                       /* The compressor is thrashing. */
-                                       memorystatus_kill_on_VM_compressor_thrashing(TRUE /* async */);
-                               } else {
-                                       /* The compressor is running low on space. */
-                                       memorystatus_kill_on_VM_compressor_space_shortage(TRUE /* async */);
-                               }
+                       if (swapout_target_age) {
+                               /* The compressor is thrashing. */
+                               memorystatus_kill_on_VM_compressor_thrashing(TRUE /* async */);
+                               compressor_thrashing_induced_jetsam++;
+                       } else if (vm_compressor_low_on_space() == TRUE) {
+                               /* The compressor is running low on space. */
+                               memorystatus_kill_on_VM_compressor_space_shortage(TRUE /* async */);
                                compressor_thrashing_induced_jetsam++;
                        } else {
                                memorystatus_kill_on_FC_thrashing(TRUE /* async */);
                                compressor_thrashing_induced_jetsam++;
                        } else {
                                memorystatus_kill_on_FC_thrashing(TRUE /* async */);
@@ -2007,6 +2259,10 @@ compressor_needs_to_swap(void)
                should_swap = FALSE;
        }
 
                should_swap = FALSE;
        }
 
+#else /* CONFIG_JETSAM */
+       if (should_swap && vm_swap_low_on_space()) {
+               vm_compressor_take_paging_space_action();
+       }
 #endif /* CONFIG_JETSAM */
 
        if (should_swap == FALSE) {
 #endif /* CONFIG_JETSAM */
 
        if (should_swap == FALSE) {
@@ -2254,9 +2510,11 @@ vm_compressor_do_delayed_compactions(boolean_t flush_all)
        boolean_t       needs_to_swap = FALSE;
 
 
        boolean_t       needs_to_swap = FALSE;
 
 
-#if !CONFIG_EMBEDDED
+       VM_DEBUG_CONSTANT_EVENT(vm_compressor_do_delayed_compactions, VM_COMPRESSOR_DO_DELAYED_COMPACTIONS, DBG_FUNC_START, c_minor_count, flush_all, 0, 0);
+
+#if XNU_TARGET_OS_OSX
        LCK_MTX_ASSERT(c_list_lock, LCK_MTX_ASSERT_OWNED);
        LCK_MTX_ASSERT(c_list_lock, LCK_MTX_ASSERT_OWNED);
-#endif /* !CONFIG_EMBEDDED */
+#endif /* XNU_TARGET_OS_OSX */
 
        while (!queue_empty(&c_minor_list_head) && needs_to_swap == FALSE) {
                c_seg = (c_segment_t)queue_first(&c_minor_list_head);
 
        while (!queue_empty(&c_minor_list_head) && needs_to_swap == FALSE) {
                c_seg = (c_segment_t)queue_first(&c_minor_list_head);
@@ -2283,6 +2541,8 @@ vm_compressor_do_delayed_compactions(boolean_t flush_all)
                }
                lck_mtx_lock_spin_always(c_list_lock);
        }
                }
                lck_mtx_lock_spin_always(c_list_lock);
        }
+
+       VM_DEBUG_CONSTANT_EVENT(vm_compressor_do_delayed_compactions, VM_COMPRESSOR_DO_DELAYED_COMPACTIONS, DBG_FUNC_END, c_minor_count, number_compacted, needs_to_swap, 0);
 }
 
 
 }
 
 
@@ -2412,7 +2672,7 @@ vm_compressor_swap_trigger_thread(void)
         * the tuneables until we are awakened via vm_pageout_scan
         * so that we are at a point where the vm_swapfile_open will
         * be operating on the correct directory (in case the default
         * the tuneables until we are awakened via vm_pageout_scan
         * so that we are at a point where the vm_swapfile_open will
         * be operating on the correct directory (in case the default
-        * of /var/vm/  is overridden by the dymanic_pager
+        * of using the VM volume is overridden by the dynamic_pager)
         */
        if (compaction_swapper_init_now) {
                vm_compaction_swapper_do_init();
         */
        if (compaction_swapper_init_now) {
                vm_compaction_swapper_do_init();
@@ -2420,6 +2680,9 @@ vm_compressor_swap_trigger_thread(void)
                if (vm_pageout_state.vm_restricted_to_single_processor == TRUE) {
                        thread_vm_bind_group_add();
                }
                if (vm_pageout_state.vm_restricted_to_single_processor == TRUE) {
                        thread_vm_bind_group_add();
                }
+#if CONFIG_THREAD_GROUPS
+               thread_group_vm_add();
+#endif
                thread_set_thread_name(current_thread(), "VM_cswap_trigger");
                compaction_swapper_init_now = 0;
        }
                thread_set_thread_name(current_thread(), "VM_cswap_trigger");
                compaction_swapper_init_now = 0;
        }
@@ -2624,15 +2887,22 @@ do_fastwake_warmup(queue_head_t *c_queue, boolean_t consider_all_cseg)
        }
 }
 
        }
 }
 
+int min_csegs_per_major_compaction = DELAYED_COMPACTIONS_PER_PASS;
+extern bool     vm_swapout_thread_running;
+extern boolean_t        compressor_store_stop_compaction;
 
 void
 vm_compressor_compact_and_swap(boolean_t flush_all)
 {
        c_segment_t     c_seg, c_seg_next;
 
 void
 vm_compressor_compact_and_swap(boolean_t flush_all)
 {
        c_segment_t     c_seg, c_seg_next;
-       boolean_t       keep_compacting;
+       boolean_t       keep_compacting, switch_state;
        clock_sec_t     now;
        clock_nsec_t    nsec;
        clock_sec_t     now;
        clock_nsec_t    nsec;
+       mach_timespec_t start_ts, end_ts;
+       unsigned int    number_considered, wanted_cseg_found, yield_after_considered_per_pass, number_yields;
+       uint64_t        bytes_to_free, bytes_freed, delta_usec;
 
 
+       VM_DEBUG_CONSTANT_EVENT(vm_compressor_compact_and_swap, VM_COMPRESSOR_COMPACT_AND_SWAP, DBG_FUNC_START, c_age_count, c_minor_count, c_major_count, vm_page_free_count);
 
        if (fastwake_warmup == TRUE) {
                uint64_t        starting_warmup_count;
 
        if (fastwake_warmup == TRUE) {
                uint64_t        starting_warmup_count;
@@ -2666,7 +2936,17 @@ vm_compressor_compact_and_swap(boolean_t flush_all)
         */
        clock_get_system_nanotime(&now, &nsec);
 
         */
        clock_get_system_nanotime(&now, &nsec);
 
-       while (!queue_empty(&c_age_list_head) && compaction_swapper_abort == 0) {
+       start_ts.tv_sec = (int) now;
+       start_ts.tv_nsec = nsec;
+       delta_usec = 0;
+       number_considered = 0;
+       wanted_cseg_found = 0;
+       number_yields = 0;
+       bytes_to_free = 0;
+       bytes_freed = 0;
+       yield_after_considered_per_pass = MAX(min_csegs_per_major_compaction, DELAYED_COMPACTIONS_PER_PASS);
+
+       while (!queue_empty(&c_age_list_head) && !compaction_swapper_abort && !compressor_store_stop_compaction) {
                if (hibernate_flushing == TRUE) {
                        clock_sec_t     sec;
 
                if (hibernate_flushing == TRUE) {
                        clock_sec_t     sec;
 
@@ -2694,11 +2974,17 @@ vm_compressor_compact_and_swap(boolean_t flush_all)
                                break;
                        }
                }
                                break;
                        }
                }
-               if (c_swapout_count >= C_SWAPOUT_LIMIT) {
+               if (!vm_swap_out_of_space() && c_swapout_count >= C_SWAPOUT_LIMIT) {
                        assert_wait_timeout((event_t) &compaction_swapper_running, THREAD_INTERRUPTIBLE, 100, 1000 * NSEC_PER_USEC);
 
                        assert_wait_timeout((event_t) &compaction_swapper_running, THREAD_INTERRUPTIBLE, 100, 1000 * NSEC_PER_USEC);
 
+                       if (!vm_swapout_thread_running) {
+                               thread_wakeup((event_t)&c_swapout_list_head);
+                       }
+
                        lck_mtx_unlock_always(c_list_lock);
 
                        lck_mtx_unlock_always(c_list_lock);
 
+                       VM_DEBUG_CONSTANT_EVENT(vm_compressor_compact_and_swap, VM_COMPRESSOR_COMPACT_AND_SWAP, DBG_FUNC_NONE, 1, c_swapout_count, 0, 0);
+
                        thread_block(THREAD_CONTINUE_NULL);
 
                        lck_mtx_lock_spin_always(c_list_lock);
                        thread_block(THREAD_CONTINUE_NULL);
 
                        lck_mtx_lock_spin_always(c_list_lock);
@@ -2710,7 +2996,7 @@ vm_compressor_compact_and_swap(boolean_t flush_all)
 
                vm_compressor_age_swapped_in_segments(flush_all);
 
 
                vm_compressor_age_swapped_in_segments(flush_all);
 
-               if (c_swapout_count >= C_SWAPOUT_LIMIT) {
+               if (!vm_swap_out_of_space() && c_swapout_count >= C_SWAPOUT_LIMIT) {
                        /*
                         * we timed out on the above thread_block
                         * let's loop around and try again
                        /*
                         * we timed out on the above thread_block
                         * let's loop around and try again
@@ -2718,6 +3004,8 @@ vm_compressor_compact_and_swap(boolean_t flush_all)
                         * to do minor compactions to make
                         * more memory available
                         */
                         * to do minor compactions to make
                         * more memory available
                         */
+                       VM_DEBUG_CONSTANT_EVENT(vm_compressor_compact_and_swap, VM_COMPRESSOR_COMPACT_AND_SWAP, DBG_FUNC_NONE, 2, c_swapout_count, 0, 0);
+
                        continue;
                }
 
                        continue;
                }
 
@@ -2731,19 +3019,16 @@ vm_compressor_compact_and_swap(boolean_t flush_all)
 
                        needs_to_swap = compressor_needs_to_swap();
 
 
                        needs_to_swap = compressor_needs_to_swap();
 
-#if !CONFIG_EMBEDDED
-                       if (needs_to_swap == TRUE && vm_swap_low_on_space()) {
-                               vm_compressor_take_paging_space_action();
-                       }
-#endif /* !CONFIG_EMBEDDED */
-
                        lck_mtx_lock_spin_always(c_list_lock);
 
                        lck_mtx_lock_spin_always(c_list_lock);
 
+                       VM_DEBUG_CONSTANT_EVENT(vm_compressor_compact_and_swap, VM_COMPRESSOR_COMPACT_AND_SWAP, DBG_FUNC_NONE, 3, needs_to_swap, 0, 0);
+
                        if (needs_to_swap == FALSE) {
                                break;
                        }
                }
                if (queue_empty(&c_age_list_head)) {
                        if (needs_to_swap == FALSE) {
                                break;
                        }
                }
                if (queue_empty(&c_age_list_head)) {
+                       VM_DEBUG_CONSTANT_EVENT(vm_compressor_compact_and_swap, VM_COMPRESSOR_COMPACT_AND_SWAP, DBG_FUNC_NONE, 4, c_age_count, 0, 0);
                        break;
                }
                c_seg = (c_segment_t) queue_first(&c_age_list_head);
                        break;
                }
                c_seg = (c_segment_t) queue_first(&c_age_list_head);
@@ -2751,12 +3036,15 @@ vm_compressor_compact_and_swap(boolean_t flush_all)
                assert(c_seg->c_state == C_ON_AGE_Q);
 
                if (flush_all == TRUE && c_seg->c_generation_id > c_generation_id_flush_barrier) {
                assert(c_seg->c_state == C_ON_AGE_Q);
 
                if (flush_all == TRUE && c_seg->c_generation_id > c_generation_id_flush_barrier) {
+                       VM_DEBUG_CONSTANT_EVENT(vm_compressor_compact_and_swap, VM_COMPRESSOR_COMPACT_AND_SWAP, DBG_FUNC_NONE, 5, 0, 0, 0);
                        break;
                }
 
                lck_mtx_lock_spin_always(&c_seg->c_lock);
 
                if (c_seg->c_busy) {
                        break;
                }
 
                lck_mtx_lock_spin_always(&c_seg->c_lock);
 
                if (c_seg->c_busy) {
+                       VM_DEBUG_CONSTANT_EVENT(vm_compressor_compact_and_swap, VM_COMPRESSOR_COMPACT_AND_SWAP, DBG_FUNC_NONE, 6, (void*) VM_KERNEL_ADDRPERM(c_seg), 0, 0);
+
                        lck_mtx_unlock_always(c_list_lock);
                        c_seg_wait_on_busy(c_seg);
                        lck_mtx_lock_spin_always(c_list_lock);
                        lck_mtx_unlock_always(c_list_lock);
                        c_seg_wait_on_busy(c_seg);
                        lck_mtx_lock_spin_always(c_list_lock);
@@ -2770,13 +3058,15 @@ vm_compressor_compact_and_swap(boolean_t flush_all)
                         * found an empty c_segment and freed it
                         * so go grab the next guy in the queue
                         */
                         * found an empty c_segment and freed it
                         * so go grab the next guy in the queue
                         */
-                       c_seg_major_compact_stats.count_of_freed_segs++;
+                       VM_DEBUG_CONSTANT_EVENT(vm_compressor_compact_and_swap, VM_COMPRESSOR_COMPACT_AND_SWAP, DBG_FUNC_NONE, 7, 0, 0, 0);
+                       c_seg_major_compact_stats[c_seg_major_compact_stats_now].count_of_freed_segs++;
                        continue;
                }
                /*
                 * Major compaction
                 */
                keep_compacting = TRUE;
                        continue;
                }
                /*
                 * Major compaction
                 */
                keep_compacting = TRUE;
+               switch_state = TRUE;
 
                while (keep_compacting == TRUE) {
                        assert(c_seg->c_busy);
 
                while (keep_compacting == TRUE) {
                        assert(c_seg->c_busy);
@@ -2791,6 +3081,8 @@ vm_compressor_compact_and_swap(boolean_t flush_all)
 
                        assert(c_seg_next->c_state == C_ON_AGE_Q);
 
 
                        assert(c_seg_next->c_state == C_ON_AGE_Q);
 
+                       number_considered++;
+
                        if (c_seg_major_compact_ok(c_seg, c_seg_next) == FALSE) {
                                break;
                        }
                        if (c_seg_major_compact_ok(c_seg, c_seg_next) == FALSE) {
                                break;
                        }
@@ -2798,7 +3090,24 @@ vm_compressor_compact_and_swap(boolean_t flush_all)
                        lck_mtx_lock_spin_always(&c_seg_next->c_lock);
 
                        if (c_seg_next->c_busy) {
                        lck_mtx_lock_spin_always(&c_seg_next->c_lock);
 
                        if (c_seg_next->c_busy) {
+                               /*
+                                * We are going to block for our neighbor.
+                                * If our c_seg is wanted, we should unbusy
+                                * it because we don't know how long we might
+                                * have to block here.
+                                */
+                               if (c_seg->c_wanted) {
+                                       lck_mtx_unlock_always(&c_seg_next->c_lock);
+                                       switch_state = FALSE;
+                                       c_seg_major_compact_stats[c_seg_major_compact_stats_now].bailed_compactions++;
+                                       wanted_cseg_found++;
+                                       break;
+                               }
+
                                lck_mtx_unlock_always(c_list_lock);
                                lck_mtx_unlock_always(c_list_lock);
+
+                               VM_DEBUG_CONSTANT_EVENT(vm_compressor_compact_and_swap, VM_COMPRESSOR_COMPACT_AND_SWAP, DBG_FUNC_NONE, 8, (void*) VM_KERNEL_ADDRPERM(c_seg_next), 0, 0);
+
                                c_seg_wait_on_busy(c_seg_next);
                                lck_mtx_lock_spin_always(c_list_lock);
 
                                c_seg_wait_on_busy(c_seg_next);
                                lck_mtx_lock_spin_always(c_list_lock);
 
@@ -2807,12 +3116,14 @@ vm_compressor_compact_and_swap(boolean_t flush_all)
                        /* grab that segment */
                        C_SEG_BUSY(c_seg_next);
 
                        /* grab that segment */
                        C_SEG_BUSY(c_seg_next);
 
+                       bytes_to_free = C_SEG_OFFSET_TO_BYTES(c_seg_next->c_populated_offset);
                        if (c_seg_do_minor_compaction_and_unlock(c_seg_next, FALSE, TRUE, TRUE)) {
                                /*
                                 * found an empty c_segment and freed it
                                 * so we can't continue to use c_seg_next
                                 */
                        if (c_seg_do_minor_compaction_and_unlock(c_seg_next, FALSE, TRUE, TRUE)) {
                                /*
                                 * found an empty c_segment and freed it
                                 * so we can't continue to use c_seg_next
                                 */
-                               c_seg_major_compact_stats.count_of_freed_segs++;
+                               bytes_freed += bytes_to_free;
+                               c_seg_major_compact_stats[c_seg_major_compact_stats_now].count_of_freed_segs++;
                                continue;
                        }
 
                                continue;
                        }
 
@@ -2823,6 +3134,8 @@ vm_compressor_compact_and_swap(boolean_t flush_all)
 
                        keep_compacting = c_seg_major_compact(c_seg, c_seg_next);
 
 
                        keep_compacting = c_seg_major_compact(c_seg, c_seg_next);
 
+                       VM_DEBUG_CONSTANT_EVENT(vm_compressor_compact_and_swap, VM_COMPRESSOR_COMPACT_AND_SWAP, DBG_FUNC_NONE, 9, keep_compacting, 0, 0);
+
                        PAGE_REPLACEMENT_DISALLOWED(TRUE);
 
                        lck_mtx_lock_spin_always(&c_seg_next->c_lock);
                        PAGE_REPLACEMENT_DISALLOWED(TRUE);
 
                        lck_mtx_lock_spin_always(&c_seg_next->c_lock);
@@ -2836,66 +3149,141 @@ vm_compressor_compact_and_swap(boolean_t flush_all)
                         * by passing TRUE, we ask for c_busy to be cleared
                         * and c_wanted to be taken care of
                         */
                         * by passing TRUE, we ask for c_busy to be cleared
                         * and c_wanted to be taken care of
                         */
+                       bytes_to_free = C_SEG_OFFSET_TO_BYTES(c_seg_next->c_populated_offset);
                        if (c_seg_minor_compaction_and_unlock(c_seg_next, TRUE)) {
                        if (c_seg_minor_compaction_and_unlock(c_seg_next, TRUE)) {
-                               c_seg_major_compact_stats.count_of_freed_segs++;
+                               bytes_freed += bytes_to_free;
+                               c_seg_major_compact_stats[c_seg_major_compact_stats_now].count_of_freed_segs++;
+                       } else {
+                               bytes_to_free -= C_SEG_OFFSET_TO_BYTES(c_seg_next->c_populated_offset);
+                               bytes_freed += bytes_to_free;
                        }
 
                        PAGE_REPLACEMENT_DISALLOWED(FALSE);
 
                        /* relock the list */
                        lck_mtx_lock_spin_always(c_list_lock);
                        }
 
                        PAGE_REPLACEMENT_DISALLOWED(FALSE);
 
                        /* relock the list */
                        lck_mtx_lock_spin_always(c_list_lock);
+
+                       if (c_seg->c_wanted) {
+                               /*
+                                * Our c_seg is in demand. Let's
+                                * unbusy it and wakeup the waiters
+                                * instead of continuing the compaction
+                                * because we could be in this loop
+                                * for a while.
+                                */
+                               switch_state = FALSE;
+                               wanted_cseg_found++;
+                               c_seg_major_compact_stats[c_seg_major_compact_stats_now].bailed_compactions++;
+                               break;
+                       }
                } /* major compaction */
 
                } /* major compaction */
 
+               VM_DEBUG_CONSTANT_EVENT(vm_compressor_compact_and_swap, VM_COMPRESSOR_COMPACT_AND_SWAP, DBG_FUNC_NONE, 10, number_considered, wanted_cseg_found, 0);
+
                lck_mtx_lock_spin_always(&c_seg->c_lock);
 
                assert(c_seg->c_busy);
                assert(!c_seg->c_on_minorcompact_q);
 
                lck_mtx_lock_spin_always(&c_seg->c_lock);
 
                assert(c_seg->c_busy);
                assert(!c_seg->c_on_minorcompact_q);
 
-               if (VM_CONFIG_SWAP_IS_ACTIVE) {
-                       /*
-                        * This mode of putting a generic c_seg on the swapout list is
-                        * only supported when we have general swapping enabled
-                        */
-                       c_seg_switch_state(c_seg, C_ON_SWAPOUT_Q, FALSE);
-               } else {
-                       if ((vm_swapout_ripe_segments == TRUE && c_overage_swapped_count < c_overage_swapped_limit)) {
-                               assert(VM_CONFIG_SWAP_IS_PRESENT);
+               if (switch_state) {
+                       if (VM_CONFIG_SWAP_IS_ACTIVE) {
                                /*
                                /*
-                                * we are running compressor sweeps with swap-behind
-                                * make sure the c_seg has aged enough before swapping it
-                                * out...
+                                * This mode of putting a generic c_seg on the swapout list is
+                                * only supported when we have general swapping enabled
                                 */
                                 */
-                               if ((now - c_seg->c_creation_ts) >= vm_ripe_target_age) {
-                                       c_seg->c_overage_swap = TRUE;
-                                       c_overage_swapped_count++;
-                                       c_seg_switch_state(c_seg, C_ON_SWAPOUT_Q, FALSE);
+                               c_seg_switch_state(c_seg, C_ON_SWAPOUT_Q, FALSE);
+                       } else {
+                               if ((vm_swapout_ripe_segments == TRUE && c_overage_swapped_count < c_overage_swapped_limit)) {
+                                       assert(VM_CONFIG_SWAP_IS_PRESENT);
+                                       /*
+                                        * we are running compressor sweeps with swap-behind
+                                        * make sure the c_seg has aged enough before swapping it
+                                        * out...
+                                        */
+                                       if ((now - c_seg->c_creation_ts) >= vm_ripe_target_age) {
+                                               c_seg->c_overage_swap = TRUE;
+                                               c_overage_swapped_count++;
+                                               c_seg_switch_state(c_seg, C_ON_SWAPOUT_Q, FALSE);
+                                       }
                                }
                        }
                                }
                        }
+                       if (c_seg->c_state == C_ON_AGE_Q) {
+                               /*
+                                * this c_seg didn't get moved to the swapout queue
+                                * so we need to move it out of the way...
+                                * we just did a major compaction on it so put it
+                                * on that queue
+                                */
+                               c_seg_switch_state(c_seg, C_ON_MAJORCOMPACT_Q, FALSE);
+                       } else {
+                               c_seg_major_compact_stats[c_seg_major_compact_stats_now].wasted_space_in_swapouts += C_SEG_BUFSIZE - c_seg->c_bytes_used;
+                               c_seg_major_compact_stats[c_seg_major_compact_stats_now].count_of_swapouts++;
+                       }
                }
                }
-               if (c_seg->c_state == C_ON_AGE_Q) {
-                       /*
-                        * this c_seg didn't get moved to the swapout queue
-                        * so we need to move it out of the way...
-                        * we just did a major compaction on it so put it
-                        * on that queue
-                        */
-                       c_seg_switch_state(c_seg, C_ON_MAJORCOMPACT_Q, FALSE);
-               } else {
-                       c_seg_major_compact_stats.wasted_space_in_swapouts += C_SEG_BUFSIZE - c_seg->c_bytes_used;
-                       c_seg_major_compact_stats.count_of_swapouts++;
-               }
+
                C_SEG_WAKEUP_DONE(c_seg);
 
                lck_mtx_unlock_always(&c_seg->c_lock);
 
                if (c_swapout_count) {
                C_SEG_WAKEUP_DONE(c_seg);
 
                lck_mtx_unlock_always(&c_seg->c_lock);
 
                if (c_swapout_count) {
-                       lck_mtx_unlock_always(c_list_lock);
+                       /*
+                        * We don't pause/yield here because we will either
+                        * yield below or at the top of the loop with the
+                        * assert_wait_timeout.
+                        */
+                       if (!vm_swapout_thread_running) {
+                               thread_wakeup((event_t)&c_swapout_list_head);
+                       }
+               }
 
 
-                       thread_wakeup((event_t)&c_swapout_list_head);
+               if (number_considered >= yield_after_considered_per_pass) {
+                       if (wanted_cseg_found) {
+                               /*
+                                * We stopped major compactions on a c_seg
+                                * that is wanted. We don't know the priority
+                                * of the waiter unfortunately but we are at
+                                * a very high priority and so, just in case
+                                * the waiter is a critical system daemon or
+                                * UI thread, let's give up the CPU in case
+                                * the system is running a few CPU intensive
+                                * tasks.
+                                */
+                               lck_mtx_unlock_always(c_list_lock);
 
 
-                       lck_mtx_lock_spin_always(c_list_lock);
+                               mutex_pause(2); /* 100us yield */
+
+                               number_yields++;
+
+                               VM_DEBUG_CONSTANT_EVENT(vm_compressor_compact_and_swap, VM_COMPRESSOR_COMPACT_AND_SWAP, DBG_FUNC_NONE, 11, number_considered, number_yields, 0);
+
+                               lck_mtx_lock_spin_always(c_list_lock);
+                       }
+
+                       number_considered = 0;
+                       wanted_cseg_found = 0;
                }
        }
                }
        }
+       clock_get_system_nanotime(&now, &nsec);
+       end_ts.tv_sec = (int) now;
+       end_ts.tv_nsec = nsec;
+
+       SUB_MACH_TIMESPEC(&end_ts, &start_ts);
+
+       delta_usec = (end_ts.tv_sec * USEC_PER_SEC) + (end_ts.tv_nsec / NSEC_PER_USEC) - (number_yields * 100);
+
+       delta_usec = MAX(1, delta_usec); /* we could have 0 usec run if conditions weren't right */
+
+       c_seg_major_compact_stats[c_seg_major_compact_stats_now].bytes_freed_rate_us = (bytes_freed / delta_usec);
+
+       if ((c_seg_major_compact_stats_now + 1) == C_SEG_MAJOR_COMPACT_STATS_MAX) {
+               c_seg_major_compact_stats_now = 0;
+       } else {
+               c_seg_major_compact_stats_now++;
+       }
+
+       assert(c_seg_major_compact_stats_now < C_SEG_MAJOR_COMPACT_STATS_MAX);
+
+       VM_DEBUG_CONSTANT_EVENT(vm_compressor_compact_and_swap, VM_COMPRESSOR_COMPACT_AND_SWAP, DBG_FUNC_END, c_age_count, c_minor_count, c_major_count, vm_page_free_count);
 }
 
 
 }
 
 
@@ -2906,11 +3294,11 @@ c_seg_allocate(c_segment_t *current_chead)
        int             min_needed;
        int             size_to_populate;
 
        int             min_needed;
        int             size_to_populate;
 
-#if !CONFIG_EMBEDDED
+#if XNU_TARGET_OS_OSX
        if (vm_compressor_low_on_space()) {
                vm_compressor_take_paging_space_action();
        }
        if (vm_compressor_low_on_space()) {
                vm_compressor_take_paging_space_action();
        }
-#endif /* !CONFIG_EMBEDDED */
+#endif /* XNU_TARGET_OS_OSX */
 
        if ((c_seg = *current_chead) == NULL) {
                uint32_t        c_segno;
 
        if ((c_seg = *current_chead) == NULL) {
                uint32_t        c_segno;
@@ -2928,8 +3316,19 @@ c_seg_allocate(c_segment_t *current_chead)
                }
                if (c_free_segno_head == (uint32_t)-1) {
                        uint32_t        c_segments_available_new;
                }
                if (c_free_segno_head == (uint32_t)-1) {
                        uint32_t        c_segments_available_new;
+                       uint32_t        compressed_pages;
+
+#if CONFIG_FREEZE
+                       if (freezer_incore_cseg_acct) {
+                               compressed_pages = c_segment_pages_compressed_incore;
+                       } else {
+                               compressed_pages = c_segment_pages_compressed;
+                       }
+#else
+                       compressed_pages = c_segment_pages_compressed;
+#endif /* CONFIG_FREEZE */
 
 
-                       if (c_segments_available >= c_segments_limit || c_segment_pages_compressed >= c_segment_pages_compressed_limit) {
+                       if (c_segments_available >= c_segments_limit || compressed_pages >= c_segment_pages_compressed_limit) {
                                lck_mtx_unlock_always(c_list_lock);
 
                                return NULL;
                                lck_mtx_unlock_always(c_list_lock);
 
                                return NULL;
@@ -2977,12 +3376,11 @@ c_seg_allocate(c_segment_t *current_chead)
 
                lck_mtx_unlock_always(c_list_lock);
 
 
                lck_mtx_unlock_always(c_list_lock);
 
-               c_seg = (c_segment_t)zalloc(compressor_segment_zone);
-               bzero((char *)c_seg, sizeof(struct c_segment));
+               c_seg = zalloc_flags(compressor_segment_zone, Z_WAITOK | Z_ZERO);
 
                c_seg->c_store.c_buffer = (int32_t *)C_SEG_BUFFER_ADDRESS(c_segno);
 
 
                c_seg->c_store.c_buffer = (int32_t *)C_SEG_BUFFER_ADDRESS(c_segno);
 
-               lck_mtx_init(&c_seg->c_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
+               lck_mtx_init(&c_seg->c_lock, &vm_compressor_lck_grp, LCK_ATTR_NULL);
 
                c_seg->c_state = C_IS_EMPTY;
                c_seg->c_firstemptyslot = C_SLOT_MAX_INDEX;
 
                c_seg->c_state = C_IS_EMPTY;
                c_seg->c_firstemptyslot = C_SLOT_MAX_INDEX;
@@ -3065,7 +3463,8 @@ c_current_seg_filled(c_segment_t c_seg, c_segment_t *current_chead)
                        compressor_map,
                        (vm_offset_t) &c_seg->c_store.c_buffer[offset_to_depopulate],
                        unused_bytes,
                        compressor_map,
                        (vm_offset_t) &c_seg->c_store.c_buffer[offset_to_depopulate],
                        unused_bytes,
-                       KMA_COMPRESSOR);
+                       KMA_COMPRESSOR,
+                       VM_KERN_MEMORY_COMPRESSOR);
 
                lck_mtx_lock_spin_always(&c_seg->c_lock);
 
 
                lck_mtx_lock_spin_always(&c_seg->c_lock);
 
@@ -3096,7 +3495,7 @@ c_current_seg_filled(c_segment_t c_seg, c_segment_t *current_chead)
 #endif
 
 #if CONFIG_FREEZE
 #endif
 
 #if CONFIG_FREEZE
-       if (current_chead == (c_segment_t*)&freezer_chead &&
+       if (current_chead == (c_segment_t*) &(freezer_context_global.freezer_ctx_chead) &&
            VM_CONFIG_SWAP_IS_PRESENT &&
            VM_CONFIG_FREEZER_SWAP_IS_ACTIVE
 #if DEVELOPMENT || DEBUG
            VM_CONFIG_SWAP_IS_PRESENT &&
            VM_CONFIG_FREEZER_SWAP_IS_ACTIVE
 #if DEVELOPMENT || DEBUG
@@ -3127,17 +3526,28 @@ c_current_seg_filled(c_segment_t c_seg, c_segment_t *current_chead)
                 * We'll need to fix this accounting as a start.
                 */
                assert(vm_darkwake_mode == FALSE);
                 * We'll need to fix this accounting as a start.
                 */
                assert(vm_darkwake_mode == FALSE);
-               c_freezer_swapout_page_count += (C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset)) / PAGE_SIZE_64;
+               c_seg_update_task_owner(c_seg, freezer_context_global.freezer_ctx_task);
+               freezer_context_global.freezer_ctx_swapped_bytes += c_seg->c_bytes_used;
        }
 #endif /* CONFIG_FREEZE */
 
        if (c_seg->c_state == C_ON_AGE_Q && C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE) {
        }
 #endif /* CONFIG_FREEZE */
 
        if (c_seg->c_state == C_ON_AGE_Q && C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE) {
+#if CONFIG_FREEZE
+               assert(c_seg->c_task_owner == NULL);
+#endif /* CONFIG_FREEZE */
                c_seg_need_delayed_compaction(c_seg, TRUE);
        }
 
        lck_mtx_unlock_always(c_list_lock);
 
        if (c_seg->c_state == C_ON_SWAPOUT_Q) {
                c_seg_need_delayed_compaction(c_seg, TRUE);
        }
 
        lck_mtx_unlock_always(c_list_lock);
 
        if (c_seg->c_state == C_ON_SWAPOUT_Q) {
+               /*
+                * Darkwake and Freeze configs always
+                * wake up the swapout thread because
+                * the compactor thread that normally handles
+                * it may not be running as much in these
+                * configs.
+                */
                thread_wakeup((event_t)&c_swapout_list_head);
        }
 
                thread_wakeup((event_t)&c_swapout_list_head);
        }
 
@@ -3235,7 +3645,7 @@ c_seg_swapin(c_segment_t c_seg, boolean_t force_minor_compaction, boolean_t age_
        if (vm_swap_get(c_seg, f_offset, io_size) != KERN_SUCCESS) {
                PAGE_REPLACEMENT_DISALLOWED(TRUE);
 
        if (vm_swap_get(c_seg, f_offset, io_size) != KERN_SUCCESS) {
                PAGE_REPLACEMENT_DISALLOWED(TRUE);
 
-               kernel_memory_depopulate(compressor_map, addr, io_size, KMA_COMPRESSOR);
+               kernel_memory_depopulate(compressor_map, addr, io_size, KMA_COMPRESSOR, VM_KERN_MEMORY_COMPRESSOR);
 
                c_seg_swapin_requeue(c_seg, FALSE, TRUE, age_on_swapin_q);
        } else {
 
                c_seg_swapin_requeue(c_seg, FALSE, TRUE, age_on_swapin_q);
        } else {
@@ -3257,6 +3667,27 @@ c_seg_swapin(c_segment_t c_seg, boolean_t force_minor_compaction, boolean_t age_
 
                c_seg_swapin_requeue(c_seg, TRUE, force_minor_compaction == TRUE ? FALSE : TRUE, age_on_swapin_q);
 
 
                c_seg_swapin_requeue(c_seg, TRUE, force_minor_compaction == TRUE ? FALSE : TRUE, age_on_swapin_q);
 
+#if CONFIG_FREEZE
+               /*
+                * c_seg_swapin_requeue() returns with the c_seg lock held.
+                */
+               if (!lck_mtx_try_lock_spin_always(c_list_lock)) {
+                       assert(c_seg->c_busy);
+
+                       lck_mtx_unlock_always(&c_seg->c_lock);
+                       lck_mtx_lock_spin_always(c_list_lock);
+                       lck_mtx_lock_spin_always(&c_seg->c_lock);
+               }
+
+               if (c_seg->c_task_owner) {
+                       c_seg_update_task_owner(c_seg, NULL);
+               }
+
+               lck_mtx_unlock_always(c_list_lock);
+
+               OSAddAtomic(c_seg->c_slots_used, &c_segment_pages_compressed_incore);
+#endif /* CONFIG_FREEZE */
+
                OSAddAtomic64(c_seg->c_bytes_used, &compressor_bytes_used);
 
                if (force_minor_compaction == TRUE) {
                OSAddAtomic64(c_seg->c_bytes_used, &compressor_bytes_used);
 
                if (force_minor_compaction == TRUE) {
@@ -3400,8 +3831,8 @@ retry:
 
        cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_seg->c_nextslot);
 
 
        cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_seg->c_nextslot);
 
+       C_SLOT_ASSERT_PACKABLE(slot_ptr);
        cs->c_packed_ptr = C_SLOT_PACK_PTR(slot_ptr);
        cs->c_packed_ptr = C_SLOT_PACK_PTR(slot_ptr);
-       assert(slot_ptr == (c_slot_mapping_t)C_SLOT_UNPACK_PTR(cs));
 
        cs->c_offset = c_seg->c_nextoffset;
 
 
        cs->c_offset = c_seg->c_nextoffset;
 
@@ -3420,12 +3851,18 @@ retry:
        if (vm_compressor_algorithm() != VM_COMPRESSOR_DEFAULT_CODEC) {
 #if defined(__arm__) || defined(__arm64__)
                uint16_t ccodec = CINVALID;
        if (vm_compressor_algorithm() != VM_COMPRESSOR_DEFAULT_CODEC) {
 #if defined(__arm__) || defined(__arm64__)
                uint16_t ccodec = CINVALID;
-
+               uint32_t inline_popcount;
                if (max_csize >= C_SEG_OFFSET_ALIGNMENT_BOUNDARY) {
                        c_size = metacompressor((const uint8_t *) src,
                            (uint8_t *) &c_seg->c_store.c_buffer[cs->c_offset],
                            max_csize_adj, &ccodec,
                if (max_csize >= C_SEG_OFFSET_ALIGNMENT_BOUNDARY) {
                        c_size = metacompressor((const uint8_t *) src,
                            (uint8_t *) &c_seg->c_store.c_buffer[cs->c_offset],
                            max_csize_adj, &ccodec,
-                           scratch_buf, &incomp_copy);
+                           scratch_buf, &incomp_copy, &inline_popcount);
+#if __ARM_WKDM_POPCNT__
+                       cs->c_inline_popcount = inline_popcount;
+#else
+                       assert(inline_popcount == C_SLOT_NO_POPCOUNT);
+#endif
+
 #if C_SEG_OFFSET_ALIGNMENT_BOUNDARY > 4
                        if (c_size > max_csize_adj) {
                                c_size = -1;
 #if C_SEG_OFFSET_ALIGNMENT_BOUNDARY > 4
                        if (c_size > max_csize_adj) {
                                c_size = -1;
@@ -3546,6 +3983,9 @@ sv_compression:
        OSAddAtomic64(PAGE_SIZE, &c_segment_input_bytes);
 
        OSAddAtomic(1, &c_segment_pages_compressed);
        OSAddAtomic64(PAGE_SIZE, &c_segment_input_bytes);
 
        OSAddAtomic(1, &c_segment_pages_compressed);
+#if CONFIG_FREEZE
+       OSAddAtomic(1, &c_segment_pages_compressed_incore);
+#endif /* CONFIG_FREEZE */
        OSAddAtomic(1, &sample_period_compression_count);
 
        KERNEL_DEBUG(0xe0400000 | DBG_FUNC_END, *current_chead, c_size, c_segment_input_bytes, c_segment_compressed_bytes, 0);
        OSAddAtomic(1, &sample_period_compression_count);
 
        KERNEL_DEBUG(0xe0400000 | DBG_FUNC_END, *current_chead, c_size, c_segment_input_bytes, c_segment_compressed_bytes, 0);
@@ -3606,7 +4046,7 @@ c_decompress_page(char *dst, volatile c_slot_mapping_t slot_ptr, int flags, int
        c_slot_t        cs;
        c_segment_t     c_seg;
        uint32_t        c_segno;
        c_slot_t        cs;
        c_segment_t     c_seg;
        uint32_t        c_segno;
-       int             c_indx;
+       uint16_t        c_indx;
        int             c_rounded_size;
        uint32_t        c_size;
        int             retval = 0;
        int             c_rounded_size;
        uint32_t        c_size;
        int             retval = 0;
@@ -3744,6 +4184,28 @@ bypass_busy_check:
                clock_nsec_t    cur_ts_nsec;
 
                if (C_SEG_IS_ONDISK(c_seg)) {
                clock_nsec_t    cur_ts_nsec;
 
                if (C_SEG_IS_ONDISK(c_seg)) {
+#if CONFIG_FREEZE
+                       if (freezer_incore_cseg_acct) {
+                               if ((c_seg->c_slots_used + c_segment_pages_compressed_incore) >= c_segment_pages_compressed_nearing_limit) {
+                                       PAGE_REPLACEMENT_DISALLOWED(FALSE);
+                                       lck_mtx_unlock_always(&c_seg->c_lock);
+
+                                       memorystatus_kill_on_VM_compressor_space_shortage(FALSE /* async */);
+
+                                       goto ReTry;
+                               }
+
+                               uint32_t incore_seg_count = c_segment_count - c_swappedout_count - c_swappedout_sparse_count;
+                               if ((incore_seg_count + 1) >= c_segments_nearing_limit) {
+                                       PAGE_REPLACEMENT_DISALLOWED(FALSE);
+                                       lck_mtx_unlock_always(&c_seg->c_lock);
+
+                                       memorystatus_kill_on_VM_compressor_space_shortage(FALSE /* async */);
+
+                                       goto ReTry;
+                               }
+                       }
+#endif /* CONFIG_FREEZE */
                        assert(kdp_mode == FALSE);
                        retval = c_seg_swapin(c_seg, FALSE, TRUE);
                        assert(retval == 0);
                        assert(kdp_mode == FALSE);
                        retval = c_seg_swapin(c_seg, FALSE, TRUE);
                        assert(retval == 0);
@@ -3812,8 +4274,32 @@ bypass_busy_check:
                        if (vm_compressor_algorithm() != VM_COMPRESSOR_DEFAULT_CODEC) {
 #if defined(__arm__) || defined(__arm64__)
                                uint16_t c_codec = cs->c_codec;
                        if (vm_compressor_algorithm() != VM_COMPRESSOR_DEFAULT_CODEC) {
 #if defined(__arm__) || defined(__arm64__)
                                uint16_t c_codec = cs->c_codec;
-                               metadecompressor((const uint8_t *) &c_seg->c_store.c_buffer[cs->c_offset],
-                                   (uint8_t *)dst, c_size, c_codec, (void *)scratch_buf);
+                               uint32_t inline_popcount;
+                               if (!metadecompressor((const uint8_t *) &c_seg->c_store.c_buffer[cs->c_offset],
+                                   (uint8_t *)dst, c_size, c_codec, (void *)scratch_buf, &inline_popcount)) {
+                                       retval = -1;
+                               } else {
+#if __ARM_WKDM_POPCNT__
+                                       if (inline_popcount != cs->c_inline_popcount) {
+                                               /*
+                                                * The codec choice in compression and
+                                                * decompression must agree, so there
+                                                * should never be a disagreement in
+                                                * whether an inline population count
+                                                * was performed.
+                                                */
+                                               assert(inline_popcount != C_SLOT_NO_POPCOUNT);
+                                               assert(cs->c_inline_popcount != C_SLOT_NO_POPCOUNT);
+                                               printf("decompression failure from physical region %llx+%05x: popcount mismatch (%d != %d)\n",
+                                                   (unsigned long long)kvtophys((uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset]), c_size,
+                                                   inline_popcount,
+                                                   cs->c_inline_popcount);
+                                               retval = -1;
+                                       }
+#else
+                                       assert(inline_popcount == C_SLOT_NO_POPCOUNT);
+#endif /* __ARM_WKDM_POPCNT__ */
+                               }
 #endif
                        } else {
 #if defined(__arm64__)
 #endif
                        } else {
 #if defined(__arm64__)
@@ -3856,6 +4342,31 @@ bypass_busy_check:
                        OSAddAtomic(1, &sample_period_decompression_count);
                }
        }
                        OSAddAtomic(1, &sample_period_decompression_count);
                }
        }
+#if CONFIG_FREEZE
+       else {
+               /*
+                * We are freeing an uncompressed page from this c_seg and so balance the ledgers.
+                */
+               if (C_SEG_IS_ONDISK(c_seg)) {
+                       /*
+                        * The compression sweep feature will push out anonymous pages to disk
+                        * without going through the freezer path and so those c_segs, while
+                        * swapped out, won't have an owner.
+                        */
+                       if (c_seg->c_task_owner) {
+                               task_update_frozen_to_swap_acct(c_seg->c_task_owner, PAGE_SIZE_64, DEBIT_FROM_SWAP);
+                       }
+
+                       /*
+                        * We are freeing a page in swap without swapping it in. We bump the in-core
+                        * count here to simulate a swapin of a page so that we can accurately
+                        * decrement it below.
+                        */
+                       OSAddAtomic(1, &c_segment_pages_compressed_incore);
+               }
+       }
+#endif /* CONFIG_FREEZE */
+
        if (flags & C_KEEP) {
                *zeroslot = 0;
                goto done;
        if (flags & C_KEEP) {
                *zeroslot = 0;
                goto done;
@@ -3875,6 +4386,10 @@ bypass_busy_check:
        }
 
        OSAddAtomic(-1, &c_segment_pages_compressed);
        }
 
        OSAddAtomic(-1, &c_segment_pages_compressed);
+#if CONFIG_FREEZE
+       OSAddAtomic(-1, &c_segment_pages_compressed_incore);
+       assertf(c_segment_pages_compressed_incore >= 0, "-ve incore count %p 0x%x", c_seg, c_segment_pages_compressed_incore);
+#endif /* CONFIG_FREEZE */
 
        if (c_seg->c_state != C_ON_BAD_Q && !(C_SEG_IS_ONDISK(c_seg))) {
                /*
 
        if (c_seg->c_state != C_ON_BAD_Q && !(C_SEG_IS_ONDISK(c_seg))) {
                /*
@@ -3909,7 +4424,9 @@ bypass_busy_check:
                                        C_SEG_BUSY(c_seg);
                                        lck_mtx_unlock_always(&c_seg->c_lock);
 
                                        C_SEG_BUSY(c_seg);
                                        lck_mtx_unlock_always(&c_seg->c_lock);
 
-                                       kernel_memory_depopulate(compressor_map, (vm_offset_t) c_seg->c_store.c_buffer, pages_populated * PAGE_SIZE, KMA_COMPRESSOR);
+                                       kernel_memory_depopulate(compressor_map,
+                                           (vm_offset_t) c_seg->c_store.c_buffer,
+                                           pages_populated * PAGE_SIZE, KMA_COMPRESSOR, VM_KERN_MEMORY_COMPRESSOR);
 
                                        lck_mtx_lock_spin_always(&c_seg->c_lock);
                                        C_SEG_WAKEUP_DONE(c_seg);
 
                                        lck_mtx_lock_spin_always(&c_seg->c_lock);
                                        C_SEG_WAKEUP_DONE(c_seg);
@@ -3956,11 +4473,11 @@ done:
                vm_swap_consider_defragmenting(VM_SWAP_FLAGS_NONE);
        }
 
                vm_swap_consider_defragmenting(VM_SWAP_FLAGS_NONE);
        }
 
-#if CONFIG_EMBEDDED
+#if !XNU_TARGET_OS_OSX
        if ((c_minor_count && COMPRESSOR_NEEDS_TO_MINOR_COMPACT()) || vm_compressor_needs_to_major_compact()) {
                vm_wake_compactor_swapper();
        }
        if ((c_minor_count && COMPRESSOR_NEEDS_TO_MINOR_COMPACT()) || vm_compressor_needs_to_major_compact()) {
                vm_wake_compactor_swapper();
        }
-#endif
+#endif /* !XNU_TARGET_OS_OSX */
 
        return retval;
 }
 
        return retval;
 }
@@ -3996,6 +4513,10 @@ vm_compressor_get(ppnum_t pn, int *slot, int flags)
                        c_segment_sv_hash_drop_ref(slot_ptr->s_cindx);
 
                        OSAddAtomic(-1, &c_segment_pages_compressed);
                        c_segment_sv_hash_drop_ref(slot_ptr->s_cindx);
 
                        OSAddAtomic(-1, &c_segment_pages_compressed);
+#if CONFIG_FREEZE
+                       OSAddAtomic(-1, &c_segment_pages_compressed_incore);
+                       assertf(c_segment_pages_compressed_incore >= 0, "-ve incore count 0x%x", c_segment_pages_compressed_incore);
+#endif /* CONFIG_FREEZE */
                        *slot = 0;
                }
                if (data) {
                        *slot = 0;
                }
                if (data) {
@@ -4029,6 +4550,95 @@ vm_compressor_get(ppnum_t pn, int *slot, int flags)
        return retval;
 }
 
        return retval;
 }
 
+#if DEVELOPMENT || DEBUG
+
+void
+vm_compressor_inject_error(int *slot)
+{
+       c_slot_mapping_t slot_ptr = (c_slot_mapping_t)slot;
+
+       /* No error detection for single-value compression. */
+       if (slot_ptr->s_cseg == C_SV_CSEG_ID) {
+               printf("%s(): cannot inject errors in SV-compressed pages\n", __func__ );
+               return;
+       }
+
+       /* s_cseg is actually "segno+1" */
+       const uint32_t c_segno = slot_ptr->s_cseg - 1;
+
+       assert(c_segno < c_segments_available);
+       assert(c_segments[c_segno].c_segno >= c_segments_available);
+
+       const c_segment_t c_seg = c_segments[c_segno].c_seg;
+
+       PAGE_REPLACEMENT_DISALLOWED(TRUE);
+
+       lck_mtx_lock_spin_always(&c_seg->c_lock);
+       assert(c_seg->c_state != C_IS_EMPTY && c_seg->c_state != C_IS_FREE);
+
+       const uint16_t c_indx = slot_ptr->s_cindx;
+       assert(c_indx < c_seg->c_nextslot);
+
+       /*
+        * To safely make this segment temporarily writable, we need to mark
+        * the segment busy, which allows us to release the segment lock.
+        */
+       while (c_seg->c_busy) {
+               c_seg_wait_on_busy(c_seg);
+               lck_mtx_lock_spin_always(&c_seg->c_lock);
+       }
+       C_SEG_BUSY(c_seg);
+
+       bool already_writable = (c_seg->c_state == C_IS_FILLING);
+       if (!already_writable) {
+               /*
+                * Protection update must be performed preemptibly, so temporarily drop
+                * the lock. Having set c_busy will prevent most other concurrent
+                * operations.
+                */
+               lck_mtx_unlock_always(&c_seg->c_lock);
+               C_SEG_MAKE_WRITEABLE(c_seg);
+               lck_mtx_lock_spin_always(&c_seg->c_lock);
+       }
+
+       /*
+        * Once we've released the lock following our c_state == C_IS_FILLING check,
+        * c_current_seg_filled() can (re-)write-protect the segment. However, it
+        * will transition from C_IS_FILLING before releasing the c_seg lock, so we
+        * can detect this by re-checking after we've reobtained the lock.
+        */
+       if (already_writable && c_seg->c_state != C_IS_FILLING) {
+               lck_mtx_unlock_always(&c_seg->c_lock);
+               C_SEG_MAKE_WRITEABLE(c_seg);
+               lck_mtx_lock_spin_always(&c_seg->c_lock);
+               already_writable = false;
+               /* Segment can't be freed while c_busy is set. */
+               assert(c_seg->c_state != C_IS_FILLING);
+       }
+
+       c_slot_t cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
+       int32_t *data = &c_seg->c_store.c_buffer[cs->c_offset];
+       /* assume that the compressed data holds at least one int32_t */
+       assert(UNPACK_C_SIZE(cs) > sizeof(*data));
+       /*
+        * This bit is known to be in the payload of a MISS packet resulting from
+        * the pattern used in the test pattern from decompression_failure.c.
+        * Flipping it should result in many corrupted bits in the test page.
+        */
+       data[0] ^= 0x00000100;
+       if (!already_writable) {
+               lck_mtx_unlock_always(&c_seg->c_lock);
+               C_SEG_WRITE_PROTECT(c_seg);
+               lck_mtx_lock_spin_always(&c_seg->c_lock);
+       }
+
+       C_SEG_WAKEUP_DONE(c_seg);
+       lck_mtx_unlock_always(&c_seg->c_lock);
+
+       PAGE_REPLACEMENT_DISALLOWED(FALSE);
+}
+
+#endif /* DEVELOPMENT || DEBUG */
 
 int
 vm_compressor_free(int *slot, int flags)
 
 int
 vm_compressor_free(int *slot, int flags)
@@ -4044,6 +4654,10 @@ vm_compressor_free(int *slot, int flags)
        if (slot_ptr->s_cseg == C_SV_CSEG_ID) {
                c_segment_sv_hash_drop_ref(slot_ptr->s_cindx);
                OSAddAtomic(-1, &c_segment_pages_compressed);
        if (slot_ptr->s_cseg == C_SV_CSEG_ID) {
                c_segment_sv_hash_drop_ref(slot_ptr->s_cindx);
                OSAddAtomic(-1, &c_segment_pages_compressed);
+#if CONFIG_FREEZE
+               OSAddAtomic(-1, &c_segment_pages_compressed_incore);
+               assertf(c_segment_pages_compressed_incore >= 0, "-ve incore count 0x%x", c_segment_pages_compressed_incore);
+#endif /* CONFIG_FREEZE */
 
                *slot = 0;
                return 0;
 
                *slot = 0;
                return 0;
@@ -4086,7 +4700,7 @@ vm_compressor_transfer(
 {
        c_slot_mapping_t        dst_slot, src_slot;
        c_segment_t             c_seg;
 {
        c_slot_mapping_t        dst_slot, src_slot;
        c_segment_t             c_seg;
-       int                     c_indx;
+       uint16_t                c_indx;
        c_slot_t                cs;
 
        src_slot = (c_slot_mapping_t) src_slot_p;
        c_slot_t                cs;
 
        src_slot = (c_slot_mapping_t) src_slot_p;
@@ -4113,6 +4727,7 @@ Retry:
        c_indx = src_slot->s_cindx;
        cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
        /* point the c_slot back to dst_slot instead of src_slot */
        c_indx = src_slot->s_cindx;
        cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
        /* point the c_slot back to dst_slot instead of src_slot */
+       C_SLOT_ASSERT_PACKABLE(dst_slot);
        cs->c_packed_ptr = C_SLOT_PACK_PTR(dst_slot);
        /* transfer */
        *dst_slot_p = *src_slot_p;
        cs->c_packed_ptr = C_SLOT_PACK_PTR(dst_slot);
        /* transfer */
        *dst_slot_p = *src_slot_p;
@@ -4171,7 +4786,7 @@ vm_compressor_relocate(
        uint16_t                dst_slot;
        c_slot_t                c_dst;
        c_slot_t                c_src;
        uint16_t                dst_slot;
        c_slot_t                c_dst;
        c_slot_t                c_src;
-       int                     c_indx;
+       uint16_t                c_indx;
        c_segment_t             c_seg_dst = NULL;
        c_segment_t             c_seg_src = NULL;
        kern_return_t           kr = KERN_SUCCESS;
        c_segment_t             c_seg_dst = NULL;
        c_segment_t             c_seg_src = NULL;
        kern_return_t           kr = KERN_SUCCESS;
@@ -4219,17 +4834,21 @@ Relookup_src:
 
        lck_mtx_lock_spin_always(&c_seg_src->c_lock);
 
 
        lck_mtx_lock_spin_always(&c_seg_src->c_lock);
 
-       if (C_SEG_IS_ONDISK(c_seg_src)) {
+       if (C_SEG_IS_ON_DISK_OR_SOQ(c_seg_src) ||
+           c_seg_src->c_state == C_IS_FILLING) {
                /*
                /*
-                * A "thaw" can mark a process as eligible for
+                * Skip this page if :-
+                * a) the src c_seg is already on-disk (or on its way there)
+                *    A "thaw" can mark a process as eligible for
                 * another freeze cycle without bringing any of
                 * its swapped out c_segs back from disk (because
                 * that is done on-demand).
                 * another freeze cycle without bringing any of
                 * its swapped out c_segs back from disk (because
                 * that is done on-demand).
+                *    Or, this page may be mapped elsewhere in the task's map,
+                * and we may have marked it for swap already.
                 *
                 *
-                * If the src c_seg we find for our pre-compressed
-                * data is already on-disk, then we are dealing
-                * with an app's data that is already packed and
-                * swapped out. Don't do anything.
+                * b) Or, the src c_seg is being filled by the compressor
+                * thread. We don't want the added latency of waiting for
+                * this c_seg in the freeze path and so we skip it.
                 */
 
                PAGE_REPLACEMENT_DISALLOWED(FALSE);
                 */
 
                PAGE_REPLACEMENT_DISALLOWED(FALSE);
@@ -4303,7 +4922,9 @@ Relookup_src:
        c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, c_seg_dst->c_nextslot);
 
        memcpy(&c_seg_dst->c_store.c_buffer[c_seg_dst->c_nextoffset], &c_seg_src->c_store.c_buffer[c_src->c_offset], c_size);
        c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, c_seg_dst->c_nextslot);
 
        memcpy(&c_seg_dst->c_store.c_buffer[c_seg_dst->c_nextoffset], &c_seg_src->c_store.c_buffer[c_src->c_offset], c_size);
-//is platform alignment actually necessary since wkdm aligns its output?
+       /*
+        * Is platform alignment actually necessary since wkdm aligns its output?
+        */
        c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
 
        cslot_copy(c_dst, c_src);
        c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
 
        cslot_copy(c_dst, c_src);
@@ -4334,7 +4955,7 @@ Relookup_src:
        c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, dst_slot);
 
        PAGE_REPLACEMENT_ALLOWED(TRUE);
        c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, dst_slot);
 
        PAGE_REPLACEMENT_ALLOWED(TRUE);
-       slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
+       slot_ptr = C_SLOT_UNPACK_PTR(c_dst);
        /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
        slot_ptr->s_cseg = c_seg_dst->c_mysegno + 1;
        slot_ptr->s_cindx = dst_slot;
        /* <csegno=0,indx=0> would mean "empty slot", so use csegno+1 */
        slot_ptr->s_cseg = c_seg_dst->c_mysegno + 1;
        slot_ptr->s_cindx = dst_slot;