[apple/xnu.git] / osfmk / vm / vm_compressor.h

/*
 * Copyright (c) 2000-2016 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#include <kern/kalloc.h>
#include <vm/vm_compressor_pager.h>
#include <vm/vm_kern.h>
#include <vm/vm_page.h>
#include <vm/vm_protos.h>
#include <vm/WKdm_new.h>
#include <vm/vm_object.h>
#include <vm/vm_map.h>
#include <machine/pmap.h>
#include <kern/locks.h>

#include <sys/kdebug.h>

#if defined(__arm64__)
#include <arm/proc_reg.h>
#endif

#define C_SEG_OFFSET_BITS       16
#define C_SEG_BUFSIZE           (1024 * 256)
#define C_SEG_MAX_PAGES         (C_SEG_BUFSIZE / PAGE_SIZE)

#if CONFIG_EMBEDDED
#define C_SEG_OFF_LIMIT         (C_SEG_BYTES_TO_OFFSET((C_SEG_BUFSIZE - 512)))
#define C_SEG_ALLOCSIZE         (C_SEG_BUFSIZE + PAGE_SIZE)
#else
#define C_SEG_OFF_LIMIT         (C_SEG_BYTES_TO_OFFSET((C_SEG_BUFSIZE - 128)))
#define C_SEG_ALLOCSIZE         (C_SEG_BUFSIZE)
#endif
#define C_SEG_MAX_POPULATE_SIZE (4 * PAGE_SIZE)

#if defined(__arm64__)

#if DEVELOPMENT || DEBUG

#if defined(PLATFORM_WatchOS)
#define VALIDATE_C_SEGMENTS (1)
#endif
#endif

#endif


#if DEBUG || COMPRESSOR_INTEGRITY_CHECKS
#define ENABLE_SWAP_CHECKS 1
#define ENABLE_COMPRESSOR_CHECKS 1
#define POPCOUNT_THE_COMPRESSED_DATA (1)
#else
#define ENABLE_SWAP_CHECKS 0
#define ENABLE_COMPRESSOR_CHECKS 0
#endif

#define CHECKSUM_THE_SWAP               ENABLE_SWAP_CHECKS      /* Debug swap data */
#define CHECKSUM_THE_DATA               ENABLE_COMPRESSOR_CHECKS        /* Debug compressor/decompressor data */
#define CHECKSUM_THE_COMPRESSED_DATA    ENABLE_COMPRESSOR_CHECKS        /* Debug compressor/decompressor compressed data */

#ifndef VALIDATE_C_SEGMENTS
#define VALIDATE_C_SEGMENTS             ENABLE_COMPRESSOR_CHECKS        /* Debug compaction */
#endif

#define RECORD_THE_COMPRESSED_DATA      0

struct c_slot {
	uint64_t        c_offset:C_SEG_OFFSET_BITS,
#if defined(__arm64__)
	c_size:14,
	    c_codec:1,
	    c_packed_ptr:33;
#elif defined(__arm__)
	c_size:12,
	c_codec:1,
	c_packed_ptr:35;
#else
	c_size:12,
	c_packed_ptr:36;
#endif
#if CHECKSUM_THE_DATA
	unsigned int    c_hash_data;
#endif
#if CHECKSUM_THE_COMPRESSED_DATA
	unsigned int    c_hash_compressed_data;
#endif
#if POPCOUNT_THE_COMPRESSED_DATA
	unsigned int    c_pop_cdata;
#endif
};

#define C_IS_EMPTY              0
#define C_IS_FREE               1
#define C_IS_FILLING            2
#define C_ON_AGE_Q              3
#define C_ON_SWAPOUT_Q          4
#define C_ON_SWAPPEDOUT_Q       5
#define C_ON_SWAPPEDOUTSPARSE_Q 6
#define C_ON_SWAPPEDIN_Q        7
#define C_ON_MAJORCOMPACT_Q     8
#define C_ON_BAD_Q              9
#define C_ON_SWAPIO_Q          10


struct c_segment {
	lck_mtx_t       c_lock;
	queue_chain_t   c_age_list;
	queue_chain_t   c_list;

#define C_SEG_MAX_LIMIT         (1 << 20)       /* this needs to track the size of c_mysegno */
	uint32_t        c_mysegno:20,
	    c_busy:1,
	    c_busy_swapping:1,
	    c_wanted:1,
	    c_on_minorcompact_q:1,              /* can also be on the age_q, the majorcompact_q or the swappedin_q */

	    c_state:4,                          /* what state is the segment in which dictates which q to find it on */
	    c_overage_swap:1,
	    c_reserved:3;

	uint32_t        c_creation_ts;
	uint64_t        c_generation_id;

	int32_t         c_bytes_used;
	int32_t         c_bytes_unused;
	uint32_t        c_slots_used;

	uint16_t        c_firstemptyslot;
	uint16_t        c_nextslot;
	uint32_t        c_nextoffset;
	uint32_t        c_populated_offset;

	uint32_t        c_swappedin_ts;

	union {
		int32_t *c_buffer;
		uint64_t c_swap_handle;
	} c_store;

#if     VALIDATE_C_SEGMENTS
	uint32_t        c_was_minor_compacted;
	uint32_t        c_was_major_compacted;
	uint32_t        c_was_major_donor;
#endif
#if CHECKSUM_THE_SWAP
	unsigned int    cseg_hash;
	unsigned int    cseg_swap_size;
#endif /* CHECKSUM_THE_SWAP */

	thread_t        c_busy_for_thread;

	int             c_slot_var_array_len;
	struct  c_slot  *c_slot_var_array;
	struct  c_slot  c_slot_fixed_array[0];
};


struct  c_slot_mapping {
	uint32_t        s_cseg:22,      /* segment number + 1 */
	    s_cindx:10;                 /* index in the segment */
};
#define C_SLOT_MAX_INDEX        (1 << 10)

typedef struct c_slot_mapping *c_slot_mapping_t;


#define C_SEG_SLOT_VAR_ARRAY_MIN_LEN    C_SEG_MAX_PAGES

extern  int             c_seg_fixed_array_len;
extern  vm_offset_t     c_buffers;
#define C_SEG_BUFFER_ADDRESS(c_segno)   ((c_buffers + ((uint64_t)c_segno * (uint64_t)C_SEG_ALLOCSIZE)))

#define C_SEG_SLOT_FROM_INDEX(cseg, index)      (index < c_seg_fixed_array_len ? &(cseg->c_slot_fixed_array[index]) : &(cseg->c_slot_var_array[index - c_seg_fixed_array_len]))

#define C_SEG_OFFSET_TO_BYTES(off)      ((off) * (int) sizeof(int32_t))
#define C_SEG_BYTES_TO_OFFSET(bytes)    ((bytes) / (int) sizeof(int32_t))

#define C_SEG_UNUSED_BYTES(cseg)        (cseg->c_bytes_unused + (C_SEG_OFFSET_TO_BYTES(cseg->c_populated_offset - cseg->c_nextoffset)))
//todo opensource

#ifndef __PLATFORM_WKDM_ALIGNMENT_MASK__
#define C_SEG_OFFSET_ALIGNMENT_MASK     0x3ULL
#define C_SEG_OFFSET_ALIGNMENT_BOUNDARY 0x4
#else
#define C_SEG_OFFSET_ALIGNMENT_MASK     __PLATFORM_WKDM_ALIGNMENT_MASK__
#define C_SEG_OFFSET_ALIGNMENT_BOUNDARY __PLATFORM_WKDM_ALIGNMENT_BOUNDARY__
#endif

#define C_SEG_SHOULD_MINORCOMPACT_NOW(cseg)     ((C_SEG_UNUSED_BYTES(cseg) >= (C_SEG_BUFSIZE / 4)) ? 1 : 0)

/*
 * the decsion to force a c_seg to be major compacted is based on 2 criteria
 * 1) is the c_seg buffer almost empty (i.e. we have a chance to merge it with another c_seg)
 * 2) are there at least a minimum number of slots unoccupied so that we have a chance
 *    of combining this c_seg with another one.
 */
#define C_SEG_SHOULD_MAJORCOMPACT_NOW(cseg)                                                                                     \
	((((cseg->c_bytes_unused + (C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES(c_seg->c_nextoffset))) >= (C_SEG_BUFSIZE / 8)) &&     \
	  ((C_SLOT_MAX_INDEX - cseg->c_slots_used) > (C_SEG_BUFSIZE / PAGE_SIZE))) \
	? 1 : 0)

#define C_SEG_ONDISK_IS_SPARSE(cseg)    ((cseg->c_bytes_used < cseg->c_bytes_unused) ? 1 : 0)
#define C_SEG_IS_ONDISK(cseg)           ((cseg->c_state == C_ON_SWAPPEDOUT_Q || cseg->c_state == C_ON_SWAPPEDOUTSPARSE_Q))
#define C_SEG_IS_ON_DISK_OR_SOQ(cseg)   ((cseg->c_state == C_ON_SWAPPEDOUT_Q || \
	                                  cseg->c_state == C_ON_SWAPPEDOUTSPARSE_Q || \
	                                  cseg->c_state == C_ON_SWAPOUT_Q || \
	                                  cseg->c_state == C_ON_SWAPIO_Q))


#define C_SEG_WAKEUP_DONE(cseg)                         \
	MACRO_BEGIN                                     \
	assert((cseg)->c_busy);                         \
	(cseg)->c_busy = 0;                             \
	assert((cseg)->c_busy_for_thread != NULL);      \
	(cseg)->c_busy_for_thread = NULL;               \
	if ((cseg)->c_wanted) {                         \
	        (cseg)->c_wanted = 0;                   \
	        thread_wakeup((event_t) (cseg));        \
	}                                               \
	MACRO_END

#define C_SEG_BUSY(cseg)                                \
	MACRO_BEGIN                                     \
	assert((cseg)->c_busy == 0);                    \
	(cseg)->c_busy = 1;                             \
	assert((cseg)->c_busy_for_thread == NULL);      \
	(cseg)->c_busy_for_thread = current_thread();   \
	MACRO_END


extern vm_map_t compressor_map;

#if DEVELOPMENT || DEBUG
extern boolean_t write_protect_c_segs;
extern int vm_compressor_test_seg_wp;

#define C_SEG_MAKE_WRITEABLE(cseg)                      \
	MACRO_BEGIN                                     \
	if (write_protect_c_segs) {                     \
	        vm_map_protect(compressor_map,                  \
	                       (vm_map_offset_t)cseg->c_store.c_buffer,         \
	                       (vm_map_offset_t)&cseg->c_store.c_buffer[C_SEG_BYTES_TO_OFFSET(C_SEG_ALLOCSIZE)],\
	                       VM_PROT_READ | VM_PROT_WRITE,    \
	                       0);                              \
	}                               \
	MACRO_END

#define C_SEG_WRITE_PROTECT(cseg)                       \
	MACRO_BEGIN                                     \
	if (write_protect_c_segs) {                     \
	        vm_map_protect(compressor_map,                  \
	                       (vm_map_offset_t)cseg->c_store.c_buffer,         \
	                       (vm_map_offset_t)&cseg->c_store.c_buffer[C_SEG_BYTES_TO_OFFSET(C_SEG_ALLOCSIZE)],\
	                       VM_PROT_READ,                    \
	                       0);                              \
	}                                                       \
	if (vm_compressor_test_seg_wp) {                                \
	        volatile uint32_t vmtstmp = *(volatile uint32_t *)cseg->c_store.c_buffer; \
	        *(volatile uint32_t *)cseg->c_store.c_buffer = 0xDEADABCD; \
	        (void) vmtstmp;                                         \
	}                                                               \
	MACRO_END
#endif

typedef struct c_segment *c_segment_t;
typedef struct c_slot   *c_slot_t;

uint64_t vm_compressor_total_compressions(void);
void vm_wake_compactor_swapper(void);
void vm_run_compactor(void);
void vm_thrashing_jetsam_done(void);
void vm_consider_waking_compactor_swapper(void);
void vm_consider_swapping(void);
void vm_compressor_flush(void);
void c_seg_free(c_segment_t);
void c_seg_free_locked(c_segment_t);
void c_seg_insert_into_age_q(c_segment_t);
void c_seg_need_delayed_compaction(c_segment_t, boolean_t);

void vm_decompressor_lock(void);
void vm_decompressor_unlock(void);

void vm_compressor_delay_trim(void);
void vm_compressor_do_warmup(void);
void vm_compressor_record_warmup_start(void);
void vm_compressor_record_warmup_end(void);

int                     vm_wants_task_throttled(task_t);

extern void             vm_compaction_swapper_do_init(void);
extern void             vm_compressor_swap_init(void);
extern void             vm_compressor_init_locks(void);
extern lck_rw_t         c_master_lock;

#if ENCRYPTED_SWAP
extern void             vm_swap_decrypt(c_segment_t);
#endif /* ENCRYPTED_SWAP */

extern int              vm_swap_low_on_space(void);
extern kern_return_t    vm_swap_get(c_segment_t, uint64_t, uint64_t);
extern void             vm_swap_free(uint64_t);
extern void             vm_swap_consider_defragmenting(int);

extern void             c_seg_swapin_requeue(c_segment_t, boolean_t, boolean_t, boolean_t);
extern int              c_seg_swapin(c_segment_t, boolean_t, boolean_t);
extern void             c_seg_wait_on_busy(c_segment_t);
extern void             c_seg_trim_tail(c_segment_t);
extern void             c_seg_switch_state(c_segment_t, int, boolean_t);

extern boolean_t        fastwake_recording_in_progress;
extern int              compaction_swapper_inited;
extern int              compaction_swapper_running;
extern uint64_t         vm_swap_put_failures;

extern int              c_overage_swapped_count;
extern int              c_overage_swapped_limit;

extern queue_head_t     c_minor_list_head;
extern queue_head_t     c_age_list_head;
extern queue_head_t     c_swapout_list_head;
extern queue_head_t     c_swappedout_list_head;
extern queue_head_t     c_swappedout_sparse_list_head;

extern uint32_t         c_age_count;
extern uint32_t         c_swapout_count;
extern uint32_t         c_swappedout_count;
extern uint32_t         c_swappedout_sparse_count;

extern int64_t          compressor_bytes_used;
extern uint64_t         first_c_segment_to_warm_generation_id;
extern uint64_t         last_c_segment_to_warm_generation_id;
extern boolean_t        hibernate_flushing;
extern boolean_t        hibernate_no_swapspace;
extern boolean_t        hibernate_in_progress_with_pinned_swap;
extern uint32_t         swapout_target_age;

extern void c_seg_insert_into_q(queue_head_t *, c_segment_t);

extern uint32_t vm_compressor_minorcompact_threshold_divisor;
extern uint32_t vm_compressor_majorcompact_threshold_divisor;
extern uint32_t vm_compressor_unthrottle_threshold_divisor;
extern uint32_t vm_compressor_catchup_threshold_divisor;

extern uint32_t vm_compressor_minorcompact_threshold_divisor_overridden;
extern uint32_t vm_compressor_majorcompact_threshold_divisor_overridden;
extern uint32_t vm_compressor_unthrottle_threshold_divisor_overridden;
extern uint32_t vm_compressor_catchup_threshold_divisor_overridden;

extern uint64_t vm_compressor_compute_elapsed_msecs(clock_sec_t, clock_nsec_t, clock_sec_t, clock_nsec_t);

extern void kdp_compressor_busy_find_owner(event64_t wait_event, thread_waitinfo_t *waitinfo);

#define PAGE_REPLACEMENT_DISALLOWED(enable)     (enable == TRUE ? lck_rw_lock_shared(&c_master_lock) : lck_rw_done(&c_master_lock))
#define PAGE_REPLACEMENT_ALLOWED(enable)        (enable == TRUE ? lck_rw_lock_exclusive(&c_master_lock) : lck_rw_done(&c_master_lock))


#define AVAILABLE_NON_COMPRESSED_MEMORY         (vm_page_active_count + vm_page_inactive_count + vm_page_free_count + vm_page_speculative_count)
#define AVAILABLE_MEMORY                        (AVAILABLE_NON_COMPRESSED_MEMORY + VM_PAGE_COMPRESSOR_COUNT)

/*
 * TODO, there may be a minor optimisation opportunity to replace these divisions
 * with multiplies and shifts
 *
 * By multiplying by 10, the divisors can have more precision w/o resorting to floating point... a divisor specified as 25 is in reality a divide by 2.5
 * By multiplying by 9, you get a number ~11% smaller which allows us to have another limit point derived from the same base
 * By multiplying by 11, you get a number ~10% bigger which allows us to generate a reset limit derived from the same base which is useful for hysteresis
 */

#define VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD            (((AVAILABLE_MEMORY) * 10) / (vm_compressor_minorcompact_threshold_divisor ? vm_compressor_minorcompact_threshold_divisor : 10))
#define VM_PAGE_COMPRESSOR_SWAP_THRESHOLD               (((AVAILABLE_MEMORY) * 10) / (vm_compressor_majorcompact_threshold_divisor ? vm_compressor_majorcompact_threshold_divisor : 10))

#define VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD    (((AVAILABLE_MEMORY) * 10) / (vm_compressor_unthrottle_threshold_divisor ? vm_compressor_unthrottle_threshold_divisor : 10))
#define VM_PAGE_COMPRESSOR_SWAP_RETHROTTLE_THRESHOLD    (((AVAILABLE_MEMORY) * 11) / (vm_compressor_unthrottle_threshold_divisor ? vm_compressor_unthrottle_threshold_divisor : 11))

#define VM_PAGE_COMPRESSOR_SWAP_HAS_CAUGHTUP_THRESHOLD  (((AVAILABLE_MEMORY) * 11) / (vm_compressor_catchup_threshold_divisor ? vm_compressor_catchup_threshold_divisor : 11))
#define VM_PAGE_COMPRESSOR_SWAP_CATCHUP_THRESHOLD       (((AVAILABLE_MEMORY) * 10) / (vm_compressor_catchup_threshold_divisor ? vm_compressor_catchup_threshold_divisor : 10))
#define VM_PAGE_COMPRESSOR_HARD_THROTTLE_THRESHOLD      (((AVAILABLE_MEMORY) * 9) / (vm_compressor_catchup_threshold_divisor ? vm_compressor_catchup_threshold_divisor : 9))

#ifdef  CONFIG_EMBEDDED
#define AVAILABLE_NON_COMPRESSED_MIN                    20000
#define COMPRESSOR_NEEDS_TO_SWAP()              (((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_THRESHOLD) || \
	                                          (AVAILABLE_NON_COMPRESSED_MEMORY < AVAILABLE_NON_COMPRESSED_MIN)) ? 1 : 0)
#else
#define COMPRESSOR_NEEDS_TO_SWAP()              ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_THRESHOLD) ? 1 : 0)
#endif

#define HARD_THROTTLE_LIMIT_REACHED()           ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_HARD_THROTTLE_THRESHOLD) ? 1 : 0)
#define SWAPPER_NEEDS_TO_UNTHROTTLE()           ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) ? 1 : 0)
#define SWAPPER_NEEDS_TO_RETHROTTLE()           ((AVAILABLE_NON_COMPRESSED_MEMORY > VM_PAGE_COMPRESSOR_SWAP_RETHROTTLE_THRESHOLD) ? 1 : 0)
#define SWAPPER_NEEDS_TO_CATCHUP()              ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_CATCHUP_THRESHOLD) ? 1 : 0)
#define SWAPPER_HAS_CAUGHTUP()                  ((AVAILABLE_NON_COMPRESSED_MEMORY > VM_PAGE_COMPRESSOR_SWAP_HAS_CAUGHTUP_THRESHOLD) ? 1 : 0)
#define COMPRESSOR_NEEDS_TO_MINOR_COMPACT()     ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD) ? 1 : 0)


#ifdef  CONFIG_EMBEDDED
#define COMPRESSOR_FREE_RESERVED_LIMIT          28
#else
#define COMPRESSOR_FREE_RESERVED_LIMIT          128
#endif

uint32_t vm_compressor_get_encode_scratch_size(void);
uint32_t vm_compressor_get_decode_scratch_size(void);

#define COMPRESSOR_SCRATCH_BUF_SIZE vm_compressor_get_encode_scratch_size()

#if RECORD_THE_COMPRESSED_DATA
extern void      c_compressed_record_init(void);
extern void      c_compressed_record_write(char *, int);
#endif

extern lck_mtx_t        *c_list_lock;

#if DEVELOPMENT || DEBUG
extern uint32_t vm_ktrace_enabled;

#define VMKDBG(x, ...)          \
MACRO_BEGIN                     \
if (vm_ktrace_enabled) {        \
	KDBG(x, ## __VA_ARGS__);\
}                               \
MACRO_END
#endif
Commit	Line	Data
39236c6e	1	/*
39037602	2	* Copyright (c) 2000-2016 Apple Inc. All rights reserved.
39236c6e A	3	*
39236c6e A	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
0a7de745	5	*
39236c6e A	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
0a7de745	14	*
39236c6e A	15	* Please obtain a copy of the License at
39236c6e A	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
0a7de745	17	*
39236c6e A	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
0a7de745	25	*
39236c6e A	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
	29	#include <kern/kalloc.h>
	30	#include <vm/vm_compressor_pager.h>
	31	#include <vm/vm_kern.h>
	32	#include <vm/vm_page.h>
	33	#include <vm/vm_protos.h>
	34	#include <vm/WKdm_new.h>
	35	#include <vm/vm_object.h>
39037602	36	#include <vm/vm_map.h>
39236c6e A	37	#include <machine/pmap.h>
	38	#include <kern/locks.h>
	39
	40	#include <sys/kdebug.h>
	41
5ba3f43e A	42	#if defined(__arm64__)
	43	#include <arm/proc_reg.h>
	44	#endif
	45
0a7de745 A	46	#define C_SEG_OFFSET_BITS 16
	47	#define C_SEG_BUFSIZE (1024 * 256)
	48	#define C_SEG_MAX_PAGES (C_SEG_BUFSIZE / PAGE_SIZE)
39236c6e	49
5ba3f43e	50	#if CONFIG_EMBEDDED
0a7de745 A	51	#define C_SEG_OFF_LIMIT (C_SEG_BYTES_TO_OFFSET((C_SEG_BUFSIZE - 512)))
0a7de745 A	52	#define C_SEG_ALLOCSIZE (C_SEG_BUFSIZE + PAGE_SIZE)
5ba3f43e	53	#else
0a7de745 A	54	#define C_SEG_OFF_LIMIT (C_SEG_BYTES_TO_OFFSET((C_SEG_BUFSIZE - 128)))
0a7de745 A	55	#define C_SEG_ALLOCSIZE (C_SEG_BUFSIZE)
5ba3f43e	56	#endif
0a7de745	57	#define C_SEG_MAX_POPULATE_SIZE (4 * PAGE_SIZE)
39236c6e	58
5ba3f43e A	59	#if defined(__arm64__)
	60
	61	#if DEVELOPMENT \|\| DEBUG
	62
d9a64523 A	63	#if defined(PLATFORM_WatchOS)
	64	#define VALIDATE_C_SEGMENTS (1)
	65	#endif
5ba3f43e A	66	#endif
	67
	68	#endif
	69
a39ff7e2	70
39037602 A	71	#if DEBUG \|\| COMPRESSOR_INTEGRITY_CHECKS
	72	#define ENABLE_SWAP_CHECKS 1
	73	#define ENABLE_COMPRESSOR_CHECKS 1
5ba3f43e	74	#define POPCOUNT_THE_COMPRESSED_DATA (1)
39037602 A	75	#else
	76	#define ENABLE_SWAP_CHECKS 0
	77	#define ENABLE_COMPRESSOR_CHECKS 0
	78	#endif
39236c6e	79
0a7de745 A	80	#define CHECKSUM_THE_SWAP ENABLE_SWAP_CHECKS /* Debug swap data */
	81	#define CHECKSUM_THE_DATA ENABLE_COMPRESSOR_CHECKS /* Debug compressor/decompressor data */
	82	#define CHECKSUM_THE_COMPRESSED_DATA ENABLE_COMPRESSOR_CHECKS /* Debug compressor/decompressor compressed data */
d9a64523 A	83
d9a64523 A	84	#ifndef VALIDATE_C_SEGMENTS
0a7de745	85	#define VALIDATE_C_SEGMENTS ENABLE_COMPRESSOR_CHECKS /* Debug compaction */
d9a64523	86	#endif
3e170ce0	87
0a7de745	88	#define RECORD_THE_COMPRESSED_DATA 0
3e170ce0	89
3e170ce0	90	struct c_slot {
0a7de745	91	uint64_t c_offset:C_SEG_OFFSET_BITS,
5ba3f43e	92	#if defined(__arm64__)
0a7de745 A	93	c_size:14,
	94	c_codec:1,
	95	c_packed_ptr:33;
5ba3f43e	96	#elif defined(__arm__)
0a7de745 A	97	c_size:12,
	98	c_codec:1,
	99	c_packed_ptr:35;
5ba3f43e	100	#else
0a7de745 A	101	c_size:12,
0a7de745 A	102	c_packed_ptr:36;
5ba3f43e	103	#endif
3e170ce0	104	#if CHECKSUM_THE_DATA
0a7de745	105	unsigned int c_hash_data;
3e170ce0 A	106	#endif
3e170ce0 A	107	#if CHECKSUM_THE_COMPRESSED_DATA
0a7de745	108	unsigned int c_hash_compressed_data;
3e170ce0	109	#endif
5ba3f43e	110	#if POPCOUNT_THE_COMPRESSED_DATA
0a7de745	111	unsigned int c_pop_cdata;
5ba3f43e	112	#endif
3e170ce0 A	113	};
3e170ce0 A	114
0a7de745 A	115	#define C_IS_EMPTY 0
	116	#define C_IS_FREE 1
	117	#define C_IS_FILLING 2
	118	#define C_ON_AGE_Q 3
	119	#define C_ON_SWAPOUT_Q 4
	120	#define C_ON_SWAPPEDOUT_Q 5
	121	#define C_ON_SWAPPEDOUTSPARSE_Q 6
	122	#define C_ON_SWAPPEDIN_Q 7
	123	#define C_ON_MAJORCOMPACT_Q 8
	124	#define C_ON_BAD_Q 9
d9a64523	125	#define C_ON_SWAPIO_Q 10
3e170ce0	126
39236c6e A	127
39236c6e A	128	struct c_segment {
0a7de745 A	129	lck_mtx_t c_lock;
	130	queue_chain_t c_age_list;
	131	queue_chain_t c_list;
39236c6e	132
0a7de745 A	133	#define C_SEG_MAX_LIMIT (1 << 20) /* this needs to track the size of c_mysegno */
	134	uint32_t c_mysegno:20,
	135	c_busy:1,
	136	c_busy_swapping:1,
	137	c_wanted:1,
	138	c_on_minorcompact_q:1, /* can also be on the age_q, the majorcompact_q or the swappedin_q */
3e170ce0	139
0a7de745 A	140	c_state:4, /* what state is the segment in which dictates which q to find it on */
	141	c_overage_swap:1,
	142	c_reserved:3;
5ba3f43e	143
0a7de745 A	144	uint32_t c_creation_ts;
0a7de745 A	145	uint64_t c_generation_id;
5ba3f43e	146
0a7de745 A	147	int32_t c_bytes_used;
	148	int32_t c_bytes_unused;
	149	uint32_t c_slots_used;
3e170ce0	150
0a7de745 A	151	uint16_t c_firstemptyslot;
	152	uint16_t c_nextslot;
	153	uint32_t c_nextoffset;
	154	uint32_t c_populated_offset;
39236c6e	155
0a7de745	156	uint32_t c_swappedin_ts;
39236c6e A	157
	158	union {
	159	int32_t *c_buffer;
	160	uint64_t c_swap_handle;
	161	} c_store;
	162
0a7de745 A	163	#if VALIDATE_C_SEGMENTS
	164	uint32_t c_was_minor_compacted;
	165	uint32_t c_was_major_compacted;
	166	uint32_t c_was_major_donor;
39236c6e	167	#endif
0a7de745 A	168	#if CHECKSUM_THE_SWAP
	169	unsigned int cseg_hash;
	170	unsigned int cseg_swap_size;
39236c6e A	171	#endif /* CHECKSUM_THE_SWAP */
39236c6e A	172
0a7de745	173	thread_t c_busy_for_thread;
fe8ab488	174
0a7de745 A	175	int c_slot_var_array_len;
	176	struct c_slot *c_slot_var_array;
	177	struct c_slot c_slot_fixed_array[0];
39236c6e A	178	};
39236c6e A	179
5ba3f43e A	180
5ba3f43e A	181	struct c_slot_mapping {
0a7de745 A	182	uint32_t s_cseg:22, /* segment number + 1 */
0a7de745 A	183	s_cindx:10; /* index in the segment */
5ba3f43e	184	};
0a7de745	185	#define C_SLOT_MAX_INDEX (1 << 10)
5ba3f43e A	186
	187	typedef struct c_slot_mapping *c_slot_mapping_t;
	188
	189
0a7de745	190	#define C_SEG_SLOT_VAR_ARRAY_MIN_LEN C_SEG_MAX_PAGES
3e170ce0	191
0a7de745 A	192	extern int c_seg_fixed_array_len;
	193	extern vm_offset_t c_buffers;
	194	#define C_SEG_BUFFER_ADDRESS(c_segno) ((c_buffers + ((uint64_t)c_segno * (uint64_t)C_SEG_ALLOCSIZE)))
39236c6e	195
0a7de745	196	#define C_SEG_SLOT_FROM_INDEX(cseg, index) (index < c_seg_fixed_array_len ? &(cseg->c_slot_fixed_array[index]) : &(cseg->c_slot_var_array[index - c_seg_fixed_array_len]))
39236c6e	197
0a7de745 A	198	#define C_SEG_OFFSET_TO_BYTES(off) ((off) * (int) sizeof(int32_t))
0a7de745 A	199	#define C_SEG_BYTES_TO_OFFSET(bytes) ((bytes) / (int) sizeof(int32_t))
39236c6e	200
0a7de745	201	#define C_SEG_UNUSED_BYTES(cseg) (cseg->c_bytes_unused + (C_SEG_OFFSET_TO_BYTES(cseg->c_populated_offset - cseg->c_nextoffset)))
5ba3f43e	202	//todo opensource
39236c6e	203
5ba3f43e	204	#ifndef __PLATFORM_WKDM_ALIGNMENT_MASK__
0a7de745 A	205	#define C_SEG_OFFSET_ALIGNMENT_MASK 0x3ULL
0a7de745 A	206	#define C_SEG_OFFSET_ALIGNMENT_BOUNDARY 0x4
5ba3f43e	207	#else
0a7de745 A	208	#define C_SEG_OFFSET_ALIGNMENT_MASK __PLATFORM_WKDM_ALIGNMENT_MASK__
0a7de745 A	209	#define C_SEG_OFFSET_ALIGNMENT_BOUNDARY __PLATFORM_WKDM_ALIGNMENT_BOUNDARY__
5ba3f43e A	210	#endif
5ba3f43e A	211
0a7de745	212	#define C_SEG_SHOULD_MINORCOMPACT_NOW(cseg) ((C_SEG_UNUSED_BYTES(cseg) >= (C_SEG_BUFSIZE / 4)) ? 1 : 0)
39236c6e	213
5ba3f43e A	214	/*
	215	* the decsion to force a c_seg to be major compacted is based on 2 criteria
	216	* 1) is the c_seg buffer almost empty (i.e. we have a chance to merge it with another c_seg)
	217	* 2) are there at least a minimum number of slots unoccupied so that we have a chance
	218	* of combining this c_seg with another one.
	219	*/
0a7de745 A	220	#define C_SEG_SHOULD_MAJORCOMPACT_NOW(cseg) \
0a7de745 A	221	((((cseg->c_bytes_unused + (C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES(c_seg->c_nextoffset))) >= (C_SEG_BUFSIZE / 8)) && \
5ba3f43e A	222	((C_SLOT_MAX_INDEX - cseg->c_slots_used) > (C_SEG_BUFSIZE / PAGE_SIZE))) \
5ba3f43e A	223	? 1 : 0)
3e170ce0	224
0a7de745 A	225	#define C_SEG_ONDISK_IS_SPARSE(cseg) ((cseg->c_bytes_used < cseg->c_bytes_unused) ? 1 : 0)
	226	#define C_SEG_IS_ONDISK(cseg) ((cseg->c_state == C_ON_SWAPPEDOUT_Q \|\| cseg->c_state == C_ON_SWAPPEDOUTSPARSE_Q))
	227	#define C_SEG_IS_ON_DISK_OR_SOQ(cseg) ((cseg->c_state == C_ON_SWAPPEDOUT_Q \|\| \
	228	cseg->c_state == C_ON_SWAPPEDOUTSPARSE_Q \|\| \
	229	cseg->c_state == C_ON_SWAPOUT_Q \|\| \
	230	cseg->c_state == C_ON_SWAPIO_Q))
	231
	232
	233	#define C_SEG_WAKEUP_DONE(cseg) \
	234	MACRO_BEGIN \
	235	assert((cseg)->c_busy); \
	236	(cseg)->c_busy = 0; \
	237	assert((cseg)->c_busy_for_thread != NULL); \
ea3f0419	238	(cseg)->c_busy_for_thread = NULL; \
0a7de745 A	239	if ((cseg)->c_wanted) { \
	240	(cseg)->c_wanted = 0; \
	241	thread_wakeup((event_t) (cseg)); \
	242	} \
39236c6e A	243	MACRO_END
39236c6e A	244
0a7de745 A	245	#define C_SEG_BUSY(cseg) \
	246	MACRO_BEGIN \
	247	assert((cseg)->c_busy == 0); \
	248	(cseg)->c_busy = 1; \
	249	assert((cseg)->c_busy_for_thread == NULL); \
ea3f0419	250	(cseg)->c_busy_for_thread = current_thread(); \
fe8ab488	251	MACRO_END
0a7de745	252
fe8ab488	253
39037602 A	254	extern vm_map_t compressor_map;
39037602 A	255
5ba3f43e A	256	#if DEVELOPMENT \|\| DEBUG
	257	extern boolean_t write_protect_c_segs;
	258	extern int vm_compressor_test_seg_wp;
	259
0a7de745 A	260	#define C_SEG_MAKE_WRITEABLE(cseg) \
	261	MACRO_BEGIN \
	262	if (write_protect_c_segs) { \
	263	vm_map_protect(compressor_map, \
	264	(vm_map_offset_t)cseg->c_store.c_buffer, \
	265	(vm_map_offset_t)&cseg->c_store.c_buffer[C_SEG_BYTES_TO_OFFSET(C_SEG_ALLOCSIZE)],\
	266	VM_PROT_READ \| VM_PROT_WRITE, \
	267	0); \
	268	} \
39037602 A	269	MACRO_END
39037602 A	270
0a7de745 A	271	#define C_SEG_WRITE_PROTECT(cseg) \
	272	MACRO_BEGIN \
	273	if (write_protect_c_segs) { \
	274	vm_map_protect(compressor_map, \
	275	(vm_map_offset_t)cseg->c_store.c_buffer, \
	276	(vm_map_offset_t)&cseg->c_store.c_buffer[C_SEG_BYTES_TO_OFFSET(C_SEG_ALLOCSIZE)],\
	277	VM_PROT_READ, \
	278	0); \
	279	} \
	280	if (vm_compressor_test_seg_wp) { \
	281	volatile uint32_t vmtstmp = (volatile uint32_t )cseg->c_store.c_buffer; \
	282	(volatile uint32_t )cseg->c_store.c_buffer = 0xDEADABCD; \
	283	(void) vmtstmp; \
	284	} \
39037602 A	285	MACRO_END
39037602 A	286	#endif
39236c6e	287
0a7de745 A	288	typedef struct c_segment *c_segment_t;
0a7de745 A	289	typedef struct c_slot *c_slot_t;
39236c6e A	290
	291	uint64_t vm_compressor_total_compressions(void);
	292	void vm_wake_compactor_swapper(void);
39037602	293	void vm_run_compactor(void);
fe8ab488	294	void vm_thrashing_jetsam_done(void);
39236c6e	295	void vm_consider_waking_compactor_swapper(void);
3e170ce0	296	void vm_consider_swapping(void);
39236c6e A	297	void vm_compressor_flush(void);
	298	void c_seg_free(c_segment_t);
	299	void c_seg_free_locked(c_segment_t);
	300	void c_seg_insert_into_age_q(c_segment_t);
39037602	301	void c_seg_need_delayed_compaction(c_segment_t, boolean_t);
39236c6e A	302
	303	void vm_decompressor_lock(void);
	304	void vm_decompressor_unlock(void);
	305
8a3053a0	306	void vm_compressor_delay_trim(void);
39236c6e A	307	void vm_compressor_do_warmup(void);
	308	void vm_compressor_record_warmup_start(void);
	309	void vm_compressor_record_warmup_end(void);
	310
0a7de745	311	int vm_wants_task_throttled(task_t);
39236c6e	312
0a7de745 A	313	extern void vm_compaction_swapper_do_init(void);
	314	extern void vm_compressor_swap_init(void);
	315	extern void vm_compressor_init_locks(void);
	316	extern lck_rw_t c_master_lock;
39236c6e	317
fe8ab488	318	#if ENCRYPTED_SWAP
0a7de745	319	extern void vm_swap_decrypt(c_segment_t);
fe8ab488	320	#endif /* ENCRYPTED_SWAP */
39236c6e	321
0a7de745 A	322	extern int vm_swap_low_on_space(void);
	323	extern kern_return_t vm_swap_get(c_segment_t, uint64_t, uint64_t);
	324	extern void vm_swap_free(uint64_t);
	325	extern void vm_swap_consider_defragmenting(int);
	326
	327	extern void c_seg_swapin_requeue(c_segment_t, boolean_t, boolean_t, boolean_t);
	328	extern int c_seg_swapin(c_segment_t, boolean_t, boolean_t);
	329	extern void c_seg_wait_on_busy(c_segment_t);
	330	extern void c_seg_trim_tail(c_segment_t);
	331	extern void c_seg_switch_state(c_segment_t, int, boolean_t);
	332
	333	extern boolean_t fastwake_recording_in_progress;
	334	extern int compaction_swapper_inited;
	335	extern int compaction_swapper_running;
	336	extern uint64_t vm_swap_put_failures;
	337
	338	extern int c_overage_swapped_count;
	339	extern int c_overage_swapped_limit;
	340
	341	extern queue_head_t c_minor_list_head;
	342	extern queue_head_t c_age_list_head;
	343	extern queue_head_t c_swapout_list_head;
	344	extern queue_head_t c_swappedout_list_head;
	345	extern queue_head_t c_swappedout_sparse_list_head;
	346
	347	extern uint32_t c_age_count;
	348	extern uint32_t c_swapout_count;
	349	extern uint32_t c_swappedout_count;
	350	extern uint32_t c_swappedout_sparse_count;
	351
	352	extern int64_t compressor_bytes_used;
	353	extern uint64_t first_c_segment_to_warm_generation_id;
	354	extern uint64_t last_c_segment_to_warm_generation_id;
	355	extern boolean_t hibernate_flushing;
	356	extern boolean_t hibernate_no_swapspace;
	357	extern boolean_t hibernate_in_progress_with_pinned_swap;
	358	extern uint32_t swapout_target_age;
39236c6e A	359
	360	extern void c_seg_insert_into_q(queue_head_t *, c_segment_t);
	361
0a7de745 A	362	extern uint32_t vm_compressor_minorcompact_threshold_divisor;
	363	extern uint32_t vm_compressor_majorcompact_threshold_divisor;
	364	extern uint32_t vm_compressor_unthrottle_threshold_divisor;
	365	extern uint32_t vm_compressor_catchup_threshold_divisor;
d9a64523	366
0a7de745 A	367	extern uint32_t vm_compressor_minorcompact_threshold_divisor_overridden;
	368	extern uint32_t vm_compressor_majorcompact_threshold_divisor_overridden;
	369	extern uint32_t vm_compressor_unthrottle_threshold_divisor_overridden;
	370	extern uint32_t vm_compressor_catchup_threshold_divisor_overridden;
d9a64523	371
fe8ab488	372	extern uint64_t vm_compressor_compute_elapsed_msecs(clock_sec_t, clock_nsec_t, clock_sec_t, clock_nsec_t);
39236c6e	373
ea3f0419 A	374	extern void kdp_compressor_busy_find_owner(event64_t wait_event, thread_waitinfo_t *waitinfo);
ea3f0419 A	375
0a7de745 A	376	#define PAGE_REPLACEMENT_DISALLOWED(enable) (enable == TRUE ? lck_rw_lock_shared(&c_master_lock) : lck_rw_done(&c_master_lock))
0a7de745 A	377	#define PAGE_REPLACEMENT_ALLOWED(enable) (enable == TRUE ? lck_rw_lock_exclusive(&c_master_lock) : lck_rw_done(&c_master_lock))
39236c6e A	378
39236c6e A	379
0a7de745 A	380	#define AVAILABLE_NON_COMPRESSED_MEMORY (vm_page_active_count + vm_page_inactive_count + vm_page_free_count + vm_page_speculative_count)
0a7de745 A	381	#define AVAILABLE_MEMORY (AVAILABLE_NON_COMPRESSED_MEMORY + VM_PAGE_COMPRESSOR_COUNT)
d9a64523 A	382
	383	/*
	384	* TODO, there may be a minor optimisation opportunity to replace these divisions
39037602	385	* with multiplies and shifts
d9a64523 A	386	*
	387	* By multiplying by 10, the divisors can have more precision w/o resorting to floating point... a divisor specified as 25 is in reality a divide by 2.5
	388	* By multiplying by 9, you get a number ~11% smaller which allows us to have another limit point derived from the same base
	389	* By multiplying by 11, you get a number ~10% bigger which allows us to generate a reset limit derived from the same base which is useful for hysteresis
39037602	390	*/
39236c6e	391
0a7de745 A	392	#define VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD (((AVAILABLE_MEMORY) * 10) / (vm_compressor_minorcompact_threshold_divisor ? vm_compressor_minorcompact_threshold_divisor : 10))
0a7de745 A	393	#define VM_PAGE_COMPRESSOR_SWAP_THRESHOLD (((AVAILABLE_MEMORY) * 10) / (vm_compressor_majorcompact_threshold_divisor ? vm_compressor_majorcompact_threshold_divisor : 10))
d9a64523	394
0a7de745 A	395	#define VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD (((AVAILABLE_MEMORY) * 10) / (vm_compressor_unthrottle_threshold_divisor ? vm_compressor_unthrottle_threshold_divisor : 10))
0a7de745 A	396	#define VM_PAGE_COMPRESSOR_SWAP_RETHROTTLE_THRESHOLD (((AVAILABLE_MEMORY) * 11) / (vm_compressor_unthrottle_threshold_divisor ? vm_compressor_unthrottle_threshold_divisor : 11))
d9a64523	397
0a7de745 A	398	#define VM_PAGE_COMPRESSOR_SWAP_HAS_CAUGHTUP_THRESHOLD (((AVAILABLE_MEMORY) * 11) / (vm_compressor_catchup_threshold_divisor ? vm_compressor_catchup_threshold_divisor : 11))
0a7de745 A	399	#define VM_PAGE_COMPRESSOR_SWAP_CATCHUP_THRESHOLD (((AVAILABLE_MEMORY) * 10) / (vm_compressor_catchup_threshold_divisor ? vm_compressor_catchup_threshold_divisor : 10))
d9a64523	400	#define VM_PAGE_COMPRESSOR_HARD_THROTTLE_THRESHOLD (((AVAILABLE_MEMORY) * 9) / (vm_compressor_catchup_threshold_divisor ? vm_compressor_catchup_threshold_divisor : 9))
39236c6e	401
0a7de745 A	402	#ifdef CONFIG_EMBEDDED
	403	#define AVAILABLE_NON_COMPRESSED_MIN 20000
	404	#define COMPRESSOR_NEEDS_TO_SWAP() (((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_THRESHOLD) \|\| \
	405	(AVAILABLE_NON_COMPRESSED_MEMORY < AVAILABLE_NON_COMPRESSED_MIN)) ? 1 : 0)
5ba3f43e	406	#else
0a7de745	407	#define COMPRESSOR_NEEDS_TO_SWAP() ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_THRESHOLD) ? 1 : 0)
5ba3f43e	408	#endif
39236c6e	409
0a7de745 A	410	#define HARD_THROTTLE_LIMIT_REACHED() ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_HARD_THROTTLE_THRESHOLD) ? 1 : 0)
	411	#define SWAPPER_NEEDS_TO_UNTHROTTLE() ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) ? 1 : 0)
	412	#define SWAPPER_NEEDS_TO_RETHROTTLE() ((AVAILABLE_NON_COMPRESSED_MEMORY > VM_PAGE_COMPRESSOR_SWAP_RETHROTTLE_THRESHOLD) ? 1 : 0)
	413	#define SWAPPER_NEEDS_TO_CATCHUP() ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_CATCHUP_THRESHOLD) ? 1 : 0)
	414	#define SWAPPER_HAS_CAUGHTUP() ((AVAILABLE_NON_COMPRESSED_MEMORY > VM_PAGE_COMPRESSOR_SWAP_HAS_CAUGHTUP_THRESHOLD) ? 1 : 0)
	415	#define COMPRESSOR_NEEDS_TO_MINOR_COMPACT() ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD) ? 1 : 0)
39236c6e	416
39236c6e	417
0a7de745 A	418	#ifdef CONFIG_EMBEDDED
0a7de745 A	419	#define COMPRESSOR_FREE_RESERVED_LIMIT 28
5ba3f43e	420	#else
0a7de745	421	#define COMPRESSOR_FREE_RESERVED_LIMIT 128
5ba3f43e	422	#endif
39236c6e	423
39037602 A	424	uint32_t vm_compressor_get_encode_scratch_size(void);
39037602 A	425	uint32_t vm_compressor_get_decode_scratch_size(void);
39236c6e	426
39037602	427	#define COMPRESSOR_SCRATCH_BUF_SIZE vm_compressor_get_encode_scratch_size()
39236c6e	428
3e170ce0	429	#if RECORD_THE_COMPRESSED_DATA
0a7de745 A	430	extern void c_compressed_record_init(void);
0a7de745 A	431	extern void c_compressed_record_write(char *, int);
3e170ce0 A	432	#endif
3e170ce0 A	433
0a7de745	434	extern lck_mtx_t *c_list_lock;
5ba3f43e A	435
	436	#if DEVELOPMENT \|\| DEBUG
	437	extern uint32_t vm_ktrace_enabled;
	438
0a7de745 A	439	#define VMKDBG(x, ...) \
	440	MACRO_BEGIN \
	441	if (vm_ktrace_enabled) { \
5ba3f43e	442	KDBG(x, ## __VA_ARGS__);\
0a7de745	443	} \
5ba3f43e A	444	MACRO_END
5ba3f43e A	445	#endif