[apple/xnu.git] / osfmk / vm / vm_compressor.h

/*
 * Copyright (c) 2000-2016 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#include <kern/kalloc.h>
#include <vm/vm_compressor_pager.h>
#include <vm/vm_kern.h>
#include <vm/vm_page.h>
#include <vm/vm_protos.h>
#include <vm/WKdm_new.h>
#include <vm/vm_object.h>
#include <vm/vm_map.h>
#include <machine/pmap.h>
#include <kern/locks.h>

#include <sys/kdebug.h>

#if defined(__arm64__)
#include <arm/proc_reg.h>
#endif

#define C_SEG_OFFSET_BITS	16
#define C_SEG_BUFSIZE		(1024 * 256)
#define	C_SEG_MAX_PAGES		(C_SEG_BUFSIZE / PAGE_SIZE)

#if CONFIG_EMBEDDED
#define C_SEG_OFF_LIMIT		(C_SEG_BYTES_TO_OFFSET((C_SEG_BUFSIZE - 512)))
#define C_SEG_ALLOCSIZE		(C_SEG_BUFSIZE + PAGE_SIZE)
#else
#define C_SEG_OFF_LIMIT		(C_SEG_BYTES_TO_OFFSET((C_SEG_BUFSIZE - 128)))
#define C_SEG_ALLOCSIZE		(C_SEG_BUFSIZE)
#endif
#define C_SEG_MAX_POPULATE_SIZE	(4 * PAGE_SIZE)

#if defined(__arm64__)

#if DEVELOPMENT || DEBUG


#endif

#endif

#if DEBUG || COMPRESSOR_INTEGRITY_CHECKS
#define ENABLE_SWAP_CHECKS 1
#define ENABLE_COMPRESSOR_CHECKS 1
#define POPCOUNT_THE_COMPRESSED_DATA (1)
#else
#define ENABLE_SWAP_CHECKS 0
#define ENABLE_COMPRESSOR_CHECKS 0
#endif

#define CHECKSUM_THE_SWAP		ENABLE_SWAP_CHECKS	/* Debug swap data */
#define CHECKSUM_THE_DATA		ENABLE_COMPRESSOR_CHECKS	/* Debug compressor/decompressor data */
#define CHECKSUM_THE_COMPRESSED_DATA	ENABLE_COMPRESSOR_CHECKS	/* Debug compressor/decompressor compressed data */
#define VALIDATE_C_SEGMENTS		ENABLE_COMPRESSOR_CHECKS	/* Debug compaction */

#define RECORD_THE_COMPRESSED_DATA	0

struct c_slot {
	uint64_t	c_offset:C_SEG_OFFSET_BITS,
#if defined(__arm64__)
		        c_size:14,
			c_codec:1,
		        c_packed_ptr:33;
#elif defined(__arm__)
		        c_size:12,
			c_codec:1,
		        c_packed_ptr:35;
#else
			c_size:12,
		        c_packed_ptr:36;
#endif
#if CHECKSUM_THE_DATA
	unsigned int	c_hash_data;
#endif
#if CHECKSUM_THE_COMPRESSED_DATA
	unsigned int	c_hash_compressed_data;
#endif
#if POPCOUNT_THE_COMPRESSED_DATA
	unsigned int	c_pop_cdata;
#endif
};

#define	C_IS_EMPTY		0
#define	C_IS_FREE		1
#define	C_IS_FILLING		2
#define C_ON_AGE_Q		3
#define C_ON_SWAPOUT_Q		4
#define C_ON_SWAPPEDOUT_Q	5
#define	C_ON_SWAPPEDOUTSPARSE_Q	6
#define	C_ON_SWAPPEDIN_Q	7
#define	C_ON_MAJORCOMPACT_Q	8
#define	C_ON_BAD_Q		9


struct c_segment {
	lck_mtx_t	c_lock;
	queue_chain_t	c_age_list;
	queue_chain_t	c_list;

#define C_SEG_MAX_LIMIT		(1 << 20)	/* this needs to track the size of c_mysegno */
	uint32_t	c_mysegno:20,
		        c_busy:1,
		        c_busy_swapping:1,
	                c_wanted:1,
		        c_on_minorcompact_q:1,	/* can also be on the age_q, the majorcompact_q or the swappedin_q */

		        c_state:4,		/* what state is the segment in which dictates which q to find it on */
		        c_overage_swap:1,
	                c_reserved:3;

	uint32_t	c_creation_ts;
	uint64_t	c_generation_id;

        int32_t		c_bytes_used;
        int32_t		c_bytes_unused;
        uint32_t	c_slots_used;

	uint16_t	c_firstemptyslot;
	uint16_t	c_nextslot;
	uint32_t	c_nextoffset;
	uint32_t	c_populated_offset;

	uint32_t	c_swappedin_ts;

	union {
		int32_t *c_buffer;
		uint64_t c_swap_handle;
	} c_store;

#if 	VALIDATE_C_SEGMENTS
        uint32_t	c_was_minor_compacted;
        uint32_t	c_was_major_compacted;
	uint32_t	c_was_major_donor;
#endif
#if CHECKSUM_THE_SWAP	
	unsigned int	cseg_hash;
	unsigned int	cseg_swap_size;
#endif /* CHECKSUM_THE_SWAP */

#if MACH_ASSERT
	thread_t	c_busy_for_thread;
#endif /* MACH_ASSERT */

	int		c_slot_var_array_len;
	struct	c_slot	*c_slot_var_array;
	struct	c_slot	c_slot_fixed_array[0];
};


struct  c_slot_mapping {
        uint32_t        s_cseg:22, 	/* segment number + 1 */
			s_cindx:10;	/* index in the segment */
};
#define C_SLOT_MAX_INDEX	(1 << 10)

typedef struct c_slot_mapping *c_slot_mapping_t;


#define C_SEG_SLOT_VAR_ARRAY_MIN_LEN	C_SEG_MAX_PAGES

extern	int		c_seg_fixed_array_len;
extern	vm_offset_t	c_buffers;
#define	C_SEG_BUFFER_ADDRESS(c_segno)	((c_buffers + ((uint64_t)c_segno * (uint64_t)C_SEG_ALLOCSIZE)))

#define C_SEG_SLOT_FROM_INDEX(cseg, index)	(index < c_seg_fixed_array_len ? &(cseg->c_slot_fixed_array[index]) : &(cseg->c_slot_var_array[index - c_seg_fixed_array_len]))

#define	C_SEG_OFFSET_TO_BYTES(off)	((off) * (int) sizeof(int32_t))
#define C_SEG_BYTES_TO_OFFSET(bytes)	((bytes) / (int) sizeof(int32_t))

#define C_SEG_UNUSED_BYTES(cseg)	(cseg->c_bytes_unused + (C_SEG_OFFSET_TO_BYTES(cseg->c_populated_offset - cseg->c_nextoffset)))
//todo opensource

#ifndef __PLATFORM_WKDM_ALIGNMENT_MASK__
#define C_SEG_OFFSET_ALIGNMENT_MASK	0x3ULL
#define C_SEG_OFFSET_ALIGNMENT_BOUNDARY	0x4
#else
#define C_SEG_OFFSET_ALIGNMENT_MASK	__PLATFORM_WKDM_ALIGNMENT_MASK__
#define C_SEG_OFFSET_ALIGNMENT_BOUNDARY	__PLATFORM_WKDM_ALIGNMENT_BOUNDARY__
#endif

#define C_SEG_SHOULD_MINORCOMPACT_NOW(cseg)	((C_SEG_UNUSED_BYTES(cseg) >= (C_SEG_BUFSIZE / 4)) ? 1 : 0)

/*
 * the decsion to force a c_seg to be major compacted is based on 2 criteria
 * 1) is the c_seg buffer almost empty (i.e. we have a chance to merge it with another c_seg)
 * 2) are there at least a minimum number of slots unoccupied so that we have a chance
 *    of combining this c_seg with another one.
 */
#define C_SEG_SHOULD_MAJORCOMPACT_NOW(cseg)											\
	((((cseg->c_bytes_unused + (C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES(c_seg->c_nextoffset))) >= (C_SEG_BUFSIZE / 8)) &&	\
	  ((C_SLOT_MAX_INDEX - cseg->c_slots_used) > (C_SEG_BUFSIZE / PAGE_SIZE))) \
	? 1 : 0)

#define	C_SEG_ONDISK_IS_SPARSE(cseg)	((cseg->c_bytes_used < cseg->c_bytes_unused) ? 1 : 0)
#define C_SEG_IS_ONDISK(cseg)		((cseg->c_state == C_ON_SWAPPEDOUT_Q || cseg->c_state == C_ON_SWAPPEDOUTSPARSE_Q))


#define C_SEG_WAKEUP_DONE(cseg)				\
	MACRO_BEGIN					\
	assert((cseg)->c_busy);				\
	(cseg)->c_busy = 0;				\
	assert((cseg)->c_busy_for_thread != NULL);	\
	assert((((cseg)->c_busy_for_thread = NULL), TRUE));	\
	if ((cseg)->c_wanted) {				\
		(cseg)->c_wanted = 0;			\
		thread_wakeup((event_t) (cseg));	\
	}						\
	MACRO_END

#define C_SEG_BUSY(cseg)				\
	MACRO_BEGIN					\
	assert((cseg)->c_busy == 0);			\
	(cseg)->c_busy = 1;				\
	assert((cseg)->c_busy_for_thread == NULL);	\
	assert((((cseg)->c_busy_for_thread = current_thread()), TRUE));	\
	MACRO_END
	

extern vm_map_t compressor_map;

#if DEVELOPMENT || DEBUG
extern boolean_t write_protect_c_segs;
extern int vm_compressor_test_seg_wp;

#define	C_SEG_MAKE_WRITEABLE(cseg)			\
	MACRO_BEGIN					\
	if (write_protect_c_segs) {			\
		vm_map_protect(compressor_map,			\
			       (vm_map_offset_t)cseg->c_store.c_buffer,		\
			       (vm_map_offset_t)&cseg->c_store.c_buffer[C_SEG_BYTES_TO_OFFSET(C_SEG_ALLOCSIZE)],\
			       VM_PROT_READ | VM_PROT_WRITE,	\
			       0);				\
	}				\
	MACRO_END

#define	C_SEG_WRITE_PROTECT(cseg)			\
	MACRO_BEGIN					\
	if (write_protect_c_segs) {			\
		vm_map_protect(compressor_map,			\
			       (vm_map_offset_t)cseg->c_store.c_buffer,		\
			       (vm_map_offset_t)&cseg->c_store.c_buffer[C_SEG_BYTES_TO_OFFSET(C_SEG_ALLOCSIZE)],\
			       VM_PROT_READ,			\
			       0);				\
	}							\
	if (vm_compressor_test_seg_wp) {				\
		volatile uint32_t vmtstmp = *(volatile uint32_t *)cseg->c_store.c_buffer; \
		*(volatile uint32_t *)cseg->c_store.c_buffer = 0xDEADABCD; \
		(void) vmtstmp;						\
	}								\
	MACRO_END
#endif

typedef	struct c_segment *c_segment_t;
typedef struct c_slot	*c_slot_t;

uint64_t vm_compressor_total_compressions(void);
void vm_wake_compactor_swapper(void);
void vm_run_compactor(void);
void vm_thrashing_jetsam_done(void);
void vm_consider_waking_compactor_swapper(void);
void vm_consider_swapping(void);
void vm_compressor_flush(void);
void c_seg_free(c_segment_t);
void c_seg_free_locked(c_segment_t);
void c_seg_insert_into_age_q(c_segment_t);
void c_seg_need_delayed_compaction(c_segment_t, boolean_t);

void vm_decompressor_lock(void);
void vm_decompressor_unlock(void);

void vm_compressor_delay_trim(void);
void vm_compressor_do_warmup(void);
void vm_compressor_record_warmup_start(void);
void vm_compressor_record_warmup_end(void);

int			vm_wants_task_throttled(task_t);

extern void		vm_compaction_swapper_do_init(void);
extern void		vm_compressor_swap_init(void);
extern void		vm_compressor_init_locks(void);
extern lck_rw_t		c_master_lock;

#if ENCRYPTED_SWAP
extern void		vm_swap_decrypt(c_segment_t);
#endif /* ENCRYPTED_SWAP */

extern int		vm_swap_low_on_space(void);
extern kern_return_t	vm_swap_get(c_segment_t, uint64_t, uint64_t);
extern void		vm_swap_free(uint64_t);
extern void		vm_swap_consider_defragmenting(void);

extern void		c_seg_swapin_requeue(c_segment_t, boolean_t, boolean_t, boolean_t);
extern int		c_seg_swapin(c_segment_t, boolean_t, boolean_t);
extern void		c_seg_wait_on_busy(c_segment_t);
extern void		c_seg_trim_tail(c_segment_t);
extern void		c_seg_switch_state(c_segment_t, int, boolean_t);

extern boolean_t	fastwake_recording_in_progress;
extern int		compaction_swapper_inited;
extern int		compaction_swapper_running;
extern uint64_t		vm_swap_put_failures;

extern int		c_overage_swapped_count;
extern int		c_overage_swapped_limit;

extern queue_head_t	c_minor_list_head;
extern queue_head_t	c_age_list_head;
extern queue_head_t	c_swapout_list_head;
extern queue_head_t	c_swappedout_list_head;
extern queue_head_t	c_swappedout_sparse_list_head;

extern uint32_t		c_age_count;
extern uint32_t		c_swapout_count;
extern uint32_t		c_swappedout_count;
extern uint32_t		c_swappedout_sparse_count;

extern int64_t		compressor_bytes_used;
extern uint64_t		first_c_segment_to_warm_generation_id;
extern uint64_t		last_c_segment_to_warm_generation_id;
extern boolean_t	hibernate_flushing;
extern boolean_t	hibernate_no_swapspace;
extern boolean_t	hibernate_in_progress_with_pinned_swap;
extern uint32_t		swapout_target_age;

extern void c_seg_insert_into_q(queue_head_t *, c_segment_t);

extern uint32_t	vm_compressor_minorcompact_threshold_divisor;
extern uint32_t	vm_compressor_majorcompact_threshold_divisor;
extern uint32_t	vm_compressor_unthrottle_threshold_divisor;
extern uint32_t	vm_compressor_catchup_threshold_divisor;
extern uint64_t vm_compressor_compute_elapsed_msecs(clock_sec_t, clock_nsec_t, clock_sec_t, clock_nsec_t);

#define PAGE_REPLACEMENT_DISALLOWED(enable)	(enable == TRUE ? lck_rw_lock_shared(&c_master_lock) : lck_rw_done(&c_master_lock))
#define PAGE_REPLACEMENT_ALLOWED(enable)	(enable == TRUE ? lck_rw_lock_exclusive(&c_master_lock) : lck_rw_done(&c_master_lock))


#define AVAILABLE_NON_COMPRESSED_MEMORY		(vm_page_active_count + vm_page_inactive_count + vm_page_free_count + vm_page_speculative_count)
#define AVAILABLE_MEMORY			(AVAILABLE_NON_COMPRESSED_MEMORY + VM_PAGE_COMPRESSOR_COUNT)
/* TODO, there may be a minor optimisation opportunity to replace these divisions
 * with multiplies and shifts
 */

#define	VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD		(((AVAILABLE_MEMORY) * 10) / (vm_compressor_minorcompact_threshold_divisor ? vm_compressor_minorcompact_threshold_divisor : 1))
#define	VM_PAGE_COMPRESSOR_SWAP_THRESHOLD		(((AVAILABLE_MEMORY) * 10) / (vm_compressor_majorcompact_threshold_divisor ? vm_compressor_majorcompact_threshold_divisor : 1))
#define	VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD	(((AVAILABLE_MEMORY) * 10) / (vm_compressor_unthrottle_threshold_divisor ? vm_compressor_unthrottle_threshold_divisor : 1))
#define VM_PAGE_COMPRESSOR_SWAP_CATCHUP_THRESHOLD	(((AVAILABLE_MEMORY) * 10) / (vm_compressor_catchup_threshold_divisor ? vm_compressor_catchup_threshold_divisor : 1))

#ifdef	CONFIG_EMBEDDED
#define AVAILABLE_NON_COMPRESSED_MIN			20000
#define COMPRESSOR_NEEDS_TO_SWAP() 		(((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_THRESHOLD) || \
						  (AVAILABLE_NON_COMPRESSED_MEMORY < AVAILABLE_NON_COMPRESSED_MIN)) ? 1 : 0)
#else
#define COMPRESSOR_NEEDS_TO_SWAP() 		((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_THRESHOLD) ? 1 : 0)
#endif

#define VM_PAGEOUT_SCAN_NEEDS_TO_THROTTLE()				\
	(vm_compressor_mode == VM_PAGER_COMPRESSOR_WITH_SWAP &&		\
	 ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_CATCHUP_THRESHOLD) ? 1 : 0))
#define HARD_THROTTLE_LIMIT_REACHED()		((AVAILABLE_NON_COMPRESSED_MEMORY < (VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) / 2) ? 1 : 0)
#define SWAPPER_NEEDS_TO_UNTHROTTLE()		((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) ? 1 : 0)
#define COMPRESSOR_NEEDS_TO_MINOR_COMPACT()	((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD) ? 1 : 0)


#ifdef	CONFIG_EMBEDDED
#define COMPRESSOR_FREE_RESERVED_LIMIT		28
#else
#define COMPRESSOR_FREE_RESERVED_LIMIT		128
#endif

uint32_t vm_compressor_get_encode_scratch_size(void);
uint32_t vm_compressor_get_decode_scratch_size(void);

#define COMPRESSOR_SCRATCH_BUF_SIZE vm_compressor_get_encode_scratch_size()

#if RECORD_THE_COMPRESSED_DATA
extern void 	 c_compressed_record_init(void);
extern void 	 c_compressed_record_write(char *, int);
#endif

extern lck_mtx_t	*c_list_lock;

#if DEVELOPMENT || DEBUG
extern uint32_t vm_ktrace_enabled;

#define VMKDBG(x, ...)		\
MACRO_BEGIN			\
if (vm_ktrace_enabled) {	\
	KDBG(x, ## __VA_ARGS__);\
}				\
MACRO_END
#endif
Commit	Line	Data
39236c6e	1	/*
39037602	2	* Copyright (c) 2000-2016 Apple Inc. All rights reserved.
39236c6e A	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
	29	#include <kern/kalloc.h>
	30	#include <vm/vm_compressor_pager.h>
	31	#include <vm/vm_kern.h>
	32	#include <vm/vm_page.h>
	33	#include <vm/vm_protos.h>
	34	#include <vm/WKdm_new.h>
	35	#include <vm/vm_object.h>
39037602	36	#include <vm/vm_map.h>
39236c6e A	37	#include <machine/pmap.h>
	38	#include <kern/locks.h>
	39
	40	#include <sys/kdebug.h>
	41
5ba3f43e A	42	#if defined(__arm64__)
	43	#include <arm/proc_reg.h>
	44	#endif
	45
39236c6e A	46	#define C_SEG_OFFSET_BITS 16
39236c6e A	47	#define C_SEG_BUFSIZE (1024 * 256)
3e170ce0	48	#define C_SEG_MAX_PAGES (C_SEG_BUFSIZE / PAGE_SIZE)
39236c6e	49
5ba3f43e A	50	#if CONFIG_EMBEDDED
	51	#define C_SEG_OFF_LIMIT (C_SEG_BYTES_TO_OFFSET((C_SEG_BUFSIZE - 512)))
	52	#define C_SEG_ALLOCSIZE (C_SEG_BUFSIZE + PAGE_SIZE)
	53	#else
3e170ce0 A	54	#define C_SEG_OFF_LIMIT (C_SEG_BYTES_TO_OFFSET((C_SEG_BUFSIZE - 128)))
3e170ce0 A	55	#define C_SEG_ALLOCSIZE (C_SEG_BUFSIZE)
5ba3f43e	56	#endif
3e170ce0	57	#define C_SEG_MAX_POPULATE_SIZE (4 * PAGE_SIZE)
39236c6e	58
5ba3f43e A	59	#if defined(__arm64__)
	60
	61	#if DEVELOPMENT \|\| DEBUG
	62
	63
	64	#endif
	65
	66	#endif
	67
39037602 A	68	#if DEBUG \|\| COMPRESSOR_INTEGRITY_CHECKS
	69	#define ENABLE_SWAP_CHECKS 1
	70	#define ENABLE_COMPRESSOR_CHECKS 1
5ba3f43e	71	#define POPCOUNT_THE_COMPRESSED_DATA (1)
39037602 A	72	#else
	73	#define ENABLE_SWAP_CHECKS 0
	74	#define ENABLE_COMPRESSOR_CHECKS 0
	75	#endif
39236c6e	76
39037602 A	77	#define CHECKSUM_THE_SWAP ENABLE_SWAP_CHECKS /* Debug swap data */
	78	#define CHECKSUM_THE_DATA ENABLE_COMPRESSOR_CHECKS /* Debug compressor/decompressor data */
	79	#define CHECKSUM_THE_COMPRESSED_DATA ENABLE_COMPRESSOR_CHECKS /* Debug compressor/decompressor compressed data */
	80	#define VALIDATE_C_SEGMENTS ENABLE_COMPRESSOR_CHECKS /* Debug compaction */
3e170ce0 A	81
	82	#define RECORD_THE_COMPRESSED_DATA 0
	83
3e170ce0 A	84	struct c_slot {
3e170ce0 A	85	uint64_t c_offset:C_SEG_OFFSET_BITS,
5ba3f43e A	86	#if defined(__arm64__)
	87	c_size:14,
	88	c_codec:1,
	89	c_packed_ptr:33;
	90	#elif defined(__arm__)
	91	c_size:12,
	92	c_codec:1,
	93	c_packed_ptr:35;
	94	#else
39037602	95	c_size:12,
3e170ce0	96	c_packed_ptr:36;
5ba3f43e	97	#endif
3e170ce0 A	98	#if CHECKSUM_THE_DATA
	99	unsigned int c_hash_data;
	100	#endif
	101	#if CHECKSUM_THE_COMPRESSED_DATA
	102	unsigned int c_hash_compressed_data;
	103	#endif
5ba3f43e A	104	#if POPCOUNT_THE_COMPRESSED_DATA
	105	unsigned int c_pop_cdata;
	106	#endif
3e170ce0 A	107	};
	108
	109	#define C_IS_EMPTY 0
	110	#define C_IS_FREE 1
	111	#define C_IS_FILLING 2
	112	#define C_ON_AGE_Q 3
	113	#define C_ON_SWAPOUT_Q 4
	114	#define C_ON_SWAPPEDOUT_Q 5
	115	#define C_ON_SWAPPEDOUTSPARSE_Q 6
	116	#define C_ON_SWAPPEDIN_Q 7
	117	#define C_ON_MAJORCOMPACT_Q 8
	118	#define C_ON_BAD_Q 9
	119
39236c6e A	120
39236c6e A	121	struct c_segment {
39236c6e	122	lck_mtx_t c_lock;
39236c6e A	123	queue_chain_t c_age_list;
	124	queue_chain_t c_list;
	125
5ba3f43e A	126	#define C_SEG_MAX_LIMIT (1 << 20) /* this needs to track the size of c_mysegno */
5ba3f43e A	127	uint32_t c_mysegno:20,
39236c6e A	128	c_busy:1,
39236c6e A	129	c_busy_swapping:1,
5ba3f43e	130	c_wanted:1,
3e170ce0 A	131	c_on_minorcompact_q:1, /* can also be on the age_q, the majorcompact_q or the swappedin_q */
	132
	133	c_state:4, /* what state is the segment in which dictates which q to find it on */
	134	c_overage_swap:1,
5ba3f43e A	135	c_reserved:3;
	136
	137	uint32_t c_creation_ts;
	138	uint64_t c_generation_id;
	139
	140	int32_t c_bytes_used;
	141	int32_t c_bytes_unused;
	142	uint32_t c_slots_used;
3e170ce0	143
39236c6e A	144	uint16_t c_firstemptyslot;
	145	uint16_t c_nextslot;
	146	uint32_t c_nextoffset;
	147	uint32_t c_populated_offset;
	148
39236c6e A	149	uint32_t c_swappedin_ts;
	150
	151	union {
	152	int32_t *c_buffer;
	153	uint64_t c_swap_handle;
	154	} c_store;
	155
39236c6e A	156	#if VALIDATE_C_SEGMENTS
	157	uint32_t c_was_minor_compacted;
	158	uint32_t c_was_major_compacted;
	159	uint32_t c_was_major_donor;
	160	#endif
	161	#if CHECKSUM_THE_SWAP
	162	unsigned int cseg_hash;
	163	unsigned int cseg_swap_size;
	164	#endif /* CHECKSUM_THE_SWAP */
	165
fe8ab488 A	166	#if MACH_ASSERT
	167	thread_t c_busy_for_thread;
	168	#endif /* MACH_ASSERT */
	169
3e170ce0 A	170	int c_slot_var_array_len;
	171	struct c_slot *c_slot_var_array;
	172	struct c_slot c_slot_fixed_array[0];
39236c6e A	173	};
39236c6e A	174
5ba3f43e A	175
	176	struct c_slot_mapping {
	177	uint32_t s_cseg:22, /* segment number + 1 */
	178	s_cindx:10; /* index in the segment */
	179	};
	180	#define C_SLOT_MAX_INDEX (1 << 10)
	181
	182	typedef struct c_slot_mapping *c_slot_mapping_t;
	183
	184
3e170ce0 A	185	#define C_SEG_SLOT_VAR_ARRAY_MIN_LEN C_SEG_MAX_PAGES
	186
	187	extern int c_seg_fixed_array_len;
	188	extern vm_offset_t c_buffers;
	189	#define C_SEG_BUFFER_ADDRESS(c_segno) ((c_buffers + ((uint64_t)c_segno * (uint64_t)C_SEG_ALLOCSIZE)))
39236c6e	190
3e170ce0	191	#define C_SEG_SLOT_FROM_INDEX(cseg, index) (index < c_seg_fixed_array_len ? &(cseg->c_slot_fixed_array[index]) : &(cseg->c_slot_var_array[index - c_seg_fixed_array_len]))
39236c6e A	192
	193	#define C_SEG_OFFSET_TO_BYTES(off) ((off) * (int) sizeof(int32_t))
	194	#define C_SEG_BYTES_TO_OFFSET(bytes) ((bytes) / (int) sizeof(int32_t))
	195
	196	#define C_SEG_UNUSED_BYTES(cseg) (cseg->c_bytes_unused + (C_SEG_OFFSET_TO_BYTES(cseg->c_populated_offset - cseg->c_nextoffset)))
5ba3f43e	197	//todo opensource
39236c6e	198
5ba3f43e A	199	#ifndef __PLATFORM_WKDM_ALIGNMENT_MASK__
	200	#define C_SEG_OFFSET_ALIGNMENT_MASK 0x3ULL
	201	#define C_SEG_OFFSET_ALIGNMENT_BOUNDARY 0x4
	202	#else
	203	#define C_SEG_OFFSET_ALIGNMENT_MASK __PLATFORM_WKDM_ALIGNMENT_MASK__
	204	#define C_SEG_OFFSET_ALIGNMENT_BOUNDARY __PLATFORM_WKDM_ALIGNMENT_BOUNDARY__
	205	#endif
	206
	207	#define C_SEG_SHOULD_MINORCOMPACT_NOW(cseg) ((C_SEG_UNUSED_BYTES(cseg) >= (C_SEG_BUFSIZE / 4)) ? 1 : 0)
39236c6e	208
5ba3f43e A	209	/*
	210	* the decsion to force a c_seg to be major compacted is based on 2 criteria
	211	* 1) is the c_seg buffer almost empty (i.e. we have a chance to merge it with another c_seg)
	212	* 2) are there at least a minimum number of slots unoccupied so that we have a chance
	213	* of combining this c_seg with another one.
	214	*/
	215	#define C_SEG_SHOULD_MAJORCOMPACT_NOW(cseg) \
	216	((((cseg->c_bytes_unused + (C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES(c_seg->c_nextoffset))) >= (C_SEG_BUFSIZE / 8)) && \
	217	((C_SLOT_MAX_INDEX - cseg->c_slots_used) > (C_SEG_BUFSIZE / PAGE_SIZE))) \
	218	? 1 : 0)
3e170ce0	219
5ba3f43e	220	#define C_SEG_ONDISK_IS_SPARSE(cseg) ((cseg->c_bytes_used < cseg->c_bytes_unused) ? 1 : 0)
3e170ce0 A	221	#define C_SEG_IS_ONDISK(cseg) ((cseg->c_state == C_ON_SWAPPEDOUT_Q \|\| cseg->c_state == C_ON_SWAPPEDOUTSPARSE_Q))
3e170ce0 A	222
39236c6e A	223
	224	#define C_SEG_WAKEUP_DONE(cseg) \
	225	MACRO_BEGIN \
fe8ab488	226	assert((cseg)->c_busy); \
39236c6e	227	(cseg)->c_busy = 0; \
fe8ab488 A	228	assert((cseg)->c_busy_for_thread != NULL); \
fe8ab488 A	229	assert((((cseg)->c_busy_for_thread = NULL), TRUE)); \
39236c6e A	230	if ((cseg)->c_wanted) { \
	231	(cseg)->c_wanted = 0; \
	232	thread_wakeup((event_t) (cseg)); \
	233	} \
	234	MACRO_END
	235
fe8ab488 A	236	#define C_SEG_BUSY(cseg) \
	237	MACRO_BEGIN \
	238	assert((cseg)->c_busy == 0); \
	239	(cseg)->c_busy = 1; \
	240	assert((cseg)->c_busy_for_thread == NULL); \
	241	assert((((cseg)->c_busy_for_thread = current_thread()), TRUE)); \
	242	MACRO_END
	243
	244
39037602 A	245	extern vm_map_t compressor_map;
39037602 A	246
5ba3f43e A	247	#if DEVELOPMENT \|\| DEBUG
	248	extern boolean_t write_protect_c_segs;
	249	extern int vm_compressor_test_seg_wp;
	250
39037602 A	251	#define C_SEG_MAKE_WRITEABLE(cseg) \
39037602 A	252	MACRO_BEGIN \
5ba3f43e A	253	if (write_protect_c_segs) { \
	254	vm_map_protect(compressor_map, \
	255	(vm_map_offset_t)cseg->c_store.c_buffer, \
	256	(vm_map_offset_t)&cseg->c_store.c_buffer[C_SEG_BYTES_TO_OFFSET(C_SEG_ALLOCSIZE)],\
	257	VM_PROT_READ \| VM_PROT_WRITE, \
	258	0); \
	259	} \
39037602 A	260	MACRO_END
	261
	262	#define C_SEG_WRITE_PROTECT(cseg) \
	263	MACRO_BEGIN \
5ba3f43e A	264	if (write_protect_c_segs) { \
	265	vm_map_protect(compressor_map, \
	266	(vm_map_offset_t)cseg->c_store.c_buffer, \
	267	(vm_map_offset_t)&cseg->c_store.c_buffer[C_SEG_BYTES_TO_OFFSET(C_SEG_ALLOCSIZE)],\
	268	VM_PROT_READ, \
	269	0); \
	270	} \
	271	if (vm_compressor_test_seg_wp) { \
	272	volatile uint32_t vmtstmp = (volatile uint32_t )cseg->c_store.c_buffer; \
	273	(volatile uint32_t )cseg->c_store.c_buffer = 0xDEADABCD; \
	274	(void) vmtstmp; \
	275	} \
39037602 A	276	MACRO_END
39037602 A	277	#endif
39236c6e A	278
	279	typedef struct c_segment *c_segment_t;
	280	typedef struct c_slot *c_slot_t;
	281
	282	uint64_t vm_compressor_total_compressions(void);
	283	void vm_wake_compactor_swapper(void);
39037602	284	void vm_run_compactor(void);
fe8ab488	285	void vm_thrashing_jetsam_done(void);
39236c6e	286	void vm_consider_waking_compactor_swapper(void);
3e170ce0	287	void vm_consider_swapping(void);
39236c6e A	288	void vm_compressor_flush(void);
	289	void c_seg_free(c_segment_t);
	290	void c_seg_free_locked(c_segment_t);
	291	void c_seg_insert_into_age_q(c_segment_t);
39037602	292	void c_seg_need_delayed_compaction(c_segment_t, boolean_t);
39236c6e A	293
	294	void vm_decompressor_lock(void);
	295	void vm_decompressor_unlock(void);
	296
8a3053a0	297	void vm_compressor_delay_trim(void);
39236c6e A	298	void vm_compressor_do_warmup(void);
	299	void vm_compressor_record_warmup_start(void);
	300	void vm_compressor_record_warmup_end(void);
	301
fe8ab488	302	int vm_wants_task_throttled(task_t);
39236c6e	303
39037602	304	extern void vm_compaction_swapper_do_init(void);
fe8ab488	305	extern void vm_compressor_swap_init(void);
39236c6e A	306	extern void vm_compressor_init_locks(void);
	307	extern lck_rw_t c_master_lock;
	308
fe8ab488	309	#if ENCRYPTED_SWAP
39236c6e	310	extern void vm_swap_decrypt(c_segment_t);
fe8ab488	311	#endif /* ENCRYPTED_SWAP */
39236c6e	312
fe8ab488	313	extern int vm_swap_low_on_space(void);
39037602	314	extern kern_return_t vm_swap_get(c_segment_t, uint64_t, uint64_t);
39236c6e A	315	extern void vm_swap_free(uint64_t);
	316	extern void vm_swap_consider_defragmenting(void);
	317
39037602 A	318	extern void c_seg_swapin_requeue(c_segment_t, boolean_t, boolean_t, boolean_t);
39037602 A	319	extern int c_seg_swapin(c_segment_t, boolean_t, boolean_t);
39236c6e A	320	extern void c_seg_wait_on_busy(c_segment_t);
39236c6e A	321	extern void c_seg_trim_tail(c_segment_t);
3e170ce0	322	extern void c_seg_switch_state(c_segment_t, int, boolean_t);
39236c6e A	323
39236c6e A	324	extern boolean_t fastwake_recording_in_progress;
39037602	325	extern int compaction_swapper_inited;
39236c6e A	326	extern int compaction_swapper_running;
	327	extern uint64_t vm_swap_put_failures;
	328
3e170ce0 A	329	extern int c_overage_swapped_count;
	330	extern int c_overage_swapped_limit;
	331
39236c6e A	332	extern queue_head_t c_minor_list_head;
	333	extern queue_head_t c_age_list_head;
	334	extern queue_head_t c_swapout_list_head;
	335	extern queue_head_t c_swappedout_list_head;
	336	extern queue_head_t c_swappedout_sparse_list_head;
	337
	338	extern uint32_t c_age_count;
	339	extern uint32_t c_swapout_count;
	340	extern uint32_t c_swappedout_count;
	341	extern uint32_t c_swappedout_sparse_count;
	342
	343	extern int64_t compressor_bytes_used;
	344	extern uint64_t first_c_segment_to_warm_generation_id;
	345	extern uint64_t last_c_segment_to_warm_generation_id;
	346	extern boolean_t hibernate_flushing;
	347	extern boolean_t hibernate_no_swapspace;
39037602	348	extern boolean_t hibernate_in_progress_with_pinned_swap;
39236c6e A	349	extern uint32_t swapout_target_age;
	350
	351	extern void c_seg_insert_into_q(queue_head_t *, c_segment_t);
	352
	353	extern uint32_t vm_compressor_minorcompact_threshold_divisor;
	354	extern uint32_t vm_compressor_majorcompact_threshold_divisor;
	355	extern uint32_t vm_compressor_unthrottle_threshold_divisor;
	356	extern uint32_t vm_compressor_catchup_threshold_divisor;
fe8ab488	357	extern uint64_t vm_compressor_compute_elapsed_msecs(clock_sec_t, clock_nsec_t, clock_sec_t, clock_nsec_t);
39236c6e A	358
	359	#define PAGE_REPLACEMENT_DISALLOWED(enable) (enable == TRUE ? lck_rw_lock_shared(&c_master_lock) : lck_rw_done(&c_master_lock))
	360	#define PAGE_REPLACEMENT_ALLOWED(enable) (enable == TRUE ? lck_rw_lock_exclusive(&c_master_lock) : lck_rw_done(&c_master_lock))
	361
	362
	363	#define AVAILABLE_NON_COMPRESSED_MEMORY (vm_page_active_count + vm_page_inactive_count + vm_page_free_count + vm_page_speculative_count)
	364	#define AVAILABLE_MEMORY (AVAILABLE_NON_COMPRESSED_MEMORY + VM_PAGE_COMPRESSOR_COUNT)
39037602 A	365	/* TODO, there may be a minor optimisation opportunity to replace these divisions
	366	* with multiplies and shifts
	367	*/
39236c6e A	368
	369	#define VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD (((AVAILABLE_MEMORY) * 10) / (vm_compressor_minorcompact_threshold_divisor ? vm_compressor_minorcompact_threshold_divisor : 1))
	370	#define VM_PAGE_COMPRESSOR_SWAP_THRESHOLD (((AVAILABLE_MEMORY) * 10) / (vm_compressor_majorcompact_threshold_divisor ? vm_compressor_majorcompact_threshold_divisor : 1))
	371	#define VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD (((AVAILABLE_MEMORY) * 10) / (vm_compressor_unthrottle_threshold_divisor ? vm_compressor_unthrottle_threshold_divisor : 1))
	372	#define VM_PAGE_COMPRESSOR_SWAP_CATCHUP_THRESHOLD (((AVAILABLE_MEMORY) * 10) / (vm_compressor_catchup_threshold_divisor ? vm_compressor_catchup_threshold_divisor : 1))
	373
5ba3f43e A	374	#ifdef CONFIG_EMBEDDED
	375	#define AVAILABLE_NON_COMPRESSED_MIN 20000
	376	#define COMPRESSOR_NEEDS_TO_SWAP() (((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_THRESHOLD) \|\| \
	377	(AVAILABLE_NON_COMPRESSED_MEMORY < AVAILABLE_NON_COMPRESSED_MIN)) ? 1 : 0)
	378	#else
39236c6e	379	#define COMPRESSOR_NEEDS_TO_SWAP() ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_THRESHOLD) ? 1 : 0)
5ba3f43e	380	#endif
39236c6e A	381
39236c6e A	382	#define VM_PAGEOUT_SCAN_NEEDS_TO_THROTTLE() \
3e170ce0	383	(vm_compressor_mode == VM_PAGER_COMPRESSOR_WITH_SWAP && \
39236c6e A	384	((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_CATCHUP_THRESHOLD) ? 1 : 0))
	385	#define HARD_THROTTLE_LIMIT_REACHED() ((AVAILABLE_NON_COMPRESSED_MEMORY < (VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) / 2) ? 1 : 0)
	386	#define SWAPPER_NEEDS_TO_UNTHROTTLE() ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) ? 1 : 0)
	387	#define COMPRESSOR_NEEDS_TO_MINOR_COMPACT() ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD) ? 1 : 0)
39236c6e	388
39236c6e	389
5ba3f43e A	390	#ifdef CONFIG_EMBEDDED
	391	#define COMPRESSOR_FREE_RESERVED_LIMIT 28
	392	#else
3e170ce0	393	#define COMPRESSOR_FREE_RESERVED_LIMIT 128
5ba3f43e	394	#endif
39236c6e	395
39037602 A	396	uint32_t vm_compressor_get_encode_scratch_size(void);
39037602 A	397	uint32_t vm_compressor_get_decode_scratch_size(void);
39236c6e	398
39037602	399	#define COMPRESSOR_SCRATCH_BUF_SIZE vm_compressor_get_encode_scratch_size()
39236c6e	400
3e170ce0 A	401	#if RECORD_THE_COMPRESSED_DATA
	402	extern void c_compressed_record_init(void);
	403	extern void c_compressed_record_write(char *, int);
	404	#endif
	405
39236c6e	406	extern lck_mtx_t *c_list_lock;
5ba3f43e A	407
	408	#if DEVELOPMENT \|\| DEBUG
	409	extern uint32_t vm_ktrace_enabled;
	410
	411	#define VMKDBG(x, ...) \
	412	MACRO_BEGIN \
	413	if (vm_ktrace_enabled) { \
	414	KDBG(x, ## __VA_ARGS__);\
	415	} \
	416	MACRO_END
	417	#endif