#include <vm/vm_compressor.h>
#define MZV_MAGIC (17185)
+#if defined(__arm64__)
+#include <arm/proc_reg.h>
+#endif
+
#define LZ4_SCRATCH_ALIGN (64)
#define WKC_SCRATCH_ALIGN (64)
vm_compressor_mode_t vm_compressor_current_codec = VM_COMPRESSOR_DEFAULT_CODEC;
+boolean_t vm_compressor_force_sw_wkdm = FALSE;
+
boolean_t verbose = FALSE;
-#if DEVELOPMENT || DEBUG
-#define VERBOSE(x...) \
+#define VMDBGSTAT (DEBUG)
+#if VMDBGSTATS
+#define VM_COMPRESSOR_STAT_DBG(x...) \
do { \
- if (verbose) \
- printf(x); \
+ (x); \
} while(0)
+#else
+#define VM_COMPRESSOR_STAT_DBG(x...) \
+ do { \
+ } while (0)
+#endif
+
+#define VMCSTATS (DEVELOPMENT || DEBUG)
+#if VMCSTATS
#define VM_COMPRESSOR_STAT(x...) \
do { \
(x); \
(x); \
} while(0)
#else
-#define VERBOSE(x...) \
- do { \
- }while (0)
#define VM_COMPRESSOR_STAT(x...) \
do { \
}while (0)
}
}
+
+static inline void WKdm_hv(uint32_t *wkbuf) {
+#if DEVELOPMENT || DEBUG
+ uint32_t *inw = (uint32_t *) wkbuf;
+ if (*inw != MZV_MAGIC) {
+ if ((*inw | *(inw + 1) | *(inw + 2)) & 0xFFFF0000) {
+ panic("WKdm(%p): invalid header 0x%x 0x%x 0x%x\n", wkbuf, *inw, *(inw +1), *(inw+2));
+ }
+ }
+#else /* DEVELOPMENT || DEBUG */
+ (void) wkbuf;
+#endif
+}
+
//todo fix clang diagnostic
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wincompatible-pointer-types"
+#if defined(__arm64__)
+#endif
+
static inline void WKdmD(WK_word* src_buf, WK_word* dest_buf, WK_word* scratch, unsigned int bytes) {
-#if DEVELOPMENT || DEBUG
- uint32_t *inw = (uint32_t *) src_buf;
- if (*inw != MZV_MAGIC) {
- if ((*inw | *(inw+1) | *(inw+2)) & 0xFFFF0000) {
- panic("WKdmDecompress: invalid header 0x%x 0x%x 0x%x\n", *inw, *(inw +1), *(inw+2));
- }
+#if defined(__arm64__)
+#endif
+ WKdm_hv(src_buf);
+#if defined(__arm64__)
+ if (PAGE_SIZE == 4096) {
+ WKdm_decompress_4k(src_buf, dest_buf, scratch, bytes);
+ } else {
+ __unused uint64_t wdsstart;
+
+ VM_COMPRESSOR_STAT_DBG(wdsstart = mach_absolute_time());
+ WKdm_decompress_16k(src_buf, dest_buf, scratch, bytes);
+
+ VM_COMPRESSOR_STAT_DBG(compressor_stats.wks_dabstime += mach_absolute_time() - wdsstart);
+ VM_COMPRESSOR_STAT(compressor_stats.wks_decompressions++);
}
-#endif /* DEVELOPMENT || DEBUG */
+#else /* !defined arm64 */
WKdm_decompress_new(src_buf, dest_buf, scratch, bytes);
+#endif
}
+#if DEVELOPMENT || DEBUG
+int precompy, wkswhw;
+#endif
-static inline int WKdmC(WK_word* src_buf, WK_word* dest_buf, WK_word* scratch, unsigned int limit) {
- return WKdm_compress_new(src_buf, dest_buf, scratch, limit);
+static inline int WKdmC(WK_word* src_buf, WK_word* dest_buf, WK_word* scratch, boolean_t *incomp_copy, unsigned int limit) {
+ (void)incomp_copy;
+ int wkcval;
+#if defined(__arm64__)
+ if (PAGE_SIZE == 4096) {
+ wkcval = WKdm_compress_4k(src_buf, dest_buf, scratch, limit);
+ } else {
+ __unused uint64_t wcswstart;
+
+ VM_COMPRESSOR_STAT_DBG(wcswstart = mach_absolute_time());
+
+ int wkswsz = WKdm_compress_16k(src_buf, dest_buf, scratch, limit);
+
+ VM_COMPRESSOR_STAT_DBG(compressor_stats.wks_cabstime += mach_absolute_time() - wcswstart);
+ VM_COMPRESSOR_STAT(compressor_stats.wks_compressions++);
+ wkcval = wkswsz;
+ }
+#else
+ wkcval = WKdm_compress_new(src_buf, dest_buf, scratch, limit);
+#endif
+ return wkcval;
}
-int metacompressor(const uint8_t *in, uint8_t *cdst, int32_t outbufsz, uint16_t *codec, void *cscratchin) {
+int metacompressor(const uint8_t *in, uint8_t *cdst, int32_t outbufsz, uint16_t *codec, void *cscratchin, boolean_t *incomp_copy) {
int sz = -1;
int dowk = FALSE, dolz4 = FALSE, skiplz4 = FALSE;
int insize = PAGE_SIZE;
if (dowk) {
*codec = CCWK;
- sz = WKdmC(in, cdst, &cscratch->wkscratch[0], outbufsz);
VM_COMPRESSOR_STAT(compressor_stats.wk_compressions++);
+ sz = WKdmC(in, cdst, &cscratch->wkscratch[0], incomp_copy, outbufsz);
- VERBOSE("WKDm Compress: %d\n", sz);
if (sz == -1) {
VM_COMPRESSOR_STAT(compressor_stats.wk_compressed_bytes_total+=PAGE_SIZE);
VM_COMPRESSOR_STAT(compressor_stats.wk_compression_failures++);
goto cexit;
} else if (sz == 0) {
VM_COMPRESSOR_STAT(compressor_stats.wk_sv_compressions++);
- VM_COMPRESSOR_STAT(compressor_stats.wk_compressed_bytes_total+=8);
+ VM_COMPRESSOR_STAT(compressor_stats.wk_compressed_bytes_total+=4);
} else {
VM_COMPRESSOR_STAT(compressor_stats.wk_compressed_bytes_total+=sz);
}
if (((sz == -1) || (sz >= vmctune.lz4_threshold)) && (skiplz4 == FALSE)) {
dolz4 = TRUE;
} else {
- __unused int wkc = (sz == -1) ? PAGE_SIZE : sz;
+#if DEVELOPMENT || DEBUG
+ int wkc = (sz == -1) ? PAGE_SIZE : sz;
+#endif
VM_COMPRESSOR_STAT(compressor_stats.wk_compressions_exclusive++);
VM_COMPRESSOR_STAT(compressor_stats.wk_compressed_bytes_exclusive+=wkc);
goto cexit;
sz = (int) lz4raw_encode_buffer(cdst, outbufsz, in, insize, &cscratch->lz4state[0]);
- VERBOSE("LZ4 Compress: %d\n", sz);
compressor_selector_update(sz, dowk, wksz);
if (sz == 0) {
sz = -1;
rval = (int)lz4raw_decode_buffer(dest, PAGE_SIZE, source, csize, &compressor_dscratch->lz4decodestate[0]);
VM_DECOMPRESSOR_STAT(compressor_stats.lz4_decompressions+=1);
VM_DECOMPRESSOR_STAT(compressor_stats.lz4_decompressed_bytes+=csize);
-
- assertf(rval == PAGE_SIZE, "LZ4 decode: size != pgsize %d", rval);
-
+#if DEVELOPMENT || DEBUG
+ uint32_t *d32 = dest;
+#endif
+ assertf(rval == PAGE_SIZE, "LZ4 decode: size != pgsize %d, header: 0x%x, 0x%x, 0x%x",
+ rval, *d32, *(d32+1), *(d32+2));
} else {
assert(ccodec == CCWK);
+
WKdmD(source, dest, &compressor_dscratch->wkdecompscratch[0], csize);
+
VM_DECOMPRESSOR_STAT(compressor_stats.wk_decompressions+=1);
VM_DECOMPRESSOR_STAT(compressor_stats.wk_decompressed_bytes+=csize);
}
void vm_compressor_algorithm_init(void) {
vm_compressor_mode_t new_codec = VM_COMPRESSOR_DEFAULT_CODEC;
+#if defined(__arm64__)
+ new_codec = CMODE_HYB;
+
+ if (PAGE_SIZE == 16384) {
+ vmctune.lz4_threshold = 12288;
+ }
+#endif
PE_parse_boot_argn("vm_compressor_codec", &new_codec, sizeof(new_codec));
assertf(((new_codec == VM_COMPRESSOR_DEFAULT_CODEC) || (new_codec == CMODE_WK) ||
(new_codec == CMODE_LZ4) || (new_codec = CMODE_HYB)),
"Invalid VM compression codec: %u", new_codec);
-
- if (PE_parse_boot_argn("-vm_compressor_wk", &new_codec, sizeof(new_codec))) {
+#if defined(__arm__)||defined(__arm64__)
+ uint32_t tmpc;
+ if (PE_parse_boot_argn("-vm_compressor_wk", &tmpc, sizeof(tmpc))) {
new_codec = VM_COMPRESSOR_DEFAULT_CODEC;
- } else if (PE_parse_boot_argn("-vm_compressor_hybrid", &new_codec, sizeof(new_codec))) {
+ } else if (PE_parse_boot_argn("-vm_compressor_hybrid", &tmpc, sizeof(tmpc))) {
new_codec = CMODE_HYB;
}
+ vm_compressor_current_codec = new_codec;
+#endif /* arm/arm64 */
}
-//TODO check open-sourceability of lz4
projects / apple / xnu.git / blobdiff