X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/3903760236c30e3b5ace7a4eefac3a269d68957c..5c9f46613a83ebfc29a5b1f099448259e96a98f0:/osfmk/vm/vm_compressor_algorithms.c diff --git a/osfmk/vm/vm_compressor_algorithms.c b/osfmk/vm/vm_compressor_algorithms.c index e7135a7fc..3af35c8ec 100644 --- a/osfmk/vm/vm_compressor_algorithms.c +++ b/osfmk/vm/vm_compressor_algorithms.c @@ -35,6 +35,10 @@ #include #define MZV_MAGIC (17185) +#if defined(__arm64__) +#include +#endif + #define LZ4_SCRATCH_ALIGN (64) #define WKC_SCRATCH_ALIGN (64) @@ -100,14 +104,24 @@ enum compressor_preselect_t { vm_compressor_mode_t vm_compressor_current_codec = VM_COMPRESSOR_DEFAULT_CODEC; +boolean_t vm_compressor_force_sw_wkdm = FALSE; + boolean_t verbose = FALSE; -#if DEVELOPMENT || DEBUG -#define VERBOSE(x...) \ +#define VMDBGSTAT (DEBUG) +#if VMDBGSTATS +#define VM_COMPRESSOR_STAT_DBG(x...) \ do { \ - if (verbose) \ - printf(x); \ + (x); \ } while(0) +#else +#define VM_COMPRESSOR_STAT_DBG(x...) \ + do { \ + } while (0) +#endif + +#define VMCSTATS (DEVELOPMENT || DEBUG) +#if VMCSTATS #define VM_COMPRESSOR_STAT(x...) \ do { \ (x); \ @@ -118,9 +132,6 @@ boolean_t verbose = FALSE; (x); \ } while(0) #else -#define VERBOSE(x...) \ - do { \ - }while (0) #define VM_COMPRESSOR_STAT(x...) \ do { \ }while (0) @@ -199,28 +210,76 @@ static inline void compressor_selector_update(int lz4sz, int didwk, int wksz) { } } + +static inline void WKdm_hv(uint32_t *wkbuf) { +#if DEVELOPMENT || DEBUG + uint32_t *inw = (uint32_t *) wkbuf; + if (*inw != MZV_MAGIC) { + if ((*inw | *(inw + 1) | *(inw + 2)) & 0xFFFF0000) { + panic("WKdm(%p): invalid header 0x%x 0x%x 0x%x\n", wkbuf, *inw, *(inw +1), *(inw+2)); + } + } +#else /* DEVELOPMENT || DEBUG */ + (void) wkbuf; +#endif +} + //todo fix clang diagnostic #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wincompatible-pointer-types" +#if defined(__arm64__) +#endif + static inline void WKdmD(WK_word* src_buf, WK_word* dest_buf, WK_word* scratch, unsigned int bytes) { -#if DEVELOPMENT || DEBUG - uint32_t *inw = (uint32_t *) src_buf; - if (*inw != MZV_MAGIC) { - if ((*inw | *(inw+1) | *(inw+2)) & 0xFFFF0000) { - panic("WKdmDecompress: invalid header 0x%x 0x%x 0x%x\n", *inw, *(inw +1), *(inw+2)); - } +#if defined(__arm64__) +#endif + WKdm_hv(src_buf); +#if defined(__arm64__) + if (PAGE_SIZE == 4096) { + WKdm_decompress_4k(src_buf, dest_buf, scratch, bytes); + } else { + __unused uint64_t wdsstart; + + VM_COMPRESSOR_STAT_DBG(wdsstart = mach_absolute_time()); + WKdm_decompress_16k(src_buf, dest_buf, scratch, bytes); + + VM_COMPRESSOR_STAT_DBG(compressor_stats.wks_dabstime += mach_absolute_time() - wdsstart); + VM_COMPRESSOR_STAT(compressor_stats.wks_decompressions++); } -#endif /* DEVELOPMENT || DEBUG */ +#else /* !defined arm64 */ WKdm_decompress_new(src_buf, dest_buf, scratch, bytes); +#endif } +#if DEVELOPMENT || DEBUG +int precompy, wkswhw; +#endif -static inline int WKdmC(WK_word* src_buf, WK_word* dest_buf, WK_word* scratch, unsigned int limit) { - return WKdm_compress_new(src_buf, dest_buf, scratch, limit); +static inline int WKdmC(WK_word* src_buf, WK_word* dest_buf, WK_word* scratch, boolean_t *incomp_copy, unsigned int limit) { + (void)incomp_copy; + int wkcval; +#if defined(__arm64__) + if (PAGE_SIZE == 4096) { + wkcval = WKdm_compress_4k(src_buf, dest_buf, scratch, limit); + } else { + __unused uint64_t wcswstart; + + VM_COMPRESSOR_STAT_DBG(wcswstart = mach_absolute_time()); + + int wkswsz = WKdm_compress_16k(src_buf, dest_buf, scratch, limit); + + VM_COMPRESSOR_STAT_DBG(compressor_stats.wks_cabstime += mach_absolute_time() - wcswstart); + VM_COMPRESSOR_STAT(compressor_stats.wks_compressions++); + wkcval = wkswsz; + } +#else + wkcval = WKdm_compress_new(src_buf, dest_buf, scratch, limit); +#endif + return wkcval; } -int metacompressor(const uint8_t *in, uint8_t *cdst, int32_t outbufsz, uint16_t *codec, void *cscratchin) { +int metacompressor(const uint8_t *in, uint8_t *cdst, int32_t outbufsz, uint16_t *codec, void *cscratchin, boolean_t *incomp_copy) { int sz = -1; int dowk = FALSE, dolz4 = FALSE, skiplz4 = FALSE; int insize = PAGE_SIZE; @@ -246,10 +305,9 @@ int metacompressor(const uint8_t *in, uint8_t *cdst, int32_t outbufsz, uint16_t if (dowk) { *codec = CCWK; - sz = WKdmC(in, cdst, &cscratch->wkscratch[0], outbufsz); VM_COMPRESSOR_STAT(compressor_stats.wk_compressions++); + sz = WKdmC(in, cdst, &cscratch->wkscratch[0], incomp_copy, outbufsz); - VERBOSE("WKDm Compress: %d\n", sz); if (sz == -1) { VM_COMPRESSOR_STAT(compressor_stats.wk_compressed_bytes_total+=PAGE_SIZE); VM_COMPRESSOR_STAT(compressor_stats.wk_compression_failures++); @@ -260,7 +318,7 @@ int metacompressor(const uint8_t *in, uint8_t *cdst, int32_t outbufsz, uint16_t goto cexit; } else if (sz == 0) { VM_COMPRESSOR_STAT(compressor_stats.wk_sv_compressions++); - VM_COMPRESSOR_STAT(compressor_stats.wk_compressed_bytes_total+=8); + VM_COMPRESSOR_STAT(compressor_stats.wk_compressed_bytes_total+=4); } else { VM_COMPRESSOR_STAT(compressor_stats.wk_compressed_bytes_total+=sz); } @@ -270,7 +328,9 @@ lz4eval: if (((sz == -1) || (sz >= vmctune.lz4_threshold)) && (skiplz4 == FALSE)) { dolz4 = TRUE; } else { - __unused int wkc = (sz == -1) ? PAGE_SIZE : sz; +#if DEVELOPMENT || DEBUG + int wkc = (sz == -1) ? PAGE_SIZE : sz; +#endif VM_COMPRESSOR_STAT(compressor_stats.wk_compressions_exclusive++); VM_COMPRESSOR_STAT(compressor_stats.wk_compressed_bytes_exclusive+=wkc); goto cexit; @@ -288,7 +348,6 @@ lz4compress: sz = (int) lz4raw_encode_buffer(cdst, outbufsz, in, insize, &cscratch->lz4state[0]); - VERBOSE("LZ4 Compress: %d\n", sz); compressor_selector_update(sz, dowk, wksz); if (sz == 0) { sz = -1; @@ -308,12 +367,16 @@ void metadecompressor(const uint8_t *source, uint8_t *dest, uint32_t csize, uint rval = (int)lz4raw_decode_buffer(dest, PAGE_SIZE, source, csize, &compressor_dscratch->lz4decodestate[0]); VM_DECOMPRESSOR_STAT(compressor_stats.lz4_decompressions+=1); VM_DECOMPRESSOR_STAT(compressor_stats.lz4_decompressed_bytes+=csize); - - assertf(rval == PAGE_SIZE, "LZ4 decode: size != pgsize %d", rval); - +#if DEVELOPMENT || DEBUG + uint32_t *d32 = dest; +#endif + assertf(rval == PAGE_SIZE, "LZ4 decode: size != pgsize %d, header: 0x%x, 0x%x, 0x%x", + rval, *d32, *(d32+1), *(d32+2)); } else { assert(ccodec == CCWK); + WKdmD(source, dest, &compressor_dscratch->wkdecompscratch[0], csize); + VM_DECOMPRESSOR_STAT(compressor_stats.wk_decompressions+=1); VM_DECOMPRESSOR_STAT(compressor_stats.wk_decompressed_bytes+=csize); } @@ -344,18 +407,27 @@ int vm_compressor_algorithm(void) { void vm_compressor_algorithm_init(void) { vm_compressor_mode_t new_codec = VM_COMPRESSOR_DEFAULT_CODEC; +#if defined(__arm64__) + new_codec = CMODE_HYB; + + if (PAGE_SIZE == 16384) { + vmctune.lz4_threshold = 12288; + } +#endif PE_parse_boot_argn("vm_compressor_codec", &new_codec, sizeof(new_codec)); assertf(((new_codec == VM_COMPRESSOR_DEFAULT_CODEC) || (new_codec == CMODE_WK) || (new_codec == CMODE_LZ4) || (new_codec = CMODE_HYB)), "Invalid VM compression codec: %u", new_codec); - - if (PE_parse_boot_argn("-vm_compressor_wk", &new_codec, sizeof(new_codec))) { +#if defined(__arm__)||defined(__arm64__) + uint32_t tmpc; + if (PE_parse_boot_argn("-vm_compressor_wk", &tmpc, sizeof(tmpc))) { new_codec = VM_COMPRESSOR_DEFAULT_CODEC; - } else if (PE_parse_boot_argn("-vm_compressor_hybrid", &new_codec, sizeof(new_codec))) { + } else if (PE_parse_boot_argn("-vm_compressor_hybrid", &tmpc, sizeof(tmpc))) { new_codec = CMODE_HYB; } + vm_compressor_current_codec = new_codec; +#endif /* arm/arm64 */ } -//TODO check open-sourceability of lz4