osfmk/vm/lz4.h

   1 /*
   2  * Copyright (c) 2016-2016 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #pragma once
  30 #include <stdint.h>
  31 #include <stddef.h>
  32 #include <kern/assert.h>
  33 #include <machine/limits.h>
  34 #include "lz4_assembly_select.h"
  35 #include "lz4_constants.h"
  36
  37 #if CONFIG_IO_COMPRESSION_STATS
  38 #include <string.h>
  39 #else
  40 #define memcpy __builtin_memcpy
  41 #endif
  42
  43 #pragma mark - Building blocks
  44
  45 //  Represents a position in the input stream
  46 typedef struct { uint32_t offset; uint32_t word; } lz4_hash_entry_t;
  47 static const size_t lz4_hash_table_size = LZ4_COMPRESS_HASH_ENTRIES * sizeof(lz4_hash_entry_t);
  48
  49 // Worker function for lz4 encode.  Underlies both the buffer and stream encode operations.
  50 // Performs lz4 encoding of up to 2gb of data, updates dst_ptr and src_ptr to point to the
  51 // first byte of output and input that couldn't be completely processed, respectively.
  52 //
  53 // If skip_final_literals is 0, the entire src buffer is encoded, by emitting a final sequence of literals
  54 // at the end of the compressed payload.
  55 //
  56 // If skip_final_literals is not 0, this final literal sequence is not emitted, and the src buffer is
  57 // partially encoded (the length of this literal sequence varies).
  58 extern void lz4_encode_2gb(uint8_t **dst_ptr, size_t dst_size,
  59     const uint8_t **src_ptr, const uint8_t *src_begin, size_t src_size,
  60     lz4_hash_entry_t hash_table[LZ4_COMPRESS_HASH_ENTRIES], int skip_final_literals);
  61
  62 extern int lz4_decode(uint8_t **dst_ptr, uint8_t *dst_begin, uint8_t *dst_end,
  63     const uint8_t **src_ptr, const uint8_t *src_end);
  64
  65 #if LZ4_ENABLE_ASSEMBLY_DECODE
  66 extern int lz4_decode_asm(uint8_t **dst_ptr, uint8_t *dst_begin, uint8_t *dst_end,
  67     const uint8_t **src_ptr, const uint8_t *src_end);
  68 #endif
  69
  70 #pragma mark - Buffer interfaces
  71
  72 static const size_t lz4_encode_scratch_size = lz4_hash_table_size;
  73 static const size_t lz4_decode_scratch_size = 0;
  74
  75 #pragma mark - Buffer interfaces (LZ4 RAW)
  76
  77 size_t lz4raw_encode_buffer(uint8_t * __restrict dst_buffer, size_t dst_size,
  78     const uint8_t * __restrict src_buffer, size_t src_size,
  79     lz4_hash_entry_t hash_table[LZ4_COMPRESS_HASH_ENTRIES]);
  80
  81 size_t lz4raw_decode_buffer(uint8_t * __restrict dst_buffer, size_t dst_size,
  82     const uint8_t * __restrict src_buffer, size_t src_size,
  83     void * __restrict work __attribute__((unused)));
  84
  85 typedef __attribute__((__ext_vector_type__(8))) uint8_t vector_uchar8;
  86 typedef __attribute__((__ext_vector_type__(16))) uint8_t vector_uchar16;
  87 typedef __attribute__((__ext_vector_type__(32))) uint8_t vector_uchar32;
  88 typedef __attribute__((__ext_vector_type__(64))) uint8_t vector_uchar64;
  89 typedef __attribute__((__ext_vector_type__(16), __aligned__(1))) uint8_t packed_uchar16;
  90 typedef __attribute__((__ext_vector_type__(32), __aligned__(1))) uint8_t packed_uchar32;
  91 typedef __attribute__((__ext_vector_type__(64), __aligned__(1))) uint8_t packed_uchar64;
  92
  93 typedef __attribute__((__ext_vector_type__(4))) uint16_t vector_ushort4;
  94 typedef __attribute__((__ext_vector_type__(4), __aligned__(2))) uint16_t packed_ushort4;
  95
  96 typedef __attribute__((__ext_vector_type__(2))) int32_t vector_int2;
  97 typedef __attribute__((__ext_vector_type__(4))) int32_t vector_int4;
  98 typedef __attribute__((__ext_vector_type__(8))) int32_t vector_int8;
  99
 100 typedef __attribute__((__ext_vector_type__(4))) uint32_t vector_uint4;
 101
 102 #define UTIL_FUNCTION static inline __attribute__((__always_inline__)) __attribute__((__overloadable__))
 103
 104 // Load N bytes from unaligned location PTR
 105 UTIL_FUNCTION uint16_t
 106 load2(const void * ptr)
 107 {
 108         uint16_t data; memcpy(&data, ptr, sizeof data); return data;
 109 }
 110 UTIL_FUNCTION uint32_t
 111 load4(const void * ptr)
 112 {
 113         uint32_t data; memcpy(&data, ptr, sizeof data); return data;
 114 }
 115 UTIL_FUNCTION uint64_t
 116 load8(const void * ptr)
 117 {
 118         uint64_t data; memcpy(&data, ptr, sizeof data); return data;
 119 }
 120 UTIL_FUNCTION vector_uchar16
 121 load16(const void * ptr)
 122 {
 123         return (const vector_uchar16)*(const packed_uchar16 *)ptr;
 124 }
 125 UTIL_FUNCTION vector_uchar32
 126 load32(const void * ptr)
 127 {
 128         return (const vector_uchar32)*(const packed_uchar32 *)ptr;
 129 }
 130 UTIL_FUNCTION vector_uchar64
 131 load64(const void * ptr)
 132 {
 133         return (const vector_uchar64)*(const packed_uchar64 *)ptr;
 134 }
 135
 136 // Store N bytes to unaligned location PTR
 137 UTIL_FUNCTION void
 138 store2(void * ptr, uint16_t data)
 139 {
 140         memcpy(ptr, &data, sizeof data);
 141 }
 142 UTIL_FUNCTION void
 143 store4(void * ptr, uint32_t data)
 144 {
 145         memcpy(ptr, &data, sizeof data);
 146 }
 147 UTIL_FUNCTION void
 148 store8(void * ptr, uint64_t data)
 149 {
 150         memcpy(ptr, &data, sizeof data);
 151 }
 152 UTIL_FUNCTION void
 153 store16(void * ptr, vector_uchar16 data)
 154 {
 155         *(packed_uchar16 *)ptr = (packed_uchar16)data;
 156 }
 157 UTIL_FUNCTION void
 158 store32(void * ptr, vector_uchar32 data)
 159 {
 160         *(packed_uchar32 *)ptr = (packed_uchar32)data;
 161 }
 162 UTIL_FUNCTION void
 163 store64(void * ptr, vector_uchar64 data)
 164 {
 165         *(packed_uchar64 *)ptr = (packed_uchar64)data;
 166 }
 167
 168 // Load+Store N bytes from unaligned locations SRC to DST. No overlap allowed.
 169 UTIL_FUNCTION void
 170 copy8(void * dst, const void * src)
 171 {
 172         store8(dst, load8(src));
 173 }
 174 UTIL_FUNCTION void
 175 copy16(void * dst, const void * src)
 176 {
 177         *(packed_uchar16 *)dst = *(const packed_uchar16 *)src;
 178 }
 179 UTIL_FUNCTION void
 180 copy32(void * dst, const void * src)
 181 {
 182         *(packed_uchar32 *)dst = *(const packed_uchar32 *)src;
 183 }
 184
 185 #undef memcpy