From e3fbd54cee3fffecbf4f7c384e0aad715fc68218 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Mon, 28 Dec 2015 00:07:03 +0100 Subject: [PATCH] FileFd: (native) LZ4 support Implement native support for LZ4 compression, using the official lz4 library. --- apt-pkg/aptconfiguration.cc | 6 ++ apt-pkg/contrib/fileutl.cc | 160 ++++++++++++++++++++++++++++++++++++ apt-pkg/contrib/fileutl.h | 3 +- apt-pkg/makefile | 3 + buildlib/config.h.in | 3 + buildlib/environment.mak.in | 1 + configure.ac | 7 ++ debian/control | 2 +- 8 files changed, 183 insertions(+), 2 deletions(-) diff --git a/apt-pkg/aptconfiguration.cc b/apt-pkg/aptconfiguration.cc index a708a77c9..53f29df79 100644 --- a/apt-pkg/aptconfiguration.cc +++ b/apt-pkg/aptconfiguration.cc @@ -365,6 +365,12 @@ const Configuration::getCompressors(bool const Cached) { setDefaultConfigurationForCompressors(); compressors.push_back(Compressor(".", "", "", NULL, NULL, 0)); + if (_config->Exists("Dir::Bin::lz4") == false || FileExists(_config->FindFile("Dir::Bin::lz4")) == true) + compressors.push_back(Compressor("lz4",".lz4","lz4","-1","-d",50)); +#ifdef HAVE_LZ4 + else + compressors.push_back(Compressor("lz4",".lz4","false", NULL, NULL, 50)); +#endif if (_config->Exists("Dir::Bin::gzip") == false || FileExists(_config->FindFile("Dir::Bin::gzip")) == true) compressors.push_back(Compressor("gzip",".gz","gzip","-6n","-d",100)); #ifdef HAVE_ZLIB diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index e50cc694a..de215a50b 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -63,6 +63,9 @@ #ifdef HAVE_LZMA #include #endif +#ifdef HAVE_LZ4 + #include +#endif #include #include @@ -1494,6 +1497,158 @@ public: explicit Bz2FileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), bz2(nullptr) {} virtual ~Bz2FileFdPrivate() { InternalClose(""); } #endif +}; + /*}}}*/ +class APT_HIDDEN Lz4FileFdPrivate: public FileFdPrivate { /*{{{*/ + static constexpr unsigned long long BLK_SIZE = 64 * 1024; + static constexpr unsigned long long LZ4_HEADER_SIZE = 19; + static constexpr unsigned long long LZ4_FOOTER_SIZE = 4; +#ifdef HAVE_LZ4 + LZ4F_decompressionContext_t dctx; + LZ4F_compressionContext_t cctx; + LZ4F_errorCode_t res; + FileFd backend; + simple_buffer lz4_buffer; + // Count of bytes that the decompressor expects to read next, or buffer size. + size_t next_to_load = BLK_SIZE; +public: + virtual bool InternalOpen(int const iFd, unsigned int const Mode) override + { + if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite) + return _error->Error("lz4 only supports write or read mode"); + + if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly) { + res = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION); + lz4_buffer.reset(LZ4F_compressBound(BLK_SIZE, nullptr) + + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE); + } else { + res = LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION); + lz4_buffer.reset(64 * 1024); + } + + filefd->Flags |= FileFd::Compressed; + + if (LZ4F_isError(res)) + return false; + + unsigned int flags = (Mode & (FileFd::WriteOnly|FileFd::ReadOnly)); + if (backend.OpenDescriptor(iFd, flags) == false) + return false; + + // Write the file header + if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly) + { + res = LZ4F_compressBegin(cctx, lz4_buffer.buffer, lz4_buffer.buffersize_max, nullptr); + if (LZ4F_isError(res) || backend.Write(lz4_buffer.buffer, res) == false) + return false; + } + + return true; + } + virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) override + { + /* Keep reading as long as the compressor still wants to read */ + while (next_to_load) { + // Fill compressed buffer; + if (lz4_buffer.empty()) { + unsigned long long read; + /* Reset - if LZ4 decompressor wants to read more, allocate more */ + lz4_buffer.reset(next_to_load); + if (backend.Read(lz4_buffer.getend(), lz4_buffer.free(), &read) == false) + return -1; + lz4_buffer.bufferend += read; + + /* Expected EOF */ + if (read == 0) { + res = -1; + return filefd->FileFdError("LZ4F: %s %s", + filefd->FileName.c_str(), + _("Unexpected end of file")), -1; + } + } + // Drain compressed buffer as far as possible. + size_t in = lz4_buffer.size(); + size_t out = Size; + + res = LZ4F_decompress(dctx, To, &out, lz4_buffer.get(), &in, nullptr); + if (LZ4F_isError(res)) + return -1; + + next_to_load = res; + lz4_buffer.bufferstart += in; + + if (out != 0) + return out; + } + + return 0; + } + virtual bool InternalReadError() override + { + char const * const errmsg = LZ4F_getErrorName(res); + + return filefd->FileFdError("LZ4F: %s %s (%zu: %s)", filefd->FileName.c_str(), _("Read error"), res, errmsg); + } + virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override + { + unsigned long long const towrite = std::min(BLK_SIZE, Size); + + res = LZ4F_compressUpdate(cctx, + lz4_buffer.buffer, lz4_buffer.buffersize_max, + From, towrite, nullptr); + + if (LZ4F_isError(res) || backend.Write(lz4_buffer.buffer, res) == false) + return -1; + + return towrite; + } + virtual bool InternalWriteError() override + { + char const * const errmsg = LZ4F_getErrorName(res); + + return filefd->FileFdError("LZ4F: %s %s (%zu: %s)", filefd->FileName.c_str(), _("Write error"), res, errmsg); + } + virtual bool InternalStream() const override { return true; } + + virtual bool InternalFlush() override + { + return backend.Flush(); + } + + virtual bool InternalClose(std::string const &) override + { + /* Reset variables */ + res = 0; + next_to_load = BLK_SIZE; + + if (cctx != nullptr) + { + res = LZ4F_compressEnd(cctx, lz4_buffer.buffer, lz4_buffer.buffersize_max, nullptr); + if (LZ4F_isError(res) || backend.Write(lz4_buffer.buffer, res) == false) + return false; + if (!backend.Flush()) + return false; + if (!backend.Close()) + return false; + + res = LZ4F_freeCompressionContext(cctx); + cctx = nullptr; + } + + if (dctx != nullptr) + { + res = LZ4F_freeDecompressionContext(dctx); + dctx = nullptr; + } + + return LZ4F_isError(res) == false; + } + + explicit Lz4FileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), dctx(nullptr), cctx(nullptr) {} + virtual ~Lz4FileFdPrivate() { + InternalClose(""); + } +#endif }; /*}}}*/ class APT_HIDDEN LzmaFileFdPrivate: public FileFdPrivate { /*{{{*/ @@ -1968,6 +2123,7 @@ bool FileFd::Open(string FileName,unsigned int const Mode,CompressMode Compress, case Bzip2: name = "bzip2"; break; case Lzma: name = "lzma"; break; case Xz: name = "xz"; break; + case Lz4: name = "lz4"; break; case Auto: case Extension: // Unreachable @@ -2084,6 +2240,7 @@ bool FileFd::OpenDescriptor(int Fd, unsigned int const Mode, CompressMode Compre case Bzip2: name = "bzip2"; break; case Lzma: name = "lzma"; break; case Xz: name = "xz"; break; + case Lz4: name = "lz4"; break; case Auto: case Extension: if (AutoClose == true && Fd != -1) @@ -2145,6 +2302,9 @@ bool FileFd::OpenInternDescriptor(unsigned int const Mode, APT::Configuration::C APT_COMPRESS_INIT("xz", LzmaFileFdPrivate); APT_COMPRESS_INIT("lzma", LzmaFileFdPrivate); #endif +#ifdef HAVE_LZ4 + APT_COMPRESS_INIT("lz4", Lz4FileFdPrivate); +#endif #undef APT_COMPRESS_INIT else if (compressor.Name == "." || compressor.Binary.empty() == true) d = new DirectFileFdPrivate(this); diff --git a/apt-pkg/contrib/fileutl.h b/apt-pkg/contrib/fileutl.h index 8619047a0..f33f7804b 100644 --- a/apt-pkg/contrib/fileutl.h +++ b/apt-pkg/contrib/fileutl.h @@ -45,6 +45,7 @@ class FileFd friend class GzipFileFdPrivate; friend class Bz2FileFdPrivate; friend class LzmaFileFdPrivate; + friend class Lz4FileFdPrivate; friend class DirectFileFdPrivate; friend class PipedFileFdPrivate; protected: @@ -75,7 +76,7 @@ class FileFd ReadOnlyGzip, WriteAtomic = ReadWrite | Create | Atomic }; - enum CompressMode { Auto = 'A', None = 'N', Extension = 'E', Gzip = 'G', Bzip2 = 'B', Lzma = 'L', Xz = 'X' }; + enum CompressMode { Auto = 'A', None = 'N', Extension = 'E', Gzip = 'G', Bzip2 = 'B', Lzma = 'L', Xz = 'X', Lz4='4' }; inline bool Read(void *To,unsigned long long Size,bool AllowEof) { diff --git a/apt-pkg/makefile b/apt-pkg/makefile index 45dcddfe3..f4e750d34 100644 --- a/apt-pkg/makefile +++ b/apt-pkg/makefile @@ -25,6 +25,9 @@ endif ifeq ($(HAVE_LZMA),yes) SLIBS+= -llzma endif +ifeq ($(HAVE_LZ4),yes) +SLIBS+= -llz4 +endif APT_DOMAIN:=libapt-pkg$(LIBAPTPKG_MAJOR) SOURCE = $(wildcard *.cc */*.cc) diff --git a/buildlib/config.h.in b/buildlib/config.h.in index a887ebf88..fb21b387f 100644 --- a/buildlib/config.h.in +++ b/buildlib/config.h.in @@ -14,6 +14,9 @@ /* Define if we have the lzma library for lzma/xz */ #undef HAVE_LZMA +/* Define if we have the lz4 library for lz4 */ +#undef HAVE_LZ4 + /* These two are used by the statvfs shim for glibc2.0 and bsd */ /* Define if we have sys/vfs.h */ #undef HAVE_VFS_H diff --git a/buildlib/environment.mak.in b/buildlib/environment.mak.in index db83f6dcd..9620722de 100644 --- a/buildlib/environment.mak.in +++ b/buildlib/environment.mak.in @@ -69,6 +69,7 @@ HAVE_STATVFS = @HAVE_STATVFS@ HAVE_ZLIB = @HAVE_ZLIB@ HAVE_BZ2 = @HAVE_BZ2@ HAVE_LZMA = @HAVE_LZMA@ +HAVE_LZ4 = @HAVE_LZ4@ NEED_SOCKLEN_T_DEFINE = @NEED_SOCKLEN_T_DEFINE@ # Shared library things diff --git a/configure.ac b/configure.ac index 60c634047..5c49a1094 100644 --- a/configure.ac +++ b/configure.ac @@ -108,6 +108,13 @@ if test "x$HAVE_ZLIB" = "xyes"; then AC_DEFINE(HAVE_ZLIB) fi +HAVE_LZ4=no +AC_CHECK_LIB(lz4, LZ4F_createCompressionContext,[AC_CHECK_HEADER(lz4frame.h, [HAVE_LZ4=yes], [])], []) +AC_SUBST(HAVE_LZ4) +if test "x$HAVE_LZ4" = "xyes"; then + AC_DEFINE(HAVE_LZ4) +fi + HAVE_BZ2=no AC_CHECK_LIB(bz2, BZ2_bzopen,[AC_CHECK_HEADER(bzlib.h, [HAVE_BZ2=yes], [])], []) AC_SUBST(HAVE_BZ2) diff --git a/debian/control b/debian/control index d36b97743..bbbd60b69 100644 --- a/debian/control +++ b/debian/control @@ -7,7 +7,7 @@ Uploaders: Michael Vogt , Standards-Version: 3.9.6 Build-Depends: dpkg-dev (>= 1.15.8), debhelper (>= 8.1.3~), libdb-dev, gettext (>= 0.12), libcurl4-gnutls-dev (>= 7.19.4~), - zlib1g-dev, libbz2-dev, liblzma-dev, + zlib1g-dev, libbz2-dev, liblzma-dev, liblz4-dev, xsltproc, docbook-xsl, docbook-xml, po4a (>= 0.34-2), autotools-dev, autoconf, automake, libgtest-dev , g++ (>= 4:5.2) -- 2.45.2