X-Git-Url: https://git.saurik.com/apt.git/blobdiff_plain/83e22e26f9f10472aed97f889967c86ee218d28d..47fcfff8a50caebe3cd1f2ee19875a2ca8bbeadc:/apt-pkg/contrib/fileutl.cc diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index c4ff36a13..004df3fe8 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -63,6 +63,9 @@ #ifdef HAVE_LZMA #include #endif +#ifdef HAVE_LZ4 + #include +#endif #include #include @@ -75,6 +78,9 @@ using namespace std; +/* Should be a multiple of the common page size (4096) */ +static constexpr unsigned long long APT_BUFFER_SIZE = 64 * 1024; + // RunScripts - Run a set of scripts from a configuration subtree /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -160,7 +166,7 @@ bool CopyFile(FileFd &From,FileFd &To) return false; // Buffered copy between fds - constexpr size_t BufSize = 64000; + constexpr size_t BufSize = APT_BUFFER_SIZE; std::unique_ptr Buf(new unsigned char[BufSize]); unsigned long long ToRead = 0; do { @@ -919,40 +925,108 @@ bool ChangeOwnerAndPermissionOfFile(char const * const requester, char const * c } /*}}}*/ +struct APT_HIDDEN simple_buffer { /*{{{*/ + size_t buffersize_max = 0; + unsigned long long bufferstart = 0; + unsigned long long bufferend = 0; + char *buffer = nullptr; + + simple_buffer() { + reset(4096); + } + ~simple_buffer() { + delete[] buffer; + } + + const char *get() const { return buffer + bufferstart; } + char *get() { return buffer + bufferstart; } + const char *getend() const { return buffer + bufferend; } + char *getend() { return buffer + bufferend; } + bool empty() const { return bufferend <= bufferstart; } + bool full() const { return bufferend == buffersize_max; } + unsigned long long free() const { return buffersize_max - bufferend; } + unsigned long long size() const { return bufferend-bufferstart; } + void reset(size_t size) + { + if (size > buffersize_max) { + delete[] buffer; + buffersize_max = size; + buffer = new char[size]; + } + reset(); + } + void reset() { bufferend = bufferstart = 0; } + ssize_t read(void *to, unsigned long long requested_size) APT_MUSTCHECK + { + if (size() < requested_size) + requested_size = size(); + memcpy(to, buffer + bufferstart, requested_size); + bufferstart += requested_size; + if (bufferstart == bufferend) + bufferstart = bufferend = 0; + return requested_size; + } + ssize_t write(const void *from, unsigned long long requested_size) APT_MUSTCHECK + { + if (free() < requested_size) + requested_size = free(); + memcpy(getend(), from, requested_size); + bufferend += requested_size; + if (bufferstart == bufferend) + bufferstart = bufferend = 0; + return requested_size; + } +}; + /*}}}*/ + class APT_HIDDEN FileFdPrivate { /*{{{*/ + friend class BufferedWriteFileFdPrivate; protected: FileFd * const filefd; - struct simple_buffer { - static constexpr size_t buffersize_max = 4096; - unsigned long long bufferstart = 0; - unsigned long long bufferend = 0; - char buffer[buffersize_max]; - - char *get() { return buffer + bufferstart; } - bool empty() { return bufferend <= bufferstart; } - unsigned long long size() { return bufferend-bufferstart; } - void reset() { bufferend = bufferstart = 0; } - ssize_t read(void *to, unsigned long long requested_size) - { - if (size() < requested_size) - requested_size = size(); - memcpy(to, buffer + bufferstart, requested_size); - bufferstart += requested_size; - if (bufferstart == bufferend) - bufferstart = bufferend = 0; - return requested_size; - } - } buffer; -public: + simple_buffer buffer; int compressed_fd; pid_t compressor_pid; bool is_pipe; APT::Configuration::Compressor compressor; unsigned int openmode; unsigned long long seekpos; +public: + explicit FileFdPrivate(FileFd * const pfilefd) : filefd(pfilefd), compressed_fd(-1), compressor_pid(-1), is_pipe(false), openmode(0), seekpos(0) {}; + virtual APT::Configuration::Compressor get_compressor() const + { + return compressor; + } + virtual void set_compressor(APT::Configuration::Compressor const &compressor) + { + this->compressor = compressor; + } + virtual unsigned int get_openmode() const + { + return openmode; + } + virtual void set_openmode(unsigned int openmode) + { + this->openmode = openmode; + } + virtual bool get_is_pipe() const + { + return is_pipe; + } + virtual void set_is_pipe(bool is_pipe) + { + this->is_pipe = is_pipe; + } + virtual unsigned long long get_seekpos() const + { + return seekpos; + } + virtual void set_seekpos(unsigned long long seekpos) + { + this->seekpos = seekpos; + } virtual bool InternalOpen(int const iFd, unsigned int const Mode) = 0; ssize_t InternalRead(void * To, unsigned long long Size) @@ -970,13 +1044,12 @@ public: { if (unlikely(Size == 0)) return nullptr; + // Read one byte less than buffer size to have space for trailing 0. --Size; - To[0] = '\0'; - if (unlikely(Size == 0)) - return To; + char * const InitialTo = To; - do { + while (Size > 0) { if (buffer.empty() == true) { buffer.reset(); @@ -995,23 +1068,23 @@ public: unsigned long long const OutputSize = std::min(Size, buffer.size()); char const * const newline = static_cast(memchr(buffer.get(), '\n', OutputSize)); + // Read until end of line or up to Size bytes from the buffer. + unsigned long long actualread = buffer.read(To, + (newline != nullptr) + ? (newline - buffer.get()) + 1 + : OutputSize); + To += actualread; + Size -= actualread; if (newline != nullptr) - { - size_t length = (newline - buffer.get()) + 1; - buffer.read(To, length); - To += length; break; - } - else - { - buffer.read(To, OutputSize); - To += OutputSize; - Size -= OutputSize; - } - } while (Size > 0); + } *To = '\0'; return InitialTo; } + virtual bool InternalFlush() + { + return true; + } virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) = 0; virtual bool InternalWriteError() { return filefd->FileFdErrno("write",_("Write error")); } virtual bool InternalSeek(unsigned long long const To) @@ -1102,6 +1175,152 @@ public: virtual ~FileFdPrivate() {} }; /*}}}*/ +class APT_HIDDEN BufferedWriteFileFdPrivate : public FileFdPrivate { /*{{{*/ +protected: + FileFdPrivate *wrapped; + simple_buffer writebuffer; + +public: + + explicit BufferedWriteFileFdPrivate(FileFdPrivate *Priv) : + FileFdPrivate(Priv->filefd), wrapped(Priv) {}; + + virtual APT::Configuration::Compressor get_compressor() const override + { + return wrapped->get_compressor(); + } + virtual void set_compressor(APT::Configuration::Compressor const &compressor) override + { + return wrapped->set_compressor(compressor); + } + virtual unsigned int get_openmode() const override + { + return wrapped->get_openmode(); + } + virtual void set_openmode(unsigned int openmode) override + { + return wrapped->set_openmode(openmode); + } + virtual bool get_is_pipe() const override + { + return wrapped->get_is_pipe(); + } + virtual void set_is_pipe(bool is_pipe) override + { + FileFdPrivate::set_is_pipe(is_pipe); + wrapped->set_is_pipe(is_pipe); + } + virtual unsigned long long get_seekpos() const override + { + return wrapped->get_seekpos(); + } + virtual void set_seekpos(unsigned long long seekpos) override + { + return wrapped->set_seekpos(seekpos); + } + virtual bool InternalOpen(int const iFd, unsigned int const Mode) override + { + if (InternalFlush() == false) + return false; + return wrapped->InternalOpen(iFd, Mode); + } + virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) override + { + if (InternalFlush() == false) + return -1; + return wrapped->InternalUnbufferedRead(To, Size); + + } + virtual bool InternalReadError() override + { + return wrapped->InternalReadError(); + } + virtual char * InternalReadLine(char * To, unsigned long long Size) override + { + if (InternalFlush() == false) + return nullptr; + return wrapped->InternalReadLine(To, Size); + } + virtual bool InternalFlush() override + { + while (writebuffer.empty() == false) { + auto written = wrapped->InternalWrite(writebuffer.get(), + writebuffer.size()); + // Ignore interrupted syscalls + if (written < 0 && errno == EINTR) + continue; + if (written < 0) + return false; + + writebuffer.bufferstart += written; + } + + writebuffer.reset(); + return true; + } + virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override + { + size_t written = 0; + + while (written < Size) { + auto buffered = writebuffer.write(static_cast(From) + written, Size - written); + + written += buffered; + + if (writebuffer.full() && InternalFlush() == false) + return -1; + } + + return written; + } + virtual bool InternalWriteError() + { + return wrapped->InternalWriteError(); + } + virtual bool InternalSeek(unsigned long long const To) + { + if (InternalFlush() == false) + return false; + return wrapped->InternalSeek(To); + } + virtual bool InternalSkip(unsigned long long Over) + { + if (InternalFlush() == false) + return false; + return wrapped->InternalSkip(Over); + } + virtual bool InternalTruncate(unsigned long long const Size) + { + if (InternalFlush() == false) + return false; + return wrapped->InternalTruncate(Size); + } + virtual unsigned long long InternalTell() + { + if (InternalFlush() == false) + return -1; + return wrapped->InternalTell(); + } + virtual unsigned long long InternalSize() + { + if (InternalFlush() == false) + return -1; + return wrapped->InternalSize(); + } + virtual bool InternalClose(std::string const &FileName) + { + return wrapped->InternalClose(FileName); + } + virtual bool InternalAlwaysAutoClose() const + { + return wrapped->InternalAlwaysAutoClose(); + } + virtual ~BufferedWriteFileFdPrivate() + { + delete wrapped; + } +}; + /*}}}*/ class APT_HIDDEN GzipFileFdPrivate: public FileFdPrivate { /*{{{*/ #ifdef HAVE_ZLIB public: @@ -1281,6 +1500,157 @@ public: explicit Bz2FileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), bz2(nullptr) {} virtual ~Bz2FileFdPrivate() { InternalClose(""); } #endif +}; + /*}}}*/ +class APT_HIDDEN Lz4FileFdPrivate: public FileFdPrivate { /*{{{*/ + static constexpr unsigned long long LZ4_HEADER_SIZE = 19; + static constexpr unsigned long long LZ4_FOOTER_SIZE = 4; +#ifdef HAVE_LZ4 + LZ4F_decompressionContext_t dctx; + LZ4F_compressionContext_t cctx; + LZ4F_errorCode_t res; + FileFd backend; + simple_buffer lz4_buffer; + // Count of bytes that the decompressor expects to read next, or buffer size. + size_t next_to_load = APT_BUFFER_SIZE; +public: + virtual bool InternalOpen(int const iFd, unsigned int const Mode) override + { + if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite) + return _error->Error("lz4 only supports write or read mode"); + + if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly) { + res = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION); + lz4_buffer.reset(LZ4F_compressBound(APT_BUFFER_SIZE, nullptr) + + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE); + } else { + res = LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION); + lz4_buffer.reset(APT_BUFFER_SIZE); + } + + filefd->Flags |= FileFd::Compressed; + + if (LZ4F_isError(res)) + return false; + + unsigned int flags = (Mode & (FileFd::WriteOnly|FileFd::ReadOnly)); + if (backend.OpenDescriptor(iFd, flags) == false) + return false; + + // Write the file header + if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly) + { + res = LZ4F_compressBegin(cctx, lz4_buffer.buffer, lz4_buffer.buffersize_max, nullptr); + if (LZ4F_isError(res) || backend.Write(lz4_buffer.buffer, res) == false) + return false; + } + + return true; + } + virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) override + { + /* Keep reading as long as the compressor still wants to read */ + while (next_to_load) { + // Fill compressed buffer; + if (lz4_buffer.empty()) { + unsigned long long read; + /* Reset - if LZ4 decompressor wants to read more, allocate more */ + lz4_buffer.reset(next_to_load); + if (backend.Read(lz4_buffer.getend(), lz4_buffer.free(), &read) == false) + return -1; + lz4_buffer.bufferend += read; + + /* Expected EOF */ + if (read == 0) { + res = -1; + return filefd->FileFdError("LZ4F: %s %s", + filefd->FileName.c_str(), + _("Unexpected end of file")), -1; + } + } + // Drain compressed buffer as far as possible. + size_t in = lz4_buffer.size(); + size_t out = Size; + + res = LZ4F_decompress(dctx, To, &out, lz4_buffer.get(), &in, nullptr); + if (LZ4F_isError(res)) + return -1; + + next_to_load = res; + lz4_buffer.bufferstart += in; + + if (out != 0) + return out; + } + + return 0; + } + virtual bool InternalReadError() override + { + char const * const errmsg = LZ4F_getErrorName(res); + + return filefd->FileFdError("LZ4F: %s %s (%zu: %s)", filefd->FileName.c_str(), _("Read error"), res, errmsg); + } + virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override + { + unsigned long long const towrite = std::min(APT_BUFFER_SIZE, Size); + + res = LZ4F_compressUpdate(cctx, + lz4_buffer.buffer, lz4_buffer.buffersize_max, + From, towrite, nullptr); + + if (LZ4F_isError(res) || backend.Write(lz4_buffer.buffer, res) == false) + return -1; + + return towrite; + } + virtual bool InternalWriteError() override + { + char const * const errmsg = LZ4F_getErrorName(res); + + return filefd->FileFdError("LZ4F: %s %s (%zu: %s)", filefd->FileName.c_str(), _("Write error"), res, errmsg); + } + virtual bool InternalStream() const override { return true; } + + virtual bool InternalFlush() override + { + return backend.Flush(); + } + + virtual bool InternalClose(std::string const &) override + { + /* Reset variables */ + res = 0; + next_to_load = APT_BUFFER_SIZE; + + if (cctx != nullptr) + { + res = LZ4F_compressEnd(cctx, lz4_buffer.buffer, lz4_buffer.buffersize_max, nullptr); + if (LZ4F_isError(res) || backend.Write(lz4_buffer.buffer, res) == false) + return false; + if (!backend.Flush()) + return false; + if (!backend.Close()) + return false; + + res = LZ4F_freeCompressionContext(cctx); + cctx = nullptr; + } + + if (dctx != nullptr) + { + res = LZ4F_freeDecompressionContext(dctx); + dctx = nullptr; + } + + return LZ4F_isError(res) == false; + } + + explicit Lz4FileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), dctx(nullptr), cctx(nullptr) {} + virtual ~Lz4FileFdPrivate() { + InternalClose(""); + } +#endif }; /*}}}*/ class APT_HIDDEN LzmaFileFdPrivate: public FileFdPrivate { /*{{{*/ @@ -1516,7 +1886,7 @@ public: SetCloseExec(Pipe[J],true); compressed_fd = filefd->iFd; - is_pipe = true; + set_is_pipe(true); if (Comp == true) filefd->iFd = Pipe[1]; @@ -1755,6 +2125,7 @@ bool FileFd::Open(string FileName,unsigned int const Mode,CompressMode Compress, case Bzip2: name = "bzip2"; break; case Lzma: name = "lzma"; break; case Xz: name = "xz"; break; + case Lz4: name = "lz4"; break; case Auto: case Extension: // Unreachable @@ -1871,6 +2242,7 @@ bool FileFd::OpenDescriptor(int Fd, unsigned int const Mode, CompressMode Compre case Bzip2: name = "bzip2"; break; case Lzma: name = "lzma"; break; case Xz: name = "xz"; break; + case Lz4: name = "lz4"; break; case Auto: case Extension: if (AutoClose == true && Fd != -1) @@ -1932,14 +2304,20 @@ bool FileFd::OpenInternDescriptor(unsigned int const Mode, APT::Configuration::C APT_COMPRESS_INIT("xz", LzmaFileFdPrivate); APT_COMPRESS_INIT("lzma", LzmaFileFdPrivate); #endif +#ifdef HAVE_LZ4 + APT_COMPRESS_INIT("lz4", Lz4FileFdPrivate); +#endif #undef APT_COMPRESS_INIT else if (compressor.Name == "." || compressor.Binary.empty() == true) d = new DirectFileFdPrivate(this); else d = new PipedFileFdPrivate(this); - d->openmode = Mode; - d->compressor = compressor; + if (Mode & BufferedWrite) + d = new BufferedWriteFileFdPrivate(d); + + d->set_openmode(Mode); + d->set_compressor(compressor); if ((Flags & AutoClose) != AutoClose && d->InternalAlwaysAutoClose()) { // Need to duplicate fd here or gz/bz2 close for cleanup will close the fd as well @@ -1997,7 +2375,7 @@ bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual) To = (char *)To + Res; Size -= Res; if (d != NULL) - d->seekpos += Res; + d->set_seekpos(d->get_seekpos() + Res); if (Actual != 0) *Actual += Res; } @@ -2027,6 +2405,15 @@ char* FileFd::ReadLine(char *To, unsigned long long const Size) return d->InternalReadLine(To, Size); } /*}}}*/ +// FileFd::Flush - Flush the file /*{{{*/ +bool FileFd::Flush() +{ + if (d == nullptr) + return true; + + return d->InternalFlush(); +} + /*}}}*/ // FileFd::Write - Write to the file /*{{{*/ bool FileFd::Write(const void *From,unsigned long long Size) { @@ -2045,7 +2432,7 @@ bool FileFd::Write(const void *From,unsigned long long Size) From = (char const *)From + Res; Size -= Res; if (d != NULL) - d->seekpos += Res; + d->set_seekpos(d->get_seekpos() + Res); } if (Size == 0) @@ -2113,13 +2500,13 @@ unsigned long long FileFd::Tell() off_t const Res = d->InternalTell(); if (Res == (off_t)-1) FileFdErrno("lseek","Failed to determine the current file position"); - d->seekpos = Res; + d->set_seekpos(Res); return Res; } /*}}}*/ static bool StatFileFd(char const * const msg, int const iFd, std::string const &FileName, struct stat &Buf, FileFdPrivate * const d) /*{{{*/ { - bool ispipe = (d != NULL && d->is_pipe == true); + bool ispipe = (d != NULL && d->get_is_pipe() == true); if (ispipe == false) { if (fstat(iFd,&Buf) != 0) @@ -2136,7 +2523,7 @@ static bool StatFileFd(char const * const msg, int const iFd, std::string const // we set it here, too, as we get the info here for free // in theory the Open-methods should take care of it already if (d != NULL) - d->is_pipe = true; + d->set_is_pipe(true); if (stat(FileName.c_str(), &Buf) != 0) return _error->Errno("fstat", "Unable to determine %s for file %s", msg, FileName.c_str()); } @@ -2180,6 +2567,8 @@ unsigned long long FileFd::Size() /* */ bool FileFd::Close() { + if (Flush() == false) + return false; if (iFd == -1) return true;