#ifdef HAVE_LZMA
#include <lzma.h>
#endif
+#ifdef HAVE_LZ4
+ #include <lz4frame.h>
+#endif
#include <endian.h>
#include <stdint.h>
using namespace std;
+/* Should be a multiple of the common page size (4096) */
+static constexpr unsigned long long APT_BUFFER_SIZE = 64 * 1024;
+
// RunScripts - Run a set of scripts from a configuration subtree /*{{{*/
// ---------------------------------------------------------------------
/* */
return false;
// Buffered copy between fds
- constexpr size_t BufSize = 64000;
+ constexpr size_t BufSize = APT_BUFFER_SIZE;
std::unique_ptr<unsigned char[]> Buf(new unsigned char[BufSize]);
unsigned long long ToRead = 0;
do {
/*}}}*/
struct APT_HIDDEN simple_buffer { /*{{{*/
- static constexpr size_t buffersize_max = 4096;
+ size_t buffersize_max = 0;
unsigned long long bufferstart = 0;
unsigned long long bufferend = 0;
- char buffer[buffersize_max];
+ char *buffer = nullptr;
+
+ simple_buffer() {
+ reset(4096);
+ }
+ ~simple_buffer() {
+ delete[] buffer;
+ }
const char *get() const { return buffer + bufferstart; }
char *get() { return buffer + bufferstart; }
+ const char *getend() const { return buffer + bufferend; }
+ char *getend() { return buffer + bufferend; }
bool empty() const { return bufferend <= bufferstart; }
bool full() const { return bufferend == buffersize_max; }
+ unsigned long long free() const { return buffersize_max - bufferend; }
unsigned long long size() const { return bufferend-bufferstart; }
+ void reset(size_t size)
+ {
+ if (size > buffersize_max) {
+ delete[] buffer;
+ buffersize_max = size;
+ buffer = new char[size];
+ }
+ reset();
+ }
void reset() { bufferend = bufferstart = 0; }
ssize_t read(void *to, unsigned long long requested_size) APT_MUSTCHECK
{
}
ssize_t write(const void *from, unsigned long long requested_size) APT_MUSTCHECK
{
- if (buffersize_max - size() < requested_size)
- requested_size = buffersize_max - size();
- memcpy(buffer + bufferend, from, requested_size);
+ if (free() < requested_size)
+ requested_size = free();
+ memcpy(getend(), from, requested_size);
bufferend += requested_size;
if (bufferstart == bufferend)
bufferstart = bufferend = 0;
/*}}}*/
class APT_HIDDEN FileFdPrivate { /*{{{*/
+ friend class BufferedWriteFileFdPrivate;
protected:
FileFd * const filefd;
simple_buffer buffer;
{
buffer.reset();
unsigned long long actualread = 0;
- if (filefd->Read(buffer.get(), buffer.buffersize_max, &actualread) == false)
+ if (filefd->Read(buffer.getend(), buffer.free(), &actualread) == false)
return nullptr;
buffer.bufferend = actualread;
if (buffer.size() == 0)
*To = '\0';
return InitialTo;
}
+ virtual bool InternalFlush()
+ {
+ return true;
+ }
virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) = 0;
virtual bool InternalWriteError() { return filefd->FileFdErrno("write",_("Write error")); }
virtual bool InternalSeek(unsigned long long const To)
return filefd->FileFdError("Seek on file %s because it couldn't be reopened", filefd->FileName.c_str());
buffer.reset();
+ set_seekpos(0);
if (To != 0)
return filefd->Skip(To);
virtual ~FileFdPrivate() {}
};
/*}}}*/
+class APT_HIDDEN BufferedWriteFileFdPrivate : public FileFdPrivate { /*{{{*/
+protected:
+ FileFdPrivate *wrapped;
+ simple_buffer writebuffer;
+
+public:
+
+ explicit BufferedWriteFileFdPrivate(FileFdPrivate *Priv) :
+ FileFdPrivate(Priv->filefd), wrapped(Priv) {};
+
+ virtual APT::Configuration::Compressor get_compressor() const override
+ {
+ return wrapped->get_compressor();
+ }
+ virtual void set_compressor(APT::Configuration::Compressor const &compressor) override
+ {
+ return wrapped->set_compressor(compressor);
+ }
+ virtual unsigned int get_openmode() const override
+ {
+ return wrapped->get_openmode();
+ }
+ virtual void set_openmode(unsigned int openmode) override
+ {
+ return wrapped->set_openmode(openmode);
+ }
+ virtual bool get_is_pipe() const override
+ {
+ return wrapped->get_is_pipe();
+ }
+ virtual void set_is_pipe(bool is_pipe) override
+ {
+ FileFdPrivate::set_is_pipe(is_pipe);
+ wrapped->set_is_pipe(is_pipe);
+ }
+ virtual unsigned long long get_seekpos() const override
+ {
+ return wrapped->get_seekpos();
+ }
+ virtual void set_seekpos(unsigned long long seekpos) override
+ {
+ return wrapped->set_seekpos(seekpos);
+ }
+ virtual bool InternalOpen(int const iFd, unsigned int const Mode) override
+ {
+ if (InternalFlush() == false)
+ return false;
+ return wrapped->InternalOpen(iFd, Mode);
+ }
+ virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) override
+ {
+ if (InternalFlush() == false)
+ return -1;
+ return wrapped->InternalUnbufferedRead(To, Size);
+
+ }
+ virtual bool InternalReadError() override
+ {
+ return wrapped->InternalReadError();
+ }
+ virtual char * InternalReadLine(char * To, unsigned long long Size) override
+ {
+ if (InternalFlush() == false)
+ return nullptr;
+ return wrapped->InternalReadLine(To, Size);
+ }
+ virtual bool InternalFlush() override
+ {
+ while (writebuffer.empty() == false) {
+ auto written = wrapped->InternalWrite(writebuffer.get(),
+ writebuffer.size());
+ // Ignore interrupted syscalls
+ if (written < 0 && errno == EINTR)
+ continue;
+ if (written < 0)
+ return wrapped->InternalWriteError();
+
+ writebuffer.bufferstart += written;
+ }
+
+ writebuffer.reset();
+ return true;
+ }
+ virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override
+ {
+ // Optimisation: If the buffer is empty and we have more to write than
+ // would fit in the buffer (or equal number of bytes), write directly.
+ if (writebuffer.empty() == true && Size >= writebuffer.free())
+ return wrapped->InternalWrite(From, Size);
+
+ // Write as much into the buffer as possible and then flush if needed
+ auto written = writebuffer.write(From, Size);
+
+ if (writebuffer.full() && InternalFlush() == false)
+ return -1;
+
+ return written;
+ }
+ virtual bool InternalWriteError()
+ {
+ return wrapped->InternalWriteError();
+ }
+ virtual bool InternalSeek(unsigned long long const To)
+ {
+ if (InternalFlush() == false)
+ return false;
+ return wrapped->InternalSeek(To);
+ }
+ virtual bool InternalSkip(unsigned long long Over)
+ {
+ if (InternalFlush() == false)
+ return false;
+ return wrapped->InternalSkip(Over);
+ }
+ virtual bool InternalTruncate(unsigned long long const Size)
+ {
+ if (InternalFlush() == false)
+ return false;
+ return wrapped->InternalTruncate(Size);
+ }
+ virtual unsigned long long InternalTell()
+ {
+ if (InternalFlush() == false)
+ return -1;
+ return wrapped->InternalTell();
+ }
+ virtual unsigned long long InternalSize()
+ {
+ if (InternalFlush() == false)
+ return -1;
+ return wrapped->InternalSize();
+ }
+ virtual bool InternalClose(std::string const &FileName)
+ {
+ return wrapped->InternalClose(FileName);
+ }
+ virtual bool InternalAlwaysAutoClose() const
+ {
+ return wrapped->InternalAlwaysAutoClose();
+ }
+ virtual ~BufferedWriteFileFdPrivate()
+ {
+ delete wrapped;
+ }
+};
+ /*}}}*/
class APT_HIDDEN GzipFileFdPrivate: public FileFdPrivate { /*{{{*/
#ifdef HAVE_ZLIB
public:
explicit Bz2FileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), bz2(nullptr) {}
virtual ~Bz2FileFdPrivate() { InternalClose(""); }
#endif
+};
+ /*}}}*/
+class APT_HIDDEN Lz4FileFdPrivate: public FileFdPrivate { /*{{{*/
+ static constexpr unsigned long long LZ4_HEADER_SIZE = 19;
+ static constexpr unsigned long long LZ4_FOOTER_SIZE = 4;
+#ifdef HAVE_LZ4
+ LZ4F_decompressionContext_t dctx;
+ LZ4F_compressionContext_t cctx;
+ LZ4F_errorCode_t res;
+ FileFd backend;
+ simple_buffer lz4_buffer;
+ // Count of bytes that the decompressor expects to read next, or buffer size.
+ size_t next_to_load = APT_BUFFER_SIZE;
+public:
+ virtual bool InternalOpen(int const iFd, unsigned int const Mode) override
+ {
+ if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite)
+ return _error->Error("lz4 only supports write or read mode");
+
+ if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly) {
+ res = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION);
+ lz4_buffer.reset(LZ4F_compressBound(APT_BUFFER_SIZE, nullptr)
+ + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE);
+ } else {
+ res = LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION);
+ lz4_buffer.reset(APT_BUFFER_SIZE);
+ }
+
+ filefd->Flags |= FileFd::Compressed;
+
+ if (LZ4F_isError(res))
+ return false;
+
+ unsigned int flags = (Mode & (FileFd::WriteOnly|FileFd::ReadOnly));
+ if (backend.OpenDescriptor(iFd, flags) == false)
+ return false;
+
+ // Write the file header
+ if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly)
+ {
+ res = LZ4F_compressBegin(cctx, lz4_buffer.buffer, lz4_buffer.buffersize_max, nullptr);
+ if (LZ4F_isError(res) || backend.Write(lz4_buffer.buffer, res) == false)
+ return false;
+ }
+
+ return true;
+ }
+ virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) override
+ {
+ /* Keep reading as long as the compressor still wants to read */
+ while (next_to_load) {
+ // Fill compressed buffer;
+ if (lz4_buffer.empty()) {
+ unsigned long long read;
+ /* Reset - if LZ4 decompressor wants to read more, allocate more */
+ lz4_buffer.reset(next_to_load);
+ if (backend.Read(lz4_buffer.getend(), lz4_buffer.free(), &read) == false)
+ return -1;
+ lz4_buffer.bufferend += read;
+
+ /* Expected EOF */
+ if (read == 0) {
+ res = -1;
+ return filefd->FileFdError("LZ4F: %s %s",
+ filefd->FileName.c_str(),
+ _("Unexpected end of file")), -1;
+ }
+ }
+ // Drain compressed buffer as far as possible.
+ size_t in = lz4_buffer.size();
+ size_t out = Size;
+
+ res = LZ4F_decompress(dctx, To, &out, lz4_buffer.get(), &in, nullptr);
+ if (LZ4F_isError(res))
+ return -1;
+
+ next_to_load = res;
+ lz4_buffer.bufferstart += in;
+
+ if (out != 0)
+ return out;
+ }
+
+ return 0;
+ }
+ virtual bool InternalReadError() override
+ {
+ char const * const errmsg = LZ4F_getErrorName(res);
+
+ return filefd->FileFdError("LZ4F: %s %s (%zu: %s)", filefd->FileName.c_str(), _("Read error"), res, errmsg);
+ }
+ virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override
+ {
+ unsigned long long const towrite = std::min(APT_BUFFER_SIZE, Size);
+
+ res = LZ4F_compressUpdate(cctx,
+ lz4_buffer.buffer, lz4_buffer.buffersize_max,
+ From, towrite, nullptr);
+
+ if (LZ4F_isError(res) || backend.Write(lz4_buffer.buffer, res) == false)
+ return -1;
+
+ return towrite;
+ }
+ virtual bool InternalWriteError() override
+ {
+ char const * const errmsg = LZ4F_getErrorName(res);
+
+ return filefd->FileFdError("LZ4F: %s %s (%zu: %s)", filefd->FileName.c_str(), _("Write error"), res, errmsg);
+ }
+ virtual bool InternalStream() const override { return true; }
+
+ virtual bool InternalFlush() override
+ {
+ return backend.Flush();
+ }
+
+ virtual bool InternalClose(std::string const &) override
+ {
+ /* Reset variables */
+ res = 0;
+ next_to_load = APT_BUFFER_SIZE;
+
+ if (cctx != nullptr)
+ {
+ res = LZ4F_compressEnd(cctx, lz4_buffer.buffer, lz4_buffer.buffersize_max, nullptr);
+ if (LZ4F_isError(res) || backend.Write(lz4_buffer.buffer, res) == false)
+ return false;
+ if (!backend.Flush())
+ return false;
+ if (!backend.Close())
+ return false;
+
+ res = LZ4F_freeCompressionContext(cctx);
+ cctx = nullptr;
+ }
+
+ if (dctx != nullptr)
+ {
+ res = LZ4F_freeDecompressionContext(dctx);
+ dctx = nullptr;
+ }
+
+ return LZ4F_isError(res) == false;
+ }
+
+ explicit Lz4FileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), dctx(nullptr), cctx(nullptr) {}
+ virtual ~Lz4FileFdPrivate() {
+ InternalClose("");
+ }
+#endif
};
/*}}}*/
class APT_HIDDEN LzmaFileFdPrivate: public FileFdPrivate { /*{{{*/
}
virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override
{
+ ssize_t Res;
lzma->stream.next_in = (uint8_t *)From;
lzma->stream.avail_in = Size;
lzma->stream.next_out = lzma->buffer;
size_t const n = sizeof(lzma->buffer)/sizeof(lzma->buffer[0]) - lzma->stream.avail_out;
size_t const m = (n == 0) ? 0 : fwrite(lzma->buffer, 1, n, lzma->file);
if (m != n)
- return -1;
+ {
+ Res = -1;
+ errno = 0;
+ }
else
- return Size - lzma->stream.avail_in;
+ {
+ Res = Size - lzma->stream.avail_in;
+ if (Res == 0)
+ {
+ // lzma run was okay, but produced no output…
+ Res = -1;
+ errno = EINTR;
+ }
+ }
+ return Res;
}
virtual bool InternalWriteError() override
{
case Bzip2: name = "bzip2"; break;
case Lzma: name = "lzma"; break;
case Xz: name = "xz"; break;
+ case Lz4: name = "lz4"; break;
case Auto:
case Extension:
// Unreachable
case Bzip2: name = "bzip2"; break;
case Lzma: name = "lzma"; break;
case Xz: name = "xz"; break;
+ case Lz4: name = "lz4"; break;
case Auto:
case Extension:
if (AutoClose == true && Fd != -1)
APT_COMPRESS_INIT("xz", LzmaFileFdPrivate);
APT_COMPRESS_INIT("lzma", LzmaFileFdPrivate);
#endif
+#ifdef HAVE_LZ4
+ APT_COMPRESS_INIT("lz4", Lz4FileFdPrivate);
+#endif
#undef APT_COMPRESS_INIT
else if (compressor.Name == "." || compressor.Binary.empty() == true)
d = new DirectFileFdPrivate(this);
else
d = new PipedFileFdPrivate(this);
+ if (Mode & BufferedWrite)
+ d = new BufferedWriteFileFdPrivate(d);
+
d->set_openmode(Mode);
d->set_compressor(compressor);
if ((Flags & AutoClose) != AutoClose && d->InternalAlwaysAutoClose())
return d->InternalReadLine(To, Size);
}
/*}}}*/
+// FileFd::Flush - Flush the file /*{{{*/
+bool FileFd::Flush()
+{
+ if (d == nullptr)
+ return true;
+
+ return d->InternalFlush();
+}
+ /*}}}*/
// FileFd::Write - Write to the file /*{{{*/
bool FileFd::Write(const void *From,unsigned long long Size)
{
while (Res > 0 && Size > 0)
{
Res = d->InternalWrite(From, Size);
- if (Res < 0 && errno == EINTR)
- continue;
+
if (Res < 0)
+ {
+ if (errno == EINTR)
+ {
+ // trick the while-loop into running again
+ Res = 1;
+ errno = 0;
+ continue;
+ }
return d->InternalWriteError();
+ }
From = (char const *)From + Res;
Size -= Res;
/* */
bool FileFd::Close()
{
+ if (Flush() == false)
+ return false;
if (iFd == -1)
return true;
projects / apt.git / blobdiff