Bug #807012 also involves package dependencies :/.

[apt.git] / apt-pkg / contrib / fileutl.cc

diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc
index 11b0e658daf208add730a772c6211f9347c58b05..dd36ffa790070a2cb3648aa5dd5411223a5cde24 100644 (file)
--- a/apt-pkg/contrib/fileutl.cc
+++ b/apt-pkg/contrib/fileutl.cc
@@ -63,6 +63,9 @@
 #ifdef HAVE_LZMA
        #include <lzma.h>
 #endif
 #ifdef HAVE_LZMA
        #include <lzma.h>
 #endif

+#ifdef HAVE_LZ4
+       #include <lz4frame.h>
+#endif

 #include <endian.h>
 #include <stdint.h>
 
 #include <endian.h>
 #include <stdint.h>
 


@@ -75,6 +78,9 @@
 
 using namespace std;
 
 
 using namespace std;
 

+/* Should be a multiple of the common page size (4096) */
+static constexpr unsigned long long APT_BUFFER_SIZE = 64 * 1024;
+

 // RunScripts - Run a set of scripts from a configuration subtree      /*{{{*/
 // ---------------------------------------------------------------------
 /* */
 // RunScripts - Run a set of scripts from a configuration subtree      /*{{{*/
 // ---------------------------------------------------------------------
 /* */


@@ -160,7 +166,7 @@ bool CopyFile(FileFd &From,FileFd &To)
       return false;
 
    // Buffered copy between fds
       return false;
 
    // Buffered copy between fds

-   constexpr size_t BufSize = 64000;
+   constexpr size_t BufSize = APT_BUFFER_SIZE;

    std::unique_ptr<unsigned char[]> Buf(new unsigned char[BufSize]);
    unsigned long long ToRead = 0;
    do {
    std::unique_ptr<unsigned char[]> Buf(new unsigned char[BufSize]);
    unsigned long long ToRead = 0;
    do {


@@ -688,6 +694,25 @@ string flAbsPath(string File)
    return AbsPath;
 }
                                                                        /*}}}*/
    return AbsPath;
 }
                                                                        /*}}}*/

+std::string flNormalize(std::string file)                              /*{{{*/
+{
+   if (file.empty())
+      return file;
+   // do some normalisation by removing // and /./ from the path
+   size_t found = string::npos;
+   while ((found = file.find("/./")) != string::npos)
+      file.replace(found, 3, "/");
+   while ((found = file.find("//")) != string::npos)
+      file.replace(found, 2, "/");
+
+   if (APT::String::Startswith(file, "/dev/null"))
+   {
+      file.erase(strlen("/dev/null"));
+      return file;
+   }
+   return file;
+}
+                                                                       /*}}}*/

 // SetCloseExec - Set the close on exec flag                           /*{{{*/
 // ---------------------------------------------------------------------
 /* */
 // SetCloseExec - Set the close on exec flag                           /*{{{*/
 // ---------------------------------------------------------------------
 /* */


@@ -910,49 +935,177 @@ bool ChangeOwnerAndPermissionOfFile(char const * const requester, char const * c
       // ensure the file is owned by root and has good permissions
       struct passwd const * const pw = getpwnam(user);
       struct group const * const gr = getgrnam(group);
       // ensure the file is owned by root and has good permissions
       struct passwd const * const pw = getpwnam(user);
       struct group const * const gr = getgrnam(group);

-      if (pw != NULL && gr != NULL && chown(file, pw->pw_uid, gr->gr_gid) != 0)
+      if (pw != NULL && gr != NULL && lchown(file, pw->pw_uid, gr->gr_gid) != 0)

         Res &= _error->WarningE(requester, "chown to %s:%s of file %s failed", user, group, file);
    }
         Res &= _error->WarningE(requester, "chown to %s:%s of file %s failed", user, group, file);
    }

+   struct stat Buf;
+   if (lstat(file, &Buf) != 0 || S_ISLNK(Buf.st_mode))
+      return Res;

    if (chmod(file, mode) != 0)
       Res &= _error->WarningE(requester, "chmod 0%o of file %s failed", mode, file);
    return Res;
 }
                                                                        /*}}}*/
 
    if (chmod(file, mode) != 0)
       Res &= _error->WarningE(requester, "chmod 0%o of file %s failed", mode, file);
    return Res;
 }
                                                                        /*}}}*/
 

-class FileFdPrivate {                                                  /*{{{*/
+struct APT_HIDDEN simple_buffer {                                                      /*{{{*/
+   size_t buffersize_max = 0;
+   unsigned long long bufferstart = 0;
+   unsigned long long bufferend = 0;
+   char *buffer = nullptr;
+
+   simple_buffer() {
+      reset(4096);
+   }
+   ~simple_buffer() {
+      delete[] buffer;
+   }
+
+   const char *get() const { return buffer + bufferstart; }
+   char *get() { return buffer + bufferstart; }
+   const char *getend() const { return buffer + bufferend; }
+   char *getend() { return buffer + bufferend; }
+   bool empty() const { return bufferend <= bufferstart; }
+   bool full() const { return bufferend == buffersize_max; }
+   unsigned long long free() const { return buffersize_max - bufferend; }
+   unsigned long long size() const { return bufferend-bufferstart; }
+   void reset(size_t size)
+   {
+      if (size > buffersize_max) {
+        delete[] buffer;
+        buffersize_max = size;
+        buffer = new char[size];
+      }
+      reset();
+   }
+   void reset() { bufferend = bufferstart = 0; }
+   ssize_t read(void *to, unsigned long long requested_size) APT_MUSTCHECK
+   {
+      if (size() < requested_size)
+        requested_size = size();
+      memcpy(to, buffer + bufferstart, requested_size);
+      bufferstart += requested_size;
+      if (bufferstart == bufferend)
+        bufferstart = bufferend = 0;
+      return requested_size;
+   }
+   ssize_t write(const void *from, unsigned long long requested_size) APT_MUSTCHECK
+   {
+      if (free() < requested_size)
+        requested_size = free();
+      memcpy(getend(), from, requested_size);
+      bufferend += requested_size;
+      if (bufferstart == bufferend)
+        bufferstart = bufferend = 0;
+      return requested_size;
+   }
+};
+                                                                       /*}}}*/
+
+class APT_HIDDEN FileFdPrivate {                                                       /*{{{*/
+   friend class BufferedWriteFileFdPrivate;

 protected:
    FileFd * const filefd;
 protected:
    FileFd * const filefd;

-public:
+   simple_buffer buffer;

    int compressed_fd;
    pid_t compressor_pid;
    bool is_pipe;
    APT::Configuration::Compressor compressor;
    unsigned int openmode;
    unsigned long long seekpos;
    int compressed_fd;
    pid_t compressor_pid;
    bool is_pipe;
    APT::Configuration::Compressor compressor;
    unsigned int openmode;
    unsigned long long seekpos;

-   FileFdPrivate(FileFd * const pfilefd) : filefd(pfilefd),
+public:
+
+   explicit FileFdPrivate(FileFd * const pfilefd) : filefd(pfilefd),

       compressed_fd(-1), compressor_pid(-1), is_pipe(false),
       openmode(0), seekpos(0) {};
       compressed_fd(-1), compressor_pid(-1), is_pipe(false),
       openmode(0), seekpos(0) {};

+   virtual APT::Configuration::Compressor get_compressor() const
+   {
+      return compressor;
+   }
+   virtual void set_compressor(APT::Configuration::Compressor const &compressor)
+   {
+      this->compressor = compressor;
+   }
+   virtual unsigned int get_openmode() const
+   {
+      return openmode;
+   }
+   virtual void set_openmode(unsigned int openmode)
+   {
+      this->openmode = openmode;
+   }
+   virtual bool get_is_pipe() const
+   {
+      return is_pipe;
+   }
+   virtual void set_is_pipe(bool is_pipe)
+   {
+      this->is_pipe = is_pipe;
+   }
+   virtual unsigned long long get_seekpos() const
+   {
+      return seekpos;
+   }
+   virtual void set_seekpos(unsigned long long seekpos)
+   {
+      this->seekpos = seekpos;
+   }

 
    virtual bool InternalOpen(int const iFd, unsigned int const Mode) = 0;
 
    virtual bool InternalOpen(int const iFd, unsigned int const Mode) = 0;

-   virtual ssize_t InternalRead(void * const To, unsigned long long const Size) = 0;
-   virtual bool InternalReadError() { return filefd->FileFdErrno("read",_("Read error")); }
-   virtual char * InternalReadLine(char * const To, unsigned long long const Size)
+   ssize_t InternalRead(void * To, unsigned long long Size)

    {
    {

-      unsigned long long read = 0;
-      while ((Size - 1) != read)
+      // Drain the buffer if needed.
+      if (buffer.empty() == false)

       {
       {

-        unsigned long long done = 0;
-        if (filefd->Read(To + read, 1, &done) == false)
-           return NULL;
-        if (done == 0)
-           break;
-        if (To[read++] == '\n')
+        return buffer.read(To, Size);
+      }
+      return InternalUnbufferedRead(To, Size);
+   }
+   virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) = 0;
+   virtual bool InternalReadError() { return filefd->FileFdErrno("read",_("Read error")); }
+   virtual char * InternalReadLine(char * To, unsigned long long Size)
+   {
+      if (unlikely(Size == 0))
+        return nullptr;
+      // Read one byte less than buffer size to have space for trailing 0.
+      --Size;
+
+      char * const InitialTo = To;
+
+      while (Size > 0) {
+        if (buffer.empty() == true)
+        {
+           buffer.reset();
+           unsigned long long actualread = 0;
+           if (filefd->Read(buffer.getend(), buffer.free(), &actualread) == false)
+              return nullptr;
+           buffer.bufferend = actualread;
+           if (buffer.size() == 0)
+           {
+              if (To == InitialTo)
+                 return nullptr;
+              break;
+           }
+           filefd->Flags &= ~FileFd::HitEof;
+        }
+
+        unsigned long long const OutputSize = std::min(Size, buffer.size());
+        char const * const newline = static_cast<char const * const>(memchr(buffer.get(), '\n', OutputSize));
+        // Read until end of line or up to Size bytes from the buffer.
+        unsigned long long actualread = buffer.read(To,
+                                                    (newline != nullptr)
+                                                    ? (newline - buffer.get()) + 1
+                                                    : OutputSize);
+        To += actualread;
+        Size -= actualread;
+        if (newline != nullptr)

            break;
       }
            break;
       }

-      if (read == 0)
-        return NULL;
-      To[read] = '\0';
-      return To;
+      *To = '\0';
+      return InitialTo;
+   }
+   virtual bool InternalFlush()
+   {
+      return true;

    }
    virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) = 0;
    virtual bool InternalWriteError() { return filefd->FileFdErrno("write",_("Write error")); }
    }
    virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) = 0;
    virtual bool InternalWriteError() { return filefd->FileFdErrno("write",_("Write error")); }


@@ -987,6 +1140,8 @@ public:
       if (filefd->OpenInternDescriptor(openmode, compressor) == false)
         return filefd->FileFdError("Seek on file %s because it couldn't be reopened", filefd->FileName.c_str());
 
       if (filefd->OpenInternDescriptor(openmode, compressor) == false)
         return filefd->FileFdError("Seek on file %s because it couldn't be reopened", filefd->FileName.c_str());
 

+      buffer.reset();
+      set_seekpos(0);

       if (To != 0)
         return filefd->Skip(To);
 
       if (To != 0)
         return filefd->Skip(To);
 


@@ -1016,7 +1171,7 @@ public:
       // seeking around, but not all users of FileFd use always Seek() and co
       // so d->seekpos isn't always true and we can just use it as a hint if
       // we have nothing else, but not always as an authority…
       // seeking around, but not all users of FileFd use always Seek() and co
       // so d->seekpos isn't always true and we can just use it as a hint if
       // we have nothing else, but not always as an authority…

-      return seekpos;
+      return seekpos - buffer.size();

    }
    virtual unsigned long long InternalSize()
    {
    }
    virtual unsigned long long InternalSize()
    {


@@ -1043,11 +1198,156 @@ public:
    virtual ~FileFdPrivate() {}
 };
                                                                        /*}}}*/
    virtual ~FileFdPrivate() {}
 };
                                                                        /*}}}*/

-class GzipFileFdPrivate: public FileFdPrivate {                                /*{{{*/
+class APT_HIDDEN BufferedWriteFileFdPrivate : public FileFdPrivate {   /*{{{*/
+protected:
+   FileFdPrivate *wrapped;
+   simple_buffer writebuffer;
+
+public:
+
+   explicit BufferedWriteFileFdPrivate(FileFdPrivate *Priv) :
+      FileFdPrivate(Priv->filefd), wrapped(Priv) {};
+
+   virtual APT::Configuration::Compressor get_compressor() const APT_OVERRIDE
+   {
+      return wrapped->get_compressor();
+   }
+   virtual void set_compressor(APT::Configuration::Compressor const &compressor)  APT_OVERRIDE
+   {
+      return wrapped->set_compressor(compressor);
+   }
+   virtual unsigned int get_openmode() const  APT_OVERRIDE
+   {
+      return wrapped->get_openmode();
+   }
+   virtual void set_openmode(unsigned int openmode)  APT_OVERRIDE
+   {
+      return wrapped->set_openmode(openmode);
+   }
+   virtual bool get_is_pipe() const  APT_OVERRIDE
+   {
+      return wrapped->get_is_pipe();
+   }
+   virtual void set_is_pipe(bool is_pipe) APT_OVERRIDE
+   {
+      FileFdPrivate::set_is_pipe(is_pipe);
+      wrapped->set_is_pipe(is_pipe);
+   }
+   virtual unsigned long long get_seekpos() const APT_OVERRIDE
+   {
+      return wrapped->get_seekpos();
+   }
+   virtual void set_seekpos(unsigned long long seekpos) APT_OVERRIDE
+   {
+      return wrapped->set_seekpos(seekpos);
+   }
+   virtual bool InternalOpen(int const iFd, unsigned int const Mode) APT_OVERRIDE
+   {
+      if (InternalFlush() == false)
+        return false;
+      return wrapped->InternalOpen(iFd, Mode);
+   }
+   virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) APT_OVERRIDE
+   {
+      if (InternalFlush() == false)
+        return -1;
+      return wrapped->InternalUnbufferedRead(To, Size);
+
+   }
+   virtual bool InternalReadError() APT_OVERRIDE
+   {
+      return wrapped->InternalReadError();
+   }
+   virtual char * InternalReadLine(char * To, unsigned long long Size) APT_OVERRIDE
+   {
+      if (InternalFlush() == false)
+        return nullptr;
+      return wrapped->InternalReadLine(To, Size);
+   }
+   virtual bool InternalFlush() APT_OVERRIDE
+   {
+      while (writebuffer.empty() == false) {
+        auto written = wrapped->InternalWrite(writebuffer.get(),
+                                              writebuffer.size());
+        // Ignore interrupted syscalls
+        if (written < 0 && errno == EINTR)
+           continue;
+        if (written < 0)
+           return wrapped->InternalWriteError();
+
+        writebuffer.bufferstart += written;
+      }
+      writebuffer.reset();
+      return wrapped->InternalFlush();
+   }
+   virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) APT_OVERRIDE
+   {
+      // Optimisation: If the buffer is empty and we have more to write than
+      // would fit in the buffer (or equal number of bytes), write directly.
+      if (writebuffer.empty() == true && Size >= writebuffer.free())
+        return wrapped->InternalWrite(From, Size);
+
+      // Write as much into the buffer as possible and then flush if needed
+      auto written = writebuffer.write(From, Size);
+
+      if (writebuffer.full() && InternalFlush() == false)
+        return -1;
+
+      return written;
+   }
+   virtual bool InternalWriteError() APT_OVERRIDE
+   {
+      return wrapped->InternalWriteError();
+   }
+   virtual bool InternalSeek(unsigned long long const To) APT_OVERRIDE
+   {
+      if (InternalFlush() == false)
+        return false;
+      return wrapped->InternalSeek(To);
+   }
+   virtual bool InternalSkip(unsigned long long Over) APT_OVERRIDE
+   {
+      if (InternalFlush() == false)
+        return false;
+      return wrapped->InternalSkip(Over);
+   }
+   virtual bool InternalTruncate(unsigned long long const Size) APT_OVERRIDE
+   {
+      if (InternalFlush() == false)
+        return false;
+      return wrapped->InternalTruncate(Size);
+   }
+   virtual unsigned long long InternalTell() APT_OVERRIDE
+   {
+      if (InternalFlush() == false)
+        return -1;
+      return wrapped->InternalTell();
+   }
+   virtual unsigned long long InternalSize() APT_OVERRIDE
+   {
+      if (InternalFlush() == false)
+        return -1;
+      return wrapped->InternalSize();
+   }
+   virtual bool InternalClose(std::string const &FileName) APT_OVERRIDE
+   {
+      return wrapped->InternalClose(FileName);
+   }
+   virtual bool InternalAlwaysAutoClose() const APT_OVERRIDE
+   {
+      return wrapped->InternalAlwaysAutoClose();
+   }
+   virtual ~BufferedWriteFileFdPrivate()
+   {
+      delete wrapped;
+   }
+};
+                                                                       /*}}}*/
+class APT_HIDDEN GzipFileFdPrivate: public FileFdPrivate {                             /*{{{*/

 #ifdef HAVE_ZLIB
 public:
    gzFile gz;
 #ifdef HAVE_ZLIB
 public:
    gzFile gz;

-   virtual bool InternalOpen(int const iFd, unsigned int const Mode) override
+   virtual bool InternalOpen(int const iFd, unsigned int const Mode) APT_OVERRIDE

    {
       if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite)
         gz = gzdopen(iFd, "r+");
    {
       if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite)
         gz = gzdopen(iFd, "r+");


@@ -1058,11 +1358,11 @@ public:
       filefd->Flags |= FileFd::Compressed;
       return gz != nullptr;
    }
       filefd->Flags |= FileFd::Compressed;
       return gz != nullptr;
    }

-   virtual ssize_t InternalRead(void * const To, unsigned long long const Size) override
+   virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) APT_OVERRIDE

    {
       return gzread(gz, To, Size);
    }
    {
       return gzread(gz, To, Size);
    }

-   virtual bool InternalReadError() override
+   virtual bool InternalReadError() APT_OVERRIDE

    {
       int err;
       char const * const errmsg = gzerror(gz, &err);
    {
       int err;
       char const * const errmsg = gzerror(gz, &err);


@@ -1070,15 +1370,15 @@ public:
         return filefd->FileFdError("gzread: %s (%d: %s)", _("Read error"), err, errmsg);
       return FileFdPrivate::InternalReadError();
    }
         return filefd->FileFdError("gzread: %s (%d: %s)", _("Read error"), err, errmsg);
       return FileFdPrivate::InternalReadError();
    }

-   virtual char * InternalReadLine(char * const To, unsigned long long const Size) override
+   virtual char * InternalReadLine(char * To, unsigned long long Size) APT_OVERRIDE

    {
       return gzgets(gz, To, Size);
    }
    {
       return gzgets(gz, To, Size);
    }

-   virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override
+   virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) APT_OVERRIDE

    {
       return gzwrite(gz,From,Size);
    }
    {
       return gzwrite(gz,From,Size);
    }

-   virtual bool InternalWriteError() override
+   virtual bool InternalWriteError() APT_OVERRIDE

    {
       int err;
       char const * const errmsg = gzerror(gz, &err);
    {
       int err;
       char const * const errmsg = gzerror(gz, &err);


@@ -1086,28 +1386,40 @@ public:
         return filefd->FileFdError("gzwrite: %s (%d: %s)", _("Write error"), err, errmsg);
       return FileFdPrivate::InternalWriteError();
    }
         return filefd->FileFdError("gzwrite: %s (%d: %s)", _("Write error"), err, errmsg);
       return FileFdPrivate::InternalWriteError();
    }

-   virtual bool InternalSeek(unsigned long long const To) override
+   virtual bool InternalSeek(unsigned long long const To) APT_OVERRIDE

    {
       off_t const res = gzseek(gz, To, SEEK_SET);
       if (res != (off_t)To)
         return filefd->FileFdError("Unable to seek to %llu", To);
    {
       off_t const res = gzseek(gz, To, SEEK_SET);
       if (res != (off_t)To)
         return filefd->FileFdError("Unable to seek to %llu", To);

-

       seekpos = To;
       seekpos = To;

+      buffer.reset();

       return true;
    }
       return true;
    }

-   virtual bool InternalSkip(unsigned long long Over) override
+   virtual bool InternalSkip(unsigned long long Over) APT_OVERRIDE

    {
    {

+      if (Over >= buffer.size())
+      {
+        Over -= buffer.size();
+        buffer.reset();
+      }
+      else
+      {
+        buffer.bufferstart += Over;
+        return true;
+      }
+      if (Over == 0)
+        return true;

       off_t const res = gzseek(gz, Over, SEEK_CUR);
       if (res < 0)
         return filefd->FileFdError("Unable to seek ahead %llu",Over);
       seekpos = res;
       return true;
    }
       off_t const res = gzseek(gz, Over, SEEK_CUR);
       if (res < 0)
         return filefd->FileFdError("Unable to seek ahead %llu",Over);
       seekpos = res;
       return true;
    }

-   virtual unsigned long long InternalTell() override
+   virtual unsigned long long InternalTell() APT_OVERRIDE

    {
    {

-      return gztell(gz);
+      return gztell(gz) - buffer.size();

    }
    }

-   virtual unsigned long long InternalSize() override
+   virtual unsigned long long InternalSize() APT_OVERRIDE

    {
       unsigned long long filesize = FileFdPrivate::InternalSize();
       // only check gzsize if we are actually a gzip file, just checking for
    {
       unsigned long long filesize = FileFdPrivate::InternalSize();
       // only check gzsize if we are actually a gzip file, just checking for


@@ -1141,7 +1453,7 @@ public:
       }
       return size;
    }
       }
       return size;
    }

-   virtual bool InternalClose(std::string const &FileName) override
+   virtual bool InternalClose(std::string const &FileName) APT_OVERRIDE

    {
       if (gz == nullptr)
         return true;
    {
       if (gz == nullptr)
         return true;


@@ -1153,16 +1465,16 @@ public:
       return true;
    }
 
       return true;
    }
 

-   GzipFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), gz(nullptr) {}
+   explicit GzipFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), gz(nullptr) {}

    virtual ~GzipFileFdPrivate() { InternalClose(""); }
 #endif
 };
                                                                        /*}}}*/
    virtual ~GzipFileFdPrivate() { InternalClose(""); }
 #endif
 };
                                                                        /*}}}*/

-class Bz2FileFdPrivate: public FileFdPrivate {                         /*{{{*/
+class APT_HIDDEN Bz2FileFdPrivate: public FileFdPrivate {                              /*{{{*/

 #ifdef HAVE_BZ2
    BZFILE* bz2;
 public:
 #ifdef HAVE_BZ2
    BZFILE* bz2;
 public:

-   virtual bool InternalOpen(int const iFd, unsigned int const Mode) override
+   virtual bool InternalOpen(int const iFd, unsigned int const Mode) APT_OVERRIDE

    {
       if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite)
         bz2 = BZ2_bzdopen(iFd, "r+");
    {
       if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite)
         bz2 = BZ2_bzdopen(iFd, "r+");


@@ -1173,11 +1485,11 @@ public:
       filefd->Flags |= FileFd::Compressed;
       return bz2 != nullptr;
    }
       filefd->Flags |= FileFd::Compressed;
       return bz2 != nullptr;
    }

-   virtual ssize_t InternalRead(void * const To, unsigned long long const Size) override
+   virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) APT_OVERRIDE

    {
       return BZ2_bzread(bz2, To, Size);
    }
    {
       return BZ2_bzread(bz2, To, Size);
    }

-   virtual bool InternalReadError() override
+   virtual bool InternalReadError() APT_OVERRIDE

    {
       int err;
       char const * const errmsg = BZ2_bzerror(bz2, &err);
    {
       int err;
       char const * const errmsg = BZ2_bzerror(bz2, &err);


@@ -1185,11 +1497,11 @@ public:
         return filefd->FileFdError("BZ2_bzread: %s %s (%d: %s)", filefd->FileName.c_str(), _("Read error"), err, errmsg);
       return FileFdPrivate::InternalReadError();
    }
         return filefd->FileFdError("BZ2_bzread: %s %s (%d: %s)", filefd->FileName.c_str(), _("Read error"), err, errmsg);
       return FileFdPrivate::InternalReadError();
    }

-   virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override
+   virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) APT_OVERRIDE

    {
       return BZ2_bzwrite(bz2, (void*)From, Size);
    }
    {
       return BZ2_bzwrite(bz2, (void*)From, Size);
    }

-   virtual bool InternalWriteError() override
+   virtual bool InternalWriteError() APT_OVERRIDE

    {
       int err;
       char const * const errmsg = BZ2_bzerror(bz2, &err);
    {
       int err;
       char const * const errmsg = BZ2_bzerror(bz2, &err);


@@ -1197,8 +1509,8 @@ public:
         return filefd->FileFdError("BZ2_bzwrite: %s %s (%d: %s)", filefd->FileName.c_str(), _("Write error"), err, errmsg);
       return FileFdPrivate::InternalWriteError();
    }
         return filefd->FileFdError("BZ2_bzwrite: %s %s (%d: %s)", filefd->FileName.c_str(), _("Write error"), err, errmsg);
       return FileFdPrivate::InternalWriteError();
    }

-   virtual bool InternalStream() const override { return true; }
-   virtual bool InternalClose(std::string const &) override
+   virtual bool InternalStream() const APT_OVERRIDE { return true; }
+   virtual bool InternalClose(std::string const &) APT_OVERRIDE

    {
       if (bz2 == nullptr)
         return true;
    {
       if (bz2 == nullptr)
         return true;


@@ -1207,25 +1519,185 @@ public:
       return true;
    }
 
       return true;
    }
 

-   Bz2FileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), bz2(nullptr) {}
+   explicit Bz2FileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), bz2(nullptr) {}

    virtual ~Bz2FileFdPrivate() { InternalClose(""); }
 #endif
 };
                                                                        /*}}}*/
    virtual ~Bz2FileFdPrivate() { InternalClose(""); }
 #endif
 };
                                                                        /*}}}*/

-class LzmaFileFdPrivate: public FileFdPrivate {                                /*{{{*/
+class APT_HIDDEN Lz4FileFdPrivate: public FileFdPrivate {                              /*{{{*/
+   static constexpr unsigned long long LZ4_HEADER_SIZE = 19;
+   static constexpr unsigned long long LZ4_FOOTER_SIZE = 4;
+#ifdef HAVE_LZ4
+   LZ4F_decompressionContext_t dctx;
+   LZ4F_compressionContext_t cctx;
+   LZ4F_errorCode_t res;
+   FileFd backend;
+   simple_buffer lz4_buffer;
+   // Count of bytes that the decompressor expects to read next, or buffer size.
+   size_t next_to_load = APT_BUFFER_SIZE;
+public:
+   virtual bool InternalOpen(int const iFd, unsigned int const Mode) APT_OVERRIDE
+   {
+      if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite)
+        return _error->Error("lz4 only supports write or read mode");
+
+      if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly) {
+        res = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION);
+        lz4_buffer.reset(LZ4F_compressBound(APT_BUFFER_SIZE, nullptr)
+                         + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE);
+      } else {
+        res = LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION);
+        lz4_buffer.reset(APT_BUFFER_SIZE);
+      }
+
+      filefd->Flags |= FileFd::Compressed;
+
+      if (LZ4F_isError(res))
+        return false;
+
+      unsigned int flags = (Mode & (FileFd::WriteOnly|FileFd::ReadOnly));
+      if (backend.OpenDescriptor(iFd, flags, FileFd::None, true) == false)
+        return false;
+
+      // Write the file header
+      if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly)
+      {
+        res = LZ4F_compressBegin(cctx, lz4_buffer.buffer, lz4_buffer.buffersize_max, nullptr);
+        if (LZ4F_isError(res) || backend.Write(lz4_buffer.buffer, res) == false)
+           return false;
+      }
+
+      return true;
+   }
+   virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) APT_OVERRIDE
+   {
+      /* Keep reading as long as the compressor still wants to read */
+      while (next_to_load) {
+        // Fill compressed buffer;
+        if (lz4_buffer.empty()) {
+           unsigned long long read;
+           /* Reset - if LZ4 decompressor wants to read more, allocate more */
+           lz4_buffer.reset(next_to_load);
+           if (backend.Read(lz4_buffer.getend(), lz4_buffer.free(), &read) == false)
+              return -1;
+           lz4_buffer.bufferend += read;
+
+           /* Expected EOF */
+           if (read == 0) {
+              res = -1;
+              return filefd->FileFdError("LZ4F: %s %s",
+                                         filefd->FileName.c_str(),
+                                         _("Unexpected end of file")), -1;
+           }
+        }
+        // Drain compressed buffer as far as possible.
+        size_t in = lz4_buffer.size();
+        size_t out = Size;
+
+        res = LZ4F_decompress(dctx, To, &out, lz4_buffer.get(), &in, nullptr);
+        if (LZ4F_isError(res))
+              return -1;
+
+        next_to_load = res;
+        lz4_buffer.bufferstart += in;
+
+        if (out != 0)
+           return out;
+      }
+
+      return 0;
+   }
+   virtual bool InternalReadError() APT_OVERRIDE
+   {
+      char const * const errmsg = LZ4F_getErrorName(res);
+
+      return filefd->FileFdError("LZ4F: %s %s (%zu: %s)", filefd->FileName.c_str(), _("Read error"), res, errmsg);
+   }
+   virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) APT_OVERRIDE
+   {
+      unsigned long long const towrite = std::min(APT_BUFFER_SIZE, Size);
+
+      res = LZ4F_compressUpdate(cctx,
+                               lz4_buffer.buffer, lz4_buffer.buffersize_max,
+                               From, towrite, nullptr);
+
+      if (LZ4F_isError(res) || backend.Write(lz4_buffer.buffer, res) == false)
+        return -1;
+
+      return towrite;
+   }
+   virtual bool InternalWriteError() APT_OVERRIDE
+   {
+      char const * const errmsg = LZ4F_getErrorName(res);
+
+      return filefd->FileFdError("LZ4F: %s %s (%zu: %s)", filefd->FileName.c_str(), _("Write error"), res, errmsg);
+   }
+   virtual bool InternalStream() const APT_OVERRIDE { return true; }
+
+   virtual bool InternalFlush() APT_OVERRIDE
+   {
+      return backend.Flush();
+   }
+
+   virtual bool InternalClose(std::string const &) APT_OVERRIDE
+   {
+      /* Reset variables */
+      res = 0;
+      next_to_load = APT_BUFFER_SIZE;
+
+      if (cctx != nullptr)
+      {
+        if (filefd->Failed() == false)
+        {
+           res = LZ4F_compressEnd(cctx, lz4_buffer.buffer, lz4_buffer.buffersize_max, nullptr);
+           if (LZ4F_isError(res) || backend.Write(lz4_buffer.buffer, res) == false)
+              return false;
+           if (!backend.Flush())
+              return false;
+        }
+        if (!backend.Close())
+           return false;
+
+        res = LZ4F_freeCompressionContext(cctx);
+        cctx = nullptr;
+      }
+
+      if (dctx != nullptr)
+      {
+        res = LZ4F_freeDecompressionContext(dctx);
+        dctx = nullptr;
+      }
+      if (backend.IsOpen())
+      {
+        backend.Close();
+        filefd->iFd = -1;
+      }
+
+      return LZ4F_isError(res) == false;
+   }
+
+   explicit Lz4FileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), dctx(nullptr), cctx(nullptr) {}
+   virtual ~Lz4FileFdPrivate() {
+      InternalClose("");
+   }
+#endif
+};
+                                                                       /*}}}*/
+class APT_HIDDEN LzmaFileFdPrivate: public FileFdPrivate {                             /*{{{*/

 #ifdef HAVE_LZMA
    struct LZMAFILE {
       FILE* file;
 #ifdef HAVE_LZMA
    struct LZMAFILE {
       FILE* file;

+      FileFd * const filefd;

       uint8_t buffer[4096];
       lzma_stream stream;
       lzma_ret err;
       bool eof;
       bool compressing;
 
       uint8_t buffer[4096];
       lzma_stream stream;
       lzma_ret err;
       bool eof;
       bool compressing;
 

-      LZMAFILE() : file(nullptr), eof(false), compressing(false) { buffer[0] = '\0'; }
+      LZMAFILE(FileFd * const fd) : file(nullptr), filefd(fd), eof(false), compressing(false) { buffer[0] = '\0'; }

       ~LZMAFILE()
       {
       ~LZMAFILE()
       {

-        if (compressing == true)
+        if (compressing == true && filefd->Failed() == false)

         {
            size_t constexpr buffersize = sizeof(buffer)/sizeof(buffer[0]);
            while(true)
         {
            size_t constexpr buffersize = sizeof(buffer)/sizeof(buffer[0]);
            while(true)


@@ -1253,14 +1725,40 @@ class LzmaFileFdPrivate: public FileFdPrivate {                         /*{{{*/
       }
    };
    LZMAFILE* lzma;
       }
    };
    LZMAFILE* lzma;

+   static uint32_t findXZlevel(std::vector<std::string> const &Args)
+   {
+      for (auto a = Args.rbegin(); a != Args.rend(); ++a)
+        if (a->empty() == false && (*a)[0] == '-' && (*a)[1] != '-')
+        {
+           auto const number = a->find_last_of("0123456789");
+           if (number == std::string::npos)
+              continue;
+           auto const extreme = a->find("e", number);
+           uint32_t level = (extreme != std::string::npos) ? LZMA_PRESET_EXTREME : 0;
+           switch ((*a)[number])
+           {
+              case '0': return level | 0;
+              case '1': return level | 1;
+              case '2': return level | 2;
+              case '3': return level | 3;
+              case '4': return level | 4;
+              case '5': return level | 5;
+              case '6': return level | 6;
+              case '7': return level | 7;
+              case '8': return level | 8;
+              case '9': return level | 9;
+           }
+        }
+      return 6;
+   }

 public:
 public:

-   virtual bool InternalOpen(int const iFd, unsigned int const Mode) override
+   virtual bool InternalOpen(int const iFd, unsigned int const Mode) APT_OVERRIDE

    {
       if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite)
         return filefd->FileFdError("ReadWrite mode is not supported for lzma/xz files %s", filefd->FileName.c_str());
 
       if (lzma == nullptr)
    {
       if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite)
         return filefd->FileFdError("ReadWrite mode is not supported for lzma/xz files %s", filefd->FileName.c_str());
 
       if (lzma == nullptr)

-        lzma = new LzmaFileFdPrivate::LZMAFILE;
+        lzma = new LzmaFileFdPrivate::LZMAFILE(filefd);

       if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly)
         lzma->file = fdopen(iFd, "w");
       else
       if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly)
         lzma->file = fdopen(iFd, "w");
       else


@@ -1269,13 +1767,12 @@ public:
       if (lzma->file == nullptr)
         return false;
 
       if (lzma->file == nullptr)
         return false;
 

-      uint32_t const xzlevel = 6;
-      uint64_t const memlimit = UINT64_MAX;

       lzma_stream tmp_stream = LZMA_STREAM_INIT;
       lzma->stream = tmp_stream;
 
       if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly)
       {
       lzma_stream tmp_stream = LZMA_STREAM_INIT;
       lzma->stream = tmp_stream;
 
       if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly)
       {

+        uint32_t const xzlevel = findXZlevel(compressor.CompressArgs);

         if (compressor.Name == "xz")
         {
            if (lzma_easy_encoder(&lzma->stream, xzlevel, LZMA_CHECK_CRC64) != LZMA_OK)
         if (compressor.Name == "xz")
         {
            if (lzma_easy_encoder(&lzma->stream, xzlevel, LZMA_CHECK_CRC64) != LZMA_OK)


@@ -1292,6 +1789,7 @@ public:
       }
       else
       {
       }
       else
       {

+        uint64_t const memlimit = UINT64_MAX;

         if (compressor.Name == "xz")
         {
            if (lzma_auto_decoder(&lzma->stream, memlimit, 0) != LZMA_OK)
         if (compressor.Name == "xz")
         {
            if (lzma_auto_decoder(&lzma->stream, memlimit, 0) != LZMA_OK)


@@ -1306,7 +1804,7 @@ public:
       }
       return true;
    }
       }
       return true;
    }

-   virtual ssize_t InternalRead(void * const To, unsigned long long const Size) override
+   virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) APT_OVERRIDE

    {
       ssize_t Res;
       if (lzma->eof == true)
    {
       ssize_t Res;
       if (lzma->eof == true)


@@ -1342,12 +1840,13 @@ public:
       }
       return Res;
    }
       }
       return Res;
    }

-   virtual bool InternalReadError() override
+   virtual bool InternalReadError() APT_OVERRIDE

    {
       return filefd->FileFdError("lzma_read: %s (%d)", _("Read error"), lzma->err);
    }
    {
       return filefd->FileFdError("lzma_read: %s (%d)", _("Read error"), lzma->err);
    }

-   virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override
+   virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) APT_OVERRIDE

    {
    {

+      ssize_t Res;

       lzma->stream.next_in = (uint8_t *)From;
       lzma->stream.avail_in = Size;
       lzma->stream.next_out = lzma->buffer;
       lzma->stream.next_in = (uint8_t *)From;
       lzma->stream.avail_in = Size;
       lzma->stream.next_out = lzma->buffer;


@@ -1358,33 +1857,45 @@ public:
       size_t const n = sizeof(lzma->buffer)/sizeof(lzma->buffer[0]) - lzma->stream.avail_out;
       size_t const m = (n == 0) ? 0 : fwrite(lzma->buffer, 1, n, lzma->file);
       if (m != n)
       size_t const n = sizeof(lzma->buffer)/sizeof(lzma->buffer[0]) - lzma->stream.avail_out;
       size_t const m = (n == 0) ? 0 : fwrite(lzma->buffer, 1, n, lzma->file);
       if (m != n)

-        return -1;
+      {
+        Res = -1;
+        errno = 0;
+      }

       else
       else

-        return Size - lzma->stream.avail_in;
+      {
+        Res = Size - lzma->stream.avail_in;
+        if (Res == 0)
+        {
+           // lzma run was okay, but produced no output…
+           Res = -1;
+           errno = EINTR;
+        }
+      }
+      return Res;

    }
    }

-   virtual bool InternalWriteError() override
+   virtual bool InternalWriteError() APT_OVERRIDE

    {
       return filefd->FileFdError("lzma_write: %s (%d)", _("Write error"), lzma->err);
    }
    {
       return filefd->FileFdError("lzma_write: %s (%d)", _("Write error"), lzma->err);
    }

-   virtual bool InternalStream() const override { return true; }
-   virtual bool InternalClose(std::string const &) override
+   virtual bool InternalStream() const APT_OVERRIDE { return true; }
+   virtual bool InternalClose(std::string const &) APT_OVERRIDE

    {
       delete lzma;
       lzma = nullptr;
       return true;
    }
 
    {
       delete lzma;
       lzma = nullptr;
       return true;
    }
 

-   LzmaFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), lzma(nullptr) {}
+   explicit LzmaFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), lzma(nullptr) {}

    virtual ~LzmaFileFdPrivate() { InternalClose(""); }
 #endif
 };
                                                                        /*}}}*/
    virtual ~LzmaFileFdPrivate() { InternalClose(""); }
 #endif
 };
                                                                        /*}}}*/

-class PipedFileFdPrivate: public FileFdPrivate                         /*{{{*/
+class APT_HIDDEN PipedFileFdPrivate: public FileFdPrivate                              /*{{{*/

 /* if we don't have a specific class dealing with library calls, we (un)compress
    by executing a specified binary and pipe in/out what we need */
 {
 public:
 /* if we don't have a specific class dealing with library calls, we (un)compress
    by executing a specified binary and pipe in/out what we need */
 {
 public:

-   virtual bool InternalOpen(int const, unsigned int const Mode) override
+   virtual bool InternalOpen(int const, unsigned int const Mode) APT_OVERRIDE

    {
       // collect zombies here in case we reopen
       if (compressor_pid > 0)
    {
       // collect zombies here in case we reopen
       if (compressor_pid > 0)


@@ -1392,13 +1903,17 @@ public:
 
       if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite)
         return filefd->FileFdError("ReadWrite mode is not supported for file %s", filefd->FileName.c_str());
 
       if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite)
         return filefd->FileFdError("ReadWrite mode is not supported for file %s", filefd->FileName.c_str());

+      if (compressor.Binary == "false")
+        return filefd->FileFdError("libapt has inbuilt support for the %s compression,"
+              " but was forced to ignore it in favor of an external binary – which isn't installed.", compressor.Name.c_str());

 
       bool const Comp = (Mode & FileFd::WriteOnly) == FileFd::WriteOnly;
 
       bool const Comp = (Mode & FileFd::WriteOnly) == FileFd::WriteOnly;

-      if (Comp == false)
+      if (Comp == false && filefd->iFd != -1)

       {
         // Handle 'decompression' of empty files
         struct stat Buf;
       {
         // Handle 'decompression' of empty files
         struct stat Buf;

-        fstat(filefd->iFd, &Buf);
+        if (fstat(filefd->iFd, &Buf) != 0)
+           return filefd->FileFdErrno("fstat", "Could not stat fd %d for file %s", filefd->iFd, filefd->FileName.c_str());

         if (Buf.st_size == 0 && S_ISFIFO(Buf.st_mode) == false)
            return true;
 
         if (Buf.st_size == 0 && S_ISFIFO(Buf.st_mode) == false)
            return true;
 


@@ -1419,7 +1934,7 @@ public:
         SetCloseExec(Pipe[J],true);
 
       compressed_fd = filefd->iFd;
         SetCloseExec(Pipe[J],true);
 
       compressed_fd = filefd->iFd;

-      is_pipe = true;
+      set_is_pipe(true);

 
       if (Comp == true)
         filefd->iFd = Pipe[1];
 
       if (Comp == true)
         filefd->iFd = Pipe[1];


@@ -1441,12 +1956,6 @@ public:
               dup2(compressed_fd,STDIN_FILENO);
            dup2(Pipe[1],STDOUT_FILENO);
         }
               dup2(compressed_fd,STDIN_FILENO);
            dup2(Pipe[1],STDOUT_FILENO);
         }

-        int const nullfd = open("/dev/null", O_WRONLY);
-        if (nullfd != -1)
-        {
-           dup2(nullfd,STDERR_FILENO);
-           close(nullfd);
-        }

 
         SetCloseExec(STDOUT_FILENO,false);
         SetCloseExec(STDIN_FILENO,false);
 
         SetCloseExec(STDOUT_FILENO,false);
         SetCloseExec(STDIN_FILENO,false);


@@ -1482,73 +1991,106 @@ public:
 
       return true;
    }
 
       return true;
    }

-   virtual ssize_t InternalRead(void * const To, unsigned long long const Size) override
+   virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) APT_OVERRIDE

    {
       return read(filefd->iFd, To, Size);
    }
    {
       return read(filefd->iFd, To, Size);
    }

-   virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override
+   virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) APT_OVERRIDE

    {
       return write(filefd->iFd, From, Size);
    }
    {
       return write(filefd->iFd, From, Size);
    }

-   virtual bool InternalClose(std::string const &) override
+   virtual bool InternalClose(std::string const &) APT_OVERRIDE

    {
       bool Ret = true;
    {
       bool Ret = true;

+      if (filefd->iFd != -1)
+      {
+        close(filefd->iFd);
+        filefd->iFd = -1;
+      }

       if (compressor_pid > 0)
         Ret &= ExecWait(compressor_pid, "FileFdCompressor", true);
       compressor_pid = -1;
       return Ret;
    }
       if (compressor_pid > 0)
         Ret &= ExecWait(compressor_pid, "FileFdCompressor", true);
       compressor_pid = -1;
       return Ret;
    }

-   PipedFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd) {}
+   explicit PipedFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd) {}

    virtual ~PipedFileFdPrivate() { InternalClose(""); }
 };
                                                                        /*}}}*/
    virtual ~PipedFileFdPrivate() { InternalClose(""); }
 };
                                                                        /*}}}*/

-class DirectFileFdPrivate: public FileFdPrivate                                /*{{{*/
+class APT_HIDDEN DirectFileFdPrivate: public FileFdPrivate                             /*{{{*/

 {
 public:
 {
 public:

-   virtual bool InternalOpen(int const, unsigned int const) override { return true; }
-   virtual ssize_t InternalRead(void * const To, unsigned long long const Size) override
+   virtual bool InternalOpen(int const, unsigned int const) APT_OVERRIDE { return true; }
+   virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) APT_OVERRIDE

    {
       return read(filefd->iFd, To, Size);
    }
    {
       return read(filefd->iFd, To, Size);
    }

-
-   virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override
+   virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) APT_OVERRIDE

    {
    {

+      // files opened read+write are strange and only really "supported" for direct files
+      if (buffer.size() != 0)
+      {
+        lseek(filefd->iFd, -buffer.size(), SEEK_CUR);
+        buffer.reset();
+      }

       return write(filefd->iFd, From, Size);
    }
       return write(filefd->iFd, From, Size);
    }

-   virtual bool InternalSeek(unsigned long long const To) override
+   virtual bool InternalSeek(unsigned long long const To) APT_OVERRIDE

    {
       off_t const res = lseek(filefd->iFd, To, SEEK_SET);
       if (res != (off_t)To)
         return filefd->FileFdError("Unable to seek to %llu", To);
       seekpos = To;
    {
       off_t const res = lseek(filefd->iFd, To, SEEK_SET);
       if (res != (off_t)To)
         return filefd->FileFdError("Unable to seek to %llu", To);
       seekpos = To;

+      buffer.reset();

       return true;
    }
       return true;
    }

-   virtual bool InternalSkip(unsigned long long Over) override
+   virtual bool InternalSkip(unsigned long long Over) APT_OVERRIDE

    {
    {

+      if (Over >= buffer.size())
+      {
+        Over -= buffer.size();
+        buffer.reset();
+      }
+      else
+      {
+        buffer.bufferstart += Over;
+        return true;
+      }
+      if (Over == 0)
+        return true;

       off_t const res = lseek(filefd->iFd, Over, SEEK_CUR);
       if (res < 0)
         return filefd->FileFdError("Unable to seek ahead %llu",Over);
       seekpos = res;
       return true;
    }
       off_t const res = lseek(filefd->iFd, Over, SEEK_CUR);
       if (res < 0)
         return filefd->FileFdError("Unable to seek ahead %llu",Over);
       seekpos = res;
       return true;
    }

-   virtual bool InternalTruncate(unsigned long long const To) override
+   virtual bool InternalTruncate(unsigned long long const To) APT_OVERRIDE

    {
    {

+      if (buffer.size() != 0)
+      {
+        unsigned long long const seekpos = lseek(filefd->iFd, 0, SEEK_CUR);
+        if ((seekpos - buffer.size()) >= To)
+           buffer.reset();
+        else if (seekpos >= To)
+           buffer.bufferend = (To - seekpos) + buffer.bufferstart;
+        else
+           buffer.reset();
+      }

       if (ftruncate(filefd->iFd, To) != 0)
         return filefd->FileFdError("Unable to truncate to %llu",To);
       return true;
    }
       if (ftruncate(filefd->iFd, To) != 0)
         return filefd->FileFdError("Unable to truncate to %llu",To);
       return true;
    }

-   virtual unsigned long long InternalTell() override
+   virtual unsigned long long InternalTell() APT_OVERRIDE

    {
    {

-      return lseek(filefd->iFd,0,SEEK_CUR);
+      return lseek(filefd->iFd,0,SEEK_CUR) - buffer.size();

    }
    }

-   virtual unsigned long long InternalSize() override
+   virtual unsigned long long InternalSize() APT_OVERRIDE

    {
       return filefd->FileSize();
    }
    {
       return filefd->FileSize();
    }

-   virtual bool InternalClose(std::string const &) override { return true; }
-   virtual bool InternalAlwaysAutoClose() const override { return false; }
+   virtual bool InternalClose(std::string const &) APT_OVERRIDE { return true; }
+   virtual bool InternalAlwaysAutoClose() const APT_OVERRIDE { return false; }

 
 

-   DirectFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd) {}
+   explicit DirectFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd) {}

    virtual ~DirectFileFdPrivate() { InternalClose(""); }
 };
                                                                        /*}}}*/
    virtual ~DirectFileFdPrivate() { InternalClose(""); }
 };
                                                                        /*}}}*/


@@ -1630,6 +2172,7 @@ bool FileFd::Open(string FileName,unsigned int const Mode,CompressMode Compress,
       case Bzip2: name = "bzip2"; break;
       case Lzma: name = "lzma"; break;
       case Xz: name = "xz"; break;
       case Bzip2: name = "bzip2"; break;
       case Lzma: name = "lzma"; break;
       case Xz: name = "xz"; break;

+      case Lz4: name = "lz4"; break;

       case Auto:
       case Extension:
         // Unreachable
       case Auto:
       case Extension:
         // Unreachable


@@ -1746,6 +2289,7 @@ bool FileFd::OpenDescriptor(int Fd, unsigned int const Mode, CompressMode Compre
    case Bzip2: name = "bzip2"; break;
    case Lzma: name = "lzma"; break;
    case Xz: name = "xz"; break;
    case Bzip2: name = "bzip2"; break;
    case Lzma: name = "lzma"; break;
    case Xz: name = "xz"; break;

+   case Lz4: name = "lz4"; break;

    case Auto:
    case Extension:
       if (AutoClose == true && Fd != -1)
    case Auto:
    case Extension:
       if (AutoClose == true && Fd != -1)


@@ -1807,14 +2351,20 @@ bool FileFd::OpenInternDescriptor(unsigned int const Mode, APT::Configuration::C
       APT_COMPRESS_INIT("xz", LzmaFileFdPrivate);
       APT_COMPRESS_INIT("lzma", LzmaFileFdPrivate);
 #endif
       APT_COMPRESS_INIT("xz", LzmaFileFdPrivate);
       APT_COMPRESS_INIT("lzma", LzmaFileFdPrivate);
 #endif

+#ifdef HAVE_LZ4
+      APT_COMPRESS_INIT("lz4", Lz4FileFdPrivate);
+#endif

 #undef APT_COMPRESS_INIT
       else if (compressor.Name == "." || compressor.Binary.empty() == true)
         d = new DirectFileFdPrivate(this);
       else
         d = new PipedFileFdPrivate(this);
 
 #undef APT_COMPRESS_INIT
       else if (compressor.Name == "." || compressor.Binary.empty() == true)
         d = new DirectFileFdPrivate(this);
       else
         d = new PipedFileFdPrivate(this);
 

-      d->openmode = Mode;
-      d->compressor = compressor;
+      if (Mode & BufferedWrite)
+        d = new BufferedWriteFileFdPrivate(d);
+
+      d->set_openmode(Mode);
+      d->set_compressor(compressor);

       if ((Flags & AutoClose) != AutoClose && d->InternalAlwaysAutoClose())
       {
         // Need to duplicate fd here or gz/bz2 close for cleanup will close the fd as well
       if ((Flags & AutoClose) != AutoClose && d->InternalAlwaysAutoClose())
       {
         // Need to duplicate fd here or gz/bz2 close for cleanup will close the fd as well


@@ -1846,7 +2396,7 @@ FileFd::~FileFd()
    gracefully. */
 bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual)
 {
    gracefully. */
 bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual)
 {

-   if (d == nullptr)
+   if (d == nullptr || Failed())

       return false;
    ssize_t Res = 1;
    errno = 0;
       return false;
    ssize_t Res = 1;
    errno = 0;


@@ -1872,7 +2422,7 @@ bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual)
       To = (char *)To + Res;
       Size -= Res;
       if (d != NULL)
       To = (char *)To + Res;
       Size -= Res;
       if (d != NULL)

-        d->seekpos += Res;
+        d->set_seekpos(d->get_seekpos() + Res);

       if (Actual != 0)
         *Actual += Res;
    }
       if (Actual != 0)
         *Actual += Res;
    }


@@ -1888,6 +2438,37 @@ bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual)
    }
 
    return FileFdError(_("read, still have %llu to read but none left"), Size);
    }
 
    return FileFdError(_("read, still have %llu to read but none left"), Size);

+}
+bool FileFd::Read(int const Fd, void *To, unsigned long long Size, unsigned long long * const Actual)
+{
+   ssize_t Res = 1;
+   errno = 0;
+   if (Actual != nullptr)
+      *Actual = 0;
+   *static_cast<char *>(To) = '\0';
+   while (Res > 0 && Size > 0)
+   {
+      Res = read(Fd, To, Size);
+      if (Res < 0)
+      {
+        if (errno == EINTR)
+        {
+           Res = 1;
+           errno = 0;
+           continue;
+        }
+        return _error->Errno("read", _("Read error"));
+      }
+      To = static_cast<char *>(To) + Res;
+      Size -= Res;
+      if (Actual != 0)
+        *Actual += Res;
+   }
+   if (Size == 0)
+      return true;
+   if (Actual != nullptr)
+      return true;
+   return _error->Error(_("read, still have %llu to read but none left"), Size);

 }
                                                                        /*}}}*/
 // FileFd::ReadLine - Read a complete line from the file               /*{{{*/
 }
                                                                        /*}}}*/
 // FileFd::ReadLine - Read a complete line from the file               /*{{{*/


@@ -1897,30 +2478,49 @@ bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual)
 char* FileFd::ReadLine(char *To, unsigned long long const Size)
 {
    *To = '\0';
 char* FileFd::ReadLine(char *To, unsigned long long const Size)
 {
    *To = '\0';

-   if (d == nullptr)
+   if (d == nullptr || Failed())

       return nullptr;
    return d->InternalReadLine(To, Size);
 }
                                                                        /*}}}*/
       return nullptr;
    return d->InternalReadLine(To, Size);
 }
                                                                        /*}}}*/

+// FileFd::Flush - Flush the file                                      /*{{{*/
+bool FileFd::Flush()
+{
+   if (Failed())
+      return false;
+   if (d == nullptr)
+      return true;
+
+   return d->InternalFlush();
+}
+                                                                       /*}}}*/

 // FileFd::Write - Write to the file                                   /*{{{*/
 bool FileFd::Write(const void *From,unsigned long long Size)
 {
 // FileFd::Write - Write to the file                                   /*{{{*/
 bool FileFd::Write(const void *From,unsigned long long Size)
 {

-   if (d == nullptr)
+   if (d == nullptr || Failed())

       return false;
    ssize_t Res = 1;
    errno = 0;
    while (Res > 0 && Size > 0)
    {
       Res = d->InternalWrite(From, Size);
       return false;
    ssize_t Res = 1;
    errno = 0;
    while (Res > 0 && Size > 0)
    {
       Res = d->InternalWrite(From, Size);

-      if (Res < 0 && errno == EINTR)
-        continue;
+

       if (Res < 0)
       if (Res < 0)

+      {
+        if (errno == EINTR)
+        {
+           // trick the while-loop into running again
+           Res = 1;
+           errno = 0;
+           continue;
+        }

         return d->InternalWriteError();
         return d->InternalWriteError();

+      }

 
       From = (char const *)From + Res;
       Size -= Res;
       if (d != NULL)
 
       From = (char const *)From + Res;
       Size -= Res;
       if (d != NULL)

-        d->seekpos += Res;
+        d->set_seekpos(d->get_seekpos() + Res);

    }
 
    if (Size == 0)
    }
 
    if (Size == 0)


@@ -1953,7 +2553,7 @@ bool FileFd::Write(int Fd, const void *From, unsigned long long Size)
 // FileFd::Seek - Seek in the file                                     /*{{{*/
 bool FileFd::Seek(unsigned long long To)
 {
 // FileFd::Seek - Seek in the file                                     /*{{{*/
 bool FileFd::Seek(unsigned long long To)
 {

-   if (d == nullptr)
+   if (d == nullptr || Failed())

       return false;
    Flags &= ~HitEof;
    return d->InternalSeek(To);
       return false;
    Flags &= ~HitEof;
    return d->InternalSeek(To);


@@ -1962,7 +2562,7 @@ bool FileFd::Seek(unsigned long long To)
 // FileFd::Skip - Skip over data in the file                           /*{{{*/
 bool FileFd::Skip(unsigned long long Over)
 {
 // FileFd::Skip - Skip over data in the file                           /*{{{*/
 bool FileFd::Skip(unsigned long long Over)
 {

-   if (d == nullptr)
+   if (d == nullptr || Failed())

       return false;
    return d->InternalSkip(Over);
 }
       return false;
    return d->InternalSkip(Over);
 }


@@ -1970,7 +2570,7 @@ bool FileFd::Skip(unsigned long long Over)
 // FileFd::Truncate - Truncate the file                                        /*{{{*/
 bool FileFd::Truncate(unsigned long long To)
 {
 // FileFd::Truncate - Truncate the file                                        /*{{{*/
 bool FileFd::Truncate(unsigned long long To)
 {

-   if (d == nullptr)
+   if (d == nullptr || Failed())

       return false;
    // truncating /dev/null is always successful - as we get an error otherwise
    if (To == 0 && FileName == "/dev/null")
       return false;
    // truncating /dev/null is always successful - as we get an error otherwise
    if (To == 0 && FileName == "/dev/null")


@@ -1983,18 +2583,18 @@ bool FileFd::Truncate(unsigned long long To)
 /* */
 unsigned long long FileFd::Tell()
 {
 /* */
 unsigned long long FileFd::Tell()
 {

-   if (d == nullptr)
+   if (d == nullptr || Failed())

       return false;
    off_t const Res = d->InternalTell();
    if (Res == (off_t)-1)
       FileFdErrno("lseek","Failed to determine the current file position");
       return false;
    off_t const Res = d->InternalTell();
    if (Res == (off_t)-1)
       FileFdErrno("lseek","Failed to determine the current file position");

-   d->seekpos = Res;
+   d->set_seekpos(Res);

    return Res;
 }
                                                                        /*}}}*/
 static bool StatFileFd(char const * const msg, int const iFd, std::string const &FileName, struct stat &Buf, FileFdPrivate * const d) /*{{{*/
 {
    return Res;
 }
                                                                        /*}}}*/
 static bool StatFileFd(char const * const msg, int const iFd, std::string const &FileName, struct stat &Buf, FileFdPrivate * const d) /*{{{*/
 {

-   bool ispipe = (d != NULL && d->is_pipe == true);
+   bool ispipe = (d != NULL && d->get_is_pipe() == true);

    if (ispipe == false)
    {
       if (fstat(iFd,&Buf) != 0)
    if (ispipe == false)
    {
       if (fstat(iFd,&Buf) != 0)


@@ -2011,7 +2611,7 @@ static bool StatFileFd(char const * const msg, int const iFd, std::string const
       // we set it here, too, as we get the info here for free
       // in theory the Open-methods should take care of it already
       if (d != NULL)
       // we set it here, too, as we get the info here for free
       // in theory the Open-methods should take care of it already
       if (d != NULL)

-        d->is_pipe = true;
+        d->set_is_pipe(true);

       if (stat(FileName.c_str(), &Buf) != 0)
         return _error->Errno("fstat", "Unable to determine %s for file %s", msg, FileName.c_str());
    }
       if (stat(FileName.c_str(), &Buf) != 0)
         return _error->Errno("fstat", "Unable to determine %s for file %s", msg, FileName.c_str());
    }


@@ -2046,7 +2646,7 @@ time_t FileFd::ModificationTime()
 unsigned long long FileFd::Size()
 {
    if (d == nullptr)
 unsigned long long FileFd::Size()
 {
    if (d == nullptr)

-      return false;
+      return 0;

    return d->InternalSize();
 }
                                                                        /*}}}*/
    return d->InternalSize();
 }
                                                                        /*}}}*/


@@ -2055,6 +2655,8 @@ unsigned long long FileFd::Size()
 /* */
 bool FileFd::Close()
 {
 /* */
 bool FileFd::Close()
 {

+   if (Failed() == false && Flush() == false)
+      return false;

    if (iFd == -1)
       return true;
 
    if (iFd == -1)
       return true;
 


@@ -2073,7 +2675,7 @@ bool FileFd::Close()
    }
 
    if ((Flags & Replace) == Replace) {
    }
 
    if ((Flags & Replace) == Replace) {

-      if (rename(TemporaryFileName.c_str(), FileName.c_str()) != 0)
+      if (Failed() == false && rename(TemporaryFileName.c_str(), FileName.c_str()) != 0)

         Res &= _error->Errno("rename",_("Problem renaming the file %s to %s"), TemporaryFileName.c_str(), FileName.c_str());
 
       FileName = TemporaryFileName; // for the unlink() below.
         Res &= _error->Errno("rename",_("Problem renaming the file %s to %s"), TemporaryFileName.c_str(), FileName.c_str());
 
       FileName = TemporaryFileName; // for the unlink() below.


@@ -2108,13 +2710,12 @@ bool FileFd::FileFdErrno(const char *Function, const char *Description,...)
    va_list args;
    size_t msgSize = 400;
    int const errsv = errno;
    va_list args;
    size_t msgSize = 400;
    int const errsv = errno;

-   while (true)
-   {
+   bool retry;
+   do {

       va_start(args,Description);
       va_start(args,Description);

-      if (_error->InsertErrno(GlobalError::ERROR, Function, Description, args, errsv, msgSize) == false)
-        break;
+      retry = _error->InsertErrno(GlobalError::ERROR, Function, Description, args, errsv, msgSize);

       va_end(args);
       va_end(args);

-   }
+   } while (retry);

    return false;
 }
                                                                        /*}}}*/
    return false;
 }
                                                                        /*}}}*/


@@ -2123,13 +2724,12 @@ bool FileFd::FileFdError(const char *Description,...) {
    Flags |= Fail;
    va_list args;
    size_t msgSize = 400;
    Flags |= Fail;
    va_list args;
    size_t msgSize = 400;

-   while (true)
-   {
+   bool retry;
+   do {

       va_start(args,Description);
       va_start(args,Description);

-      if (_error->Insert(GlobalError::ERROR, Description, args, msgSize) == false)
-        break;
+      retry = _error->Insert(GlobalError::ERROR, Description, args, msgSize);

       va_end(args);
       va_end(args);

-   }
+   } while (retry);

    return false;
 }
                                                                        /*}}}*/
    return false;
 }
                                                                        /*}}}*/


@@ -2172,9 +2772,9 @@ std::vector<std::string> Glob(std::string const &pattern, int flags)
    return result;
 }
                                                                        /*}}}*/
    return result;
 }
                                                                        /*}}}*/

-std::string GetTempDir()                                               /*{{{*/
+static std::string APT_NONNULL(1) GetTempDirEnv(char const * const env)        /*{{{*/

 {
 {

-   const char *tmpdir = getenv("TMPDIR");
+   const char *tmpdir = getenv(env);

 
 #ifdef P_tmpdir
    if (!tmpdir)
 
 #ifdef P_tmpdir
    if (!tmpdir)


@@ -2190,6 +2790,11 @@ std::string GetTempDir()                                         /*{{{*/
       tmpdir = "/tmp";
 
    return string(tmpdir);
       tmpdir = "/tmp";
 
    return string(tmpdir);

+}
+                                                                       /*}}}*/
+std::string GetTempDir()                                               /*{{{*/
+{
+   return GetTempDirEnv("TMPDIR");

 }
 std::string GetTempDir(std::string const &User)
 {
 }
 std::string GetTempDir(std::string const &User)
 {


@@ -2254,6 +2859,11 @@ bool Rename(std::string From, std::string To)                            /*{{{*/
 }
                                                                        /*}}}*/
 bool Popen(const char* Args[], FileFd &Fd, pid_t &Child, FileFd::OpenMode Mode)/*{{{*/
 }
                                                                        /*}}}*/
 bool Popen(const char* Args[], FileFd &Fd, pid_t &Child, FileFd::OpenMode Mode)/*{{{*/

+{
+   return Popen(Args, Fd, Child, Mode, true);
+}
+                                                                       /*}}}*/
+bool Popen(const char* Args[], FileFd &Fd, pid_t &Child, FileFd::OpenMode Mode, bool CaptureStderr)/*{{{*/

 {
    int fd;
    if (Mode != FileFd::ReadOnly && Mode != FileFd::WriteOnly)
 {
    int fd;
    if (Mode != FileFd::ReadOnly && Mode != FileFd::WriteOnly)


@@ -2285,7 +2895,8 @@ bool Popen(const char* Args[], FileFd &Fd, pid_t &Child, FileFd::OpenMode Mode)/
       if(Mode == FileFd::ReadOnly)
       {
          dup2(fd, 1);
       if(Mode == FileFd::ReadOnly)
       {
          dup2(fd, 1);

-         dup2(fd, 2);
+        if (CaptureStderr == true)
+           dup2(fd, 2);

       } else if(Mode == FileFd::WriteOnly)
          dup2(fd, 0);
 
       } else if(Mode == FileFd::WriteOnly)
          dup2(fd, 0);
 


@@ -2439,6 +3050,32 @@ bool DropPrivileges()                                                    /*{{{*/
         return _error->Error("Could restore a uid to root, privilege dropping did not work");
    }
 
         return _error->Error("Could restore a uid to root, privilege dropping did not work");
    }
 

+   if (_config->FindB("APT::Sandbox::ResetEnvironment", true))
+   {
+      setenv("HOME", pw->pw_dir, 1);
+      setenv("USER", pw->pw_name, 1);
+      setenv("USERNAME", pw->pw_name, 1);
+      setenv("LOGNAME", pw->pw_name, 1);
+      auto const shell = flNotDir(pw->pw_shell);
+      if (shell == "false" || shell == "nologin")
+        setenv("SHELL", "/bin/sh", 1);
+      else
+        setenv("SHELL", pw->pw_shell, 1);
+      auto const apt_setenv_tmp = [](char const * const env) {
+        auto const tmpdir = getenv(env);
+        if (tmpdir != nullptr)
+        {
+           auto const ourtmpdir = GetTempDirEnv(env);
+           if (ourtmpdir != tmpdir)
+              setenv(env, ourtmpdir.c_str(), 1);
+        }
+      };
+      apt_setenv_tmp("TMPDIR");
+      apt_setenv_tmp("TEMPDIR");
+      apt_setenv_tmp("TMP");
+      apt_setenv_tmp("TEMP");
+   }
+

    return true;
 }
                                                                        /*}}}*/
    return true;
 }
                                                                        /*}}}*/