BufferedFileFdPrivate: Make InternalFlush() save against errors

[apt.git] / apt-pkg / contrib / fileutl.cc

diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc
index df409fa36b54e21eeebc1eeef0bb6f51e2b7cb8f..7f3ed673a13144e7563a7c8e136d57371c98415a 100644 (file)
--- a/apt-pkg/contrib/fileutl.cc
+++ b/apt-pkg/contrib/fileutl.cc
@@ -52,6 +52,7 @@
 
 #include <set>
 #include <algorithm>
 
 #include <set>
 #include <algorithm>

+#include <memory>

 
 #ifdef HAVE_ZLIB
        #include <zlib.h>
 
 #ifdef HAVE_ZLIB
        #include <zlib.h>


@@ -157,24 +158,33 @@ bool CopyFile(FileFd &From,FileFd &To)
    if (From.IsOpen() == false || To.IsOpen() == false ||
         From.Failed() == true || To.Failed() == true)
       return false;
    if (From.IsOpen() == false || To.IsOpen() == false ||
         From.Failed() == true || To.Failed() == true)
       return false;

-   
+

    // Buffered copy between fds
    // Buffered copy between fds

-   SPtrArray<unsigned char> Buf = new unsigned char[64000];
-   unsigned long long Size = From.Size();
-   while (Size != 0)
-   {
-      unsigned long long ToRead = Size;
-      if (Size > 64000)
-        ToRead = 64000;
-      
-      if (From.Read(Buf,ToRead) == false || 
-         To.Write(Buf,ToRead) == false)
+   constexpr size_t BufSize = 64000;
+   std::unique_ptr<unsigned char[]> Buf(new unsigned char[BufSize]);
+   unsigned long long ToRead = 0;
+   do {
+      if (From.Read(Buf.get(),BufSize, &ToRead) == false ||
+         To.Write(Buf.get(),ToRead) == false)

         return false;
         return false;

-      
-      Size -= ToRead;
-   }
+   } while (ToRead != 0);
+
+   return true;
+}
+                                                                       /*}}}*/
+bool RemoveFile(char const * const Function, std::string const &FileName)/*{{{*/
+{
+   if (FileName == "/dev/null")
+      return true;
+   errno = 0;
+   if (unlink(FileName.c_str()) != 0)
+   {
+      if (errno == ENOENT)
+        return true;

 
 

-   return true;   
+      return _error->WarningE(Function,_("Problem unlinking the file %s"), FileName.c_str());
+   }
+   return true;

 }
                                                                        /*}}}*/
 // GetLock - Gets a lock file                                          /*{{{*/
 }
                                                                        /*}}}*/
 // GetLock - Gets a lock file                                          /*{{{*/


@@ -670,7 +680,7 @@ string flAbsPath(string File)
    char *p = realpath(File.c_str(), NULL);
    if (p == NULL)
    {
    char *p = realpath(File.c_str(), NULL);
    if (p == NULL)
    {

-      _error->Errno("realpath", "flAbsPath failed");
+      _error->Errno("realpath", "flAbsPath on %s failed", File.c_str());

       return "";
    }
    std::string AbsPath(p);
       return "";
    }
    std::string AbsPath(p);


@@ -800,11 +810,26 @@ pid_t ExecFork(std::set<int> KeepFDs)
       signal(SIGCONT,SIG_DFL);
       signal(SIGTSTP,SIG_DFL);
 
       signal(SIGCONT,SIG_DFL);
       signal(SIGTSTP,SIG_DFL);
 

-      // Close all of our FDs - just in case
-      for (int K = 3; K != sysconf(_SC_OPEN_MAX); K++)
+      DIR *dir = opendir("/proc/self/fd");
+      if (dir != NULL)

       {
       {

-        if(KeepFDs.find(K) == KeepFDs.end())
-           fcntl(K,F_SETFD,FD_CLOEXEC);
+        struct dirent *ent;
+        while ((ent = readdir(dir)))
+        {
+           int fd = atoi(ent->d_name);
+           // If fd > 0, it was a fd number and not . or ..
+           if (fd >= 3 && KeepFDs.find(fd) == KeepFDs.end())
+              fcntl(fd,F_SETFD,FD_CLOEXEC);
+        }
+        closedir(dir);
+      } else {
+        long ScOpenMax = sysconf(_SC_OPEN_MAX);
+        // Close all of our FDs - just in case
+        for (int K = 3; K != ScOpenMax; K++)
+        {
+           if(KeepFDs.find(K) == KeepFDs.end())
+              fcntl(K,F_SETFD,FD_CLOEXEC);
+        }

       }
    }
    
       }
    }
    


@@ -857,11 +882,7 @@ bool ExecWait(pid_t Pid,const char *Name,bool Reap)
    return true;
 }
                                                                        /*}}}*/
    return true;
 }
                                                                        /*}}}*/

-
-
-// StartsWithGPGClearTextSignature - Check if a file is Pgp/GPG clearsigned     /*{{{*/
-// ---------------------------------------------------------------------
-/* */
+// StartsWithGPGClearTextSignature - Check if a file is Pgp/GPG clearsigned    /*{{{*/

 bool StartsWithGPGClearTextSignature(string const &FileName)
 {
    static const char* SIGMSG = "-----BEGIN PGP SIGNED MESSAGE-----\n";
 bool StartsWithGPGClearTextSignature(string const &FileName)
 {
    static const char* SIGMSG = "-----BEGIN PGP SIGNED MESSAGE-----\n";


@@ -877,551 +898,1255 @@ bool StartsWithGPGClearTextSignature(string const &FileName)
 
    return true;
 }
 
    return true;
 }

-
-
-class FileFdPrivate {                                                  /*{{{*/
-       public:
-#ifdef HAVE_ZLIB
-       gzFile gz;
-#endif
-#ifdef HAVE_BZ2
-       BZFILE* bz2;
-#endif
-#ifdef HAVE_LZMA
-       struct LZMAFILE {
-          FILE* file;
-          uint8_t buffer[4096];
-          lzma_stream stream;
-          lzma_ret err;
-          bool eof;
-          bool compressing;
-
-          LZMAFILE() : file(NULL), eof(false), compressing(false) { buffer[0] = '\0'; }
-          ~LZMAFILE() {
-             if (compressing == true)
-             {
-               for (;;) {
-                       stream.avail_out = sizeof(buffer)/sizeof(buffer[0]);
-                       stream.next_out = buffer;
-                       err = lzma_code(&stream, LZMA_FINISH);
-                       if (err != LZMA_OK && err != LZMA_STREAM_END)
-                       {
-                               _error->Error("~LZMAFILE: Compress finalisation failed");
-                               break;
-                       }
-                       size_t const n =  sizeof(buffer)/sizeof(buffer[0]) - stream.avail_out;
-                       if (n && fwrite(buffer, 1, n, file) != n)
-                       {
-                               _error->Errno("~LZMAFILE",_("Write error"));
-                               break;
-                       }
-                       if (err == LZMA_STREAM_END)
-                               break;
-               }
-             }
-             lzma_end(&stream);
-             fclose(file);
-          }
-       };
-       LZMAFILE* lzma;
-#endif
-       int compressed_fd;
-       pid_t compressor_pid;
-       bool pipe;
-       APT::Configuration::Compressor compressor;
-       unsigned int openmode;
-       unsigned long long seekpos;
-       FileFdPrivate() :
-#ifdef HAVE_ZLIB
-                         gz(NULL),
-#endif
-#ifdef HAVE_BZ2
-                         bz2(NULL),
-#endif
-#ifdef HAVE_LZMA
-                         lzma(NULL),
-#endif
-                         compressed_fd(-1), compressor_pid(-1), pipe(false),
-                         openmode(0), seekpos(0) {};
-       bool InternalClose(std::string const &FileName)
-       {
-          if (false)
-             /* dummy so that the rest can be 'else if's */;
-#ifdef HAVE_ZLIB
-          else if (gz != NULL) {
-             int const e = gzclose(gz);
-             gz = NULL;
-             // gzdclose() on empty files always fails with "buffer error" here, ignore that
-             if (e != 0 && e != Z_BUF_ERROR)
-                return _error->Errno("close",_("Problem closing the gzip file %s"), FileName.c_str());
-          }
-#endif
-#ifdef HAVE_BZ2
-          else if (bz2 != NULL) {
-             BZ2_bzclose(bz2);
-             bz2 = NULL;
-          }
-#endif
-#ifdef HAVE_LZMA
-          else if (lzma != NULL) {
-             delete lzma;
-             lzma = NULL;
-          }
-#endif
-          return true;
-       }
-       bool CloseDown(std::string const &FileName)
-       {
-          bool const Res = InternalClose(FileName);
-
-          if (compressor_pid > 0)
-             ExecWait(compressor_pid, "FileFdCompressor", true);
-          compressor_pid = -1;
-
-          return Res;
-       }
-       bool InternalStream() const {
-          return false
-#ifdef HAVE_BZ2
-             || bz2 != NULL
-#endif
-#ifdef HAVE_LZMA
-             || lzma != NULL
-#endif
-             ;
-       }
-
-
-       ~FileFdPrivate() { CloseDown(""); }
-};

                                                                        /*}}}*/
                                                                        /*}}}*/

-// FileFd::Open - Open a file                                          /*{{{*/
-// ---------------------------------------------------------------------
-/* The most commonly used open mode combinations are given with Mode */
-bool FileFd::Open(string FileName,unsigned int const Mode,CompressMode Compress, unsigned long const AccessMode)
+// ChangeOwnerAndPermissionOfFile - set file attributes to requested values /*{{{*/
+bool ChangeOwnerAndPermissionOfFile(char const * const requester, char const * const file, char const * const user, char const * const group, mode_t const mode)

 {
 {

-   if (Mode == ReadOnlyGzip)
-      return Open(FileName, ReadOnly, Gzip, AccessMode);
+   if (strcmp(file, "/dev/null") == 0)
+      return true;
+   bool Res = true;
+   if (getuid() == 0 && strlen(user) != 0 && strlen(group) != 0) // if we aren't root, we can't chown, so don't try it
+   {
+      // ensure the file is owned by root and has good permissions
+      struct passwd const * const pw = getpwnam(user);
+      struct group const * const gr = getgrnam(group);
+      if (pw != NULL && gr != NULL && chown(file, pw->pw_uid, gr->gr_gid) != 0)
+        Res &= _error->WarningE(requester, "chown to %s:%s of file %s failed", user, group, file);
+   }
+   if (chmod(file, mode) != 0)
+      Res &= _error->WarningE(requester, "chmod 0%o of file %s failed", mode, file);
+   return Res;
+}
+                                                                       /*}}}*/

 
 

-   if (Compress == Auto && (Mode & WriteOnly) == WriteOnly)
-      return FileFdError("Autodetection on %s only works in ReadOnly openmode!", FileName.c_str());
+struct APT_HIDDEN simple_buffer {                                                      /*{{{*/
+   static constexpr size_t buffersize_max = 4096;
+   unsigned long long bufferstart = 0;
+   unsigned long long bufferend = 0;
+   char buffer[buffersize_max];
+
+   const char *get() const { return buffer + bufferstart; }
+   char *get() { return buffer + bufferstart; }
+   bool empty() const { return bufferend <= bufferstart; }
+   bool full() const { return bufferend == buffersize_max; }
+   unsigned long long size() const { return bufferend-bufferstart; }
+   void reset() { bufferend = bufferstart = 0; }
+   ssize_t read(void *to, unsigned long long requested_size) APT_MUSTCHECK
+   {
+      if (size() < requested_size)
+        requested_size = size();
+      memcpy(to, buffer + bufferstart, requested_size);
+      bufferstart += requested_size;
+      if (bufferstart == bufferend)
+        bufferstart = bufferend = 0;
+      return requested_size;
+   }
+   ssize_t write(const void *from, unsigned long long requested_size) APT_MUSTCHECK
+   {
+      if (buffersize_max - size() < requested_size)
+        requested_size = buffersize_max - size();
+      memcpy(buffer + bufferend, from, requested_size);
+      bufferend += requested_size;
+      if (bufferstart == bufferend)
+        bufferstart = bufferend = 0;
+      return requested_size;
+   }
+};
+                                                                       /*}}}*/

 
 

-   std::vector<APT::Configuration::Compressor> const compressors = APT::Configuration::getCompressors();
-   std::vector<APT::Configuration::Compressor>::const_iterator compressor = compressors.begin();
-   if (Compress == Auto)
+class APT_HIDDEN FileFdPrivate {                                                       /*{{{*/
+   friend class BufferedWriteFileFdPrivate;
+protected:
+   FileFd * const filefd;
+   simple_buffer buffer;
+   int compressed_fd;
+   pid_t compressor_pid;
+   bool is_pipe;
+   APT::Configuration::Compressor compressor;
+   unsigned int openmode;
+   unsigned long long seekpos;
+public:
+
+   explicit FileFdPrivate(FileFd * const pfilefd) : filefd(pfilefd),
+      compressed_fd(-1), compressor_pid(-1), is_pipe(false),
+      openmode(0), seekpos(0) {};
+   virtual APT::Configuration::Compressor get_compressor() const

    {
    {

-      for (; compressor != compressors.end(); ++compressor)
-      {
-        std::string file = FileName + compressor->Extension;
-        if (FileExists(file) == false)
-           continue;
-        FileName = file;
-        break;
-      }
+      return compressor;

    }
    }

-   else if (Compress == Extension)
+   virtual void set_compressor(APT::Configuration::Compressor const &compressor)

    {
    {

-      std::string::size_type const found = FileName.find_last_of('.');
-      std::string ext;
-      if (found != std::string::npos)
-      {
-        ext = FileName.substr(found);
-        if (ext == ".new" || ext == ".bak")
-        {
-           std::string::size_type const found2 = FileName.find_last_of('.', found - 1);
-           if (found2 != std::string::npos)
-              ext = FileName.substr(found2, found - found2);
-           else
-              ext.clear();
-        }
-      }
-      for (; compressor != compressors.end(); ++compressor)
-        if (ext == compressor->Extension)
-           break;
-      // no matching extension - assume uncompressed (imagine files like 'example.org_Packages')
-      if (compressor == compressors.end())
-        for (compressor = compressors.begin(); compressor != compressors.end(); ++compressor)
-           if (compressor->Name == ".")
-              break;
+      this->compressor = compressor;

    }
    }

-   else
+   virtual unsigned int get_openmode() const

    {
    {

-      std::string name;
-      switch (Compress)
-      {
-      case None: name = "."; break;
-      case Gzip: name = "gzip"; break;
-      case Bzip2: name = "bzip2"; break;
-      case Lzma: name = "lzma"; break;
-      case Xz: name = "xz"; break;
-      case Auto:
-      case Extension:
-        // Unreachable
-        return FileFdError("Opening File %s in None, Auto or Extension should be already handled?!?", FileName.c_str());
-      }
-      for (; compressor != compressors.end(); ++compressor)
-        if (compressor->Name == name)
-           break;
-      if (compressor == compressors.end())
-        return FileFdError("Can't find a configured compressor %s for file %s", name.c_str(), FileName.c_str());
+      return openmode;

    }
    }

-
-   if (compressor == compressors.end())
-      return FileFdError("Can't find a match for specified compressor mode for file %s", FileName.c_str());
-   return Open(FileName, Mode, *compressor, AccessMode);
-}
-bool FileFd::Open(string FileName,unsigned int const Mode,APT::Configuration::Compressor const &compressor, unsigned long const AccessMode)
-{
-   Close();
-   Flags = AutoClose;
-
-   if ((Mode & WriteOnly) != WriteOnly && (Mode & (Atomic | Create | Empty | Exclusive)) != 0)
-      return FileFdError("ReadOnly mode for %s doesn't accept additional flags!", FileName.c_str());
-   if ((Mode & ReadWrite) == 0)
-      return FileFdError("No openmode provided in FileFd::Open for %s", FileName.c_str());
-
-   if ((Mode & Atomic) == Atomic)
+   virtual void set_openmode(unsigned int openmode)

    {
    {

-      Flags |= Replace;
+      this->openmode = openmode;

    }
    }

-   else if ((Mode & (Exclusive | Create)) == (Exclusive | Create))
+   virtual bool get_is_pipe() const

    {
    {

-      // for atomic, this will be done by rename in Close()
-      unlink(FileName.c_str());
+      return is_pipe;

    }
    }

-   if ((Mode & Empty) == Empty)
+   virtual void set_is_pipe(bool is_pipe)

    {
    {

-      struct stat Buf;
-      if (lstat(FileName.c_str(),&Buf) == 0 && S_ISLNK(Buf.st_mode))
-        unlink(FileName.c_str());
+      this->is_pipe = is_pipe;
+   }
+   virtual unsigned long long get_seekpos() const
+   {
+      return seekpos;
+   }
+   virtual void set_seekpos(unsigned long long seekpos)
+   {
+      this->seekpos = seekpos;

    }
 
    }
 

-   int fileflags = 0;
-   #define if_FLAGGED_SET(FLAG, MODE) if ((Mode & FLAG) == FLAG) fileflags |= MODE
-   if_FLAGGED_SET(ReadWrite, O_RDWR);
-   else if_FLAGGED_SET(ReadOnly, O_RDONLY);
-   else if_FLAGGED_SET(WriteOnly, O_WRONLY);
+   virtual bool InternalOpen(int const iFd, unsigned int const Mode) = 0;
+   ssize_t InternalRead(void * To, unsigned long long Size)
+   {
+      // Drain the buffer if needed.
+      if (buffer.empty() == false)
+      {
+        return buffer.read(To, Size);
+      }
+      return InternalUnbufferedRead(To, Size);
+   }
+   virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) = 0;
+   virtual bool InternalReadError() { return filefd->FileFdErrno("read",_("Read error")); }
+   virtual char * InternalReadLine(char * To, unsigned long long Size)
+   {
+      if (unlikely(Size == 0))
+        return nullptr;
+      // Read one byte less than buffer size to have space for trailing 0.
+      --Size;

 
 

-   if_FLAGGED_SET(Create, O_CREAT);
-   if_FLAGGED_SET(Empty, O_TRUNC);
-   if_FLAGGED_SET(Exclusive, O_EXCL);
-   #undef if_FLAGGED_SET
+      char * const InitialTo = To;

 
 

-   if ((Mode & Atomic) == Atomic)
-   {
-      char *name = strdup((FileName + ".XXXXXX").c_str());
+      while (Size > 0) {
+        if (buffer.empty() == true)
+        {
+           buffer.reset();
+           unsigned long long actualread = 0;
+           if (filefd->Read(buffer.get(), buffer.buffersize_max, &actualread) == false)
+              return nullptr;
+           buffer.bufferend = actualread;
+           if (buffer.size() == 0)
+           {
+              if (To == InitialTo)
+                 return nullptr;
+              break;
+           }
+           filefd->Flags &= ~FileFd::HitEof;
+        }

 
 

-      if((iFd = mkstemp(name)) == -1)
+        unsigned long long const OutputSize = std::min(Size, buffer.size());
+        char const * const newline = static_cast<char const * const>(memchr(buffer.get(), '\n', OutputSize));
+        // Read until end of line or up to Size bytes from the buffer.
+        unsigned long long actualread = buffer.read(To,
+                                                    (newline != nullptr)
+                                                    ? (newline - buffer.get()) + 1
+                                                    : OutputSize);
+        To += actualread;
+        Size -= actualread;
+        if (newline != nullptr)
+           break;
+      }
+      *To = '\0';
+      return InitialTo;
+   }
+   virtual bool InternalFlush()
+   {
+      return true;
+   }
+   virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) = 0;
+   virtual bool InternalWriteError() { return filefd->FileFdErrno("write",_("Write error")); }
+   virtual bool InternalSeek(unsigned long long const To)
+   {
+      // Our poor man seeking is costly, so try to avoid it
+      unsigned long long const iseekpos = filefd->Tell();
+      if (iseekpos == To)
+        return true;
+      else if (iseekpos < To)
+        return filefd->Skip(To - iseekpos);
+
+      if ((openmode & FileFd::ReadOnly) != FileFd::ReadOnly)
+        return filefd->FileFdError("Reopen is only implemented for read-only files!");
+      InternalClose(filefd->FileName);
+      if (filefd->iFd != -1)
+        close(filefd->iFd);
+      filefd->iFd = -1;
+      if (filefd->TemporaryFileName.empty() == false)
+        filefd->iFd = open(filefd->TemporaryFileName.c_str(), O_RDONLY);
+      else if (filefd->FileName.empty() == false)
+        filefd->iFd = open(filefd->FileName.c_str(), O_RDONLY);
+      else

       {
       {

-          free(name);
-          return FileFdErrno("mkstemp", "Could not create temporary file for %s", FileName.c_str());
+        if (compressed_fd > 0)
+           if (lseek(compressed_fd, 0, SEEK_SET) != 0)
+              filefd->iFd = compressed_fd;
+        if (filefd->iFd < 0)
+           return filefd->FileFdError("Reopen is not implemented for pipes opened with FileFd::OpenDescriptor()!");

       }
 
       }
 

-      TemporaryFileName = string(name);
-      free(name);
+      if (filefd->OpenInternDescriptor(openmode, compressor) == false)
+        return filefd->FileFdError("Seek on file %s because it couldn't be reopened", filefd->FileName.c_str());

 
 

-      // umask() will always set the umask and return the previous value, so
-      // we first set the umask and then reset it to the old value
-      mode_t const CurrentUmask = umask(0);
-      umask(CurrentUmask);
-      // calculate the actual file permissions (just like open/creat)
-      mode_t const FilePermissions = (AccessMode & ~CurrentUmask);
+      buffer.reset();
+      if (To != 0)
+        return filefd->Skip(To);

 
 

-      if(fchmod(iFd, FilePermissions) == -1)
-          return FileFdErrno("fchmod", "Could not change permissions for temporary file %s", TemporaryFileName.c_str());
+      seekpos = To;
+      return true;

    }
    }

-   else
-      iFd = open(FileName.c_str(), fileflags, AccessMode);
-
-   this->FileName = FileName;
-   if (iFd == -1 || OpenInternDescriptor(Mode, compressor) == false)
+   virtual bool InternalSkip(unsigned long long Over)

    {
    {

-      if (iFd != -1)
+      unsigned long long constexpr buffersize = 1024;
+      char buffer[buffersize];
+      while (Over != 0)

       {
       {

-        close (iFd);
-        iFd = -1;
+        unsigned long long toread = std::min(buffersize, Over);
+        if (filefd->Read(buffer, toread) == false)
+           return filefd->FileFdError("Unable to seek ahead %llu",Over);
+        Over -= toread;

       }
       }

-      return FileFdErrno("open",_("Could not open file %s"), FileName.c_str());
+      return true;
+   }
+   virtual bool InternalTruncate(unsigned long long const)
+   {
+      return filefd->FileFdError("Truncating compressed files is not implemented (%s)", filefd->FileName.c_str());
+   }
+   virtual unsigned long long InternalTell()
+   {
+      // In theory, we could just return seekpos here always instead of
+      // seeking around, but not all users of FileFd use always Seek() and co
+      // so d->seekpos isn't always true and we can just use it as a hint if
+      // we have nothing else, but not always as an authority…
+      return seekpos - buffer.size();

    }
    }

+   virtual unsigned long long InternalSize()
+   {
+      unsigned long long size = 0;
+      unsigned long long const oldSeek = filefd->Tell();
+      unsigned long long constexpr ignoresize = 1024;
+      char ignore[ignoresize];
+      unsigned long long read = 0;
+      do {
+        if (filefd->Read(ignore, ignoresize, &read) == false)
+        {
+           filefd->Seek(oldSeek);
+           return 0;
+        }
+      } while(read != 0);
+      size = filefd->Tell();
+      filefd->Seek(oldSeek);
+      return size;
+   }
+   virtual bool InternalClose(std::string const &FileName) = 0;
+   virtual bool InternalStream() const { return false; }
+   virtual bool InternalAlwaysAutoClose() const { return true; }

 
 

-   SetCloseExec(iFd,true);
-   return true;
-}
+   virtual ~FileFdPrivate() {}
+};

                                                                        /*}}}*/
                                                                        /*}}}*/

-// FileFd::OpenDescriptor - Open a filedescriptor                      /*{{{*/
-// ---------------------------------------------------------------------
-/* */
-bool FileFd::OpenDescriptor(int Fd, unsigned int const Mode, CompressMode Compress, bool AutoClose)
-{
-   std::vector<APT::Configuration::Compressor> const compressors = APT::Configuration::getCompressors();
-   std::vector<APT::Configuration::Compressor>::const_iterator compressor = compressors.begin();
-   std::string name;
+class APT_HIDDEN BufferedWriteFileFdPrivate : public FileFdPrivate {                                                   /*{{{*/
+protected:
+   FileFdPrivate *wrapped;
+   simple_buffer writebuffer;

 
 

-   // compat with the old API
-   if (Mode == ReadOnlyGzip && Compress == None)
-      Compress = Gzip;
+public:

 
 

-   switch (Compress)
+   explicit BufferedWriteFileFdPrivate(FileFdPrivate *Priv) :
+      FileFdPrivate(Priv->filefd), wrapped(Priv) {};
+
+   virtual APT::Configuration::Compressor get_compressor() const override

    {
    {

-   case None: name = "."; break;
-   case Gzip: name = "gzip"; break;
-   case Bzip2: name = "bzip2"; break;
-   case Lzma: name = "lzma"; break;
-   case Xz: name = "xz"; break;
-   case Auto:
-   case Extension:
-      if (AutoClose == true && Fd != -1)
-        close(Fd);
-      return FileFdError("Opening Fd %d in Auto or Extension compression mode is not supported", Fd);
+      return wrapped->get_compressor();

    }
    }

-   for (; compressor != compressors.end(); ++compressor)
-      if (compressor->Name == name)
-        break;
-   if (compressor == compressors.end())
+   virtual void set_compressor(APT::Configuration::Compressor const &compressor)  override

    {
    {

-      if (AutoClose == true && Fd != -1)
-        close(Fd);
-      return FileFdError("Can't find a configured compressor %s for file %s", name.c_str(), FileName.c_str());
+      return wrapped->set_compressor(compressor);

    }
    }

-   return OpenDescriptor(Fd, Mode, *compressor, AutoClose);
-}
-bool FileFd::OpenDescriptor(int Fd, unsigned int const Mode, APT::Configuration::Compressor const &compressor, bool AutoClose)
-{
-   Close();
-   Flags = (AutoClose) ? FileFd::AutoClose : 0;
-   iFd = Fd;
-   this->FileName = "";
-   if (OpenInternDescriptor(Mode, compressor) == false)
+   virtual unsigned int get_openmode() const  override

    {
    {

-      if (iFd != -1 && (
-       (Flags & Compressed) == Compressed ||
-       AutoClose == true))
-      {
-        close (iFd);
-        iFd = -1;
-      }
-      return FileFdError(_("Could not open file descriptor %d"), Fd);
+      return wrapped->get_openmode();

    }
    }

-   return true;
-}
-bool FileFd::OpenInternDescriptor(unsigned int const Mode, APT::Configuration::Compressor const &compressor)
-{
-   if (iFd == -1)
-      return false;
-   if (compressor.Name == "." || compressor.Binary.empty() == true)
+   virtual void set_openmode(unsigned int openmode)  override
+   {
+      return wrapped->set_openmode(openmode);
+   }
+   virtual bool get_is_pipe() const  override
+   {
+      return wrapped->get_is_pipe();
+   }
+   virtual void set_is_pipe(bool is_pipe) override
+   {
+      FileFdPrivate::set_is_pipe(is_pipe);
+      wrapped->set_is_pipe(is_pipe);
+   }
+   virtual unsigned long long get_seekpos() const override
+   {
+      return wrapped->get_seekpos();
+   }
+   virtual void set_seekpos(unsigned long long seekpos) override
+   {
+      return wrapped->set_seekpos(seekpos);
+   }
+   virtual bool InternalOpen(int const iFd, unsigned int const Mode) override
+   {
+      if (InternalFlush() == false)
+        return false;
+      return wrapped->InternalOpen(iFd, Mode);
+   }
+   virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) override
+   {
+      if (InternalFlush() == false)
+        return -1;
+      return wrapped->InternalUnbufferedRead(To, Size);
+
+   }
+   virtual bool InternalReadError() override
+   {
+      return wrapped->InternalReadError();
+   }
+   virtual char * InternalReadLine(char * To, unsigned long long Size) override
+   {
+      if (InternalFlush() == false)
+        return nullptr;
+      return wrapped->InternalReadLine(To, Size);
+   }
+   virtual bool InternalFlush() override
+   {
+      while (writebuffer.empty() == false) {
+        auto written = wrapped->InternalWrite(writebuffer.get(),
+                                              writebuffer.size());
+        // Ignore interrupted syscalls
+        if (written < 0 && errno == EINTR)
+           continue;
+        if (written < 0)
+           return false;
+
+        writebuffer.bufferstart += written;
+      }
+
+      writebuffer.reset();

       return true;
       return true;

+   }
+   virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override
+   {
+      size_t written = 0;
+
+      while (written < Size) {
+        auto buffered = writebuffer.write(static_cast<char const*>(From) + written, Size - written);

 
 

-#if defined HAVE_ZLIB || defined HAVE_BZ2 || defined HAVE_LZMA
-   // the API to open files is similar, so setup to avoid code duplicates later
-   // and while at it ensure that we close before opening (if its a reopen)
-   void* (*compress_open)(int, const char *) = NULL;
-   if (false)
-      /* dummy so that the rest can be 'else if's */;
-#define APT_COMPRESS_INIT(NAME,OPEN) \
-   else if (compressor.Name == NAME) \
-   { \
-      compress_open = (void*(*)(int, const char *)) OPEN; \
-      if (d != NULL) d->InternalClose(FileName); \
+        written += buffered;
+
+        if (writebuffer.full() && InternalFlush() == false)
+              return -1;
+      }
+
+      return written;
+   }
+   virtual bool InternalWriteError()
+   {
+      return wrapped->InternalWriteError();
+   }
+   virtual bool InternalSeek(unsigned long long const To)
+   {
+      if (InternalFlush() == false)
+        return false;
+      return wrapped->InternalSeek(To);

    }
    }

+   virtual bool InternalSkip(unsigned long long Over)
+   {
+      if (InternalFlush() == false)
+        return false;
+      return wrapped->InternalSkip(Over);
+   }
+   virtual bool InternalTruncate(unsigned long long const Size)
+   {
+      if (InternalFlush() == false)
+        return false;
+      return wrapped->InternalTruncate(Size);
+   }
+   virtual unsigned long long InternalTell()
+   {
+      if (InternalFlush() == false)
+        return -1;
+      return wrapped->InternalTell();
+   }
+   virtual unsigned long long InternalSize()
+   {
+      if (InternalFlush() == false)
+        return -1;
+      return wrapped->InternalSize();
+   }
+   virtual bool InternalClose(std::string const &FileName)
+   {
+      return wrapped->InternalClose(FileName);
+   }
+   virtual bool InternalAlwaysAutoClose() const
+   {
+      return wrapped->InternalAlwaysAutoClose();
+   }
+   virtual ~BufferedWriteFileFdPrivate()
+   {
+      delete wrapped;
+   }
+};
+                                                                       /*}}}*/
+class APT_HIDDEN GzipFileFdPrivate: public FileFdPrivate {                             /*{{{*/

 #ifdef HAVE_ZLIB
 #ifdef HAVE_ZLIB

-   APT_COMPRESS_INIT("gzip", gzdopen)
-#endif
-#ifdef HAVE_BZ2
-   APT_COMPRESS_INIT("bzip2", BZ2_bzdopen)
-#endif
-#ifdef HAVE_LZMA
-   APT_COMPRESS_INIT("xz", fdopen)
-   APT_COMPRESS_INIT("lzma", fdopen)
-#endif
-#undef APT_COMPRESS_INIT
-#endif
-
-   if (d == NULL)
+public:
+   gzFile gz;
+   virtual bool InternalOpen(int const iFd, unsigned int const Mode) override
+   {
+      if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite)
+        gz = gzdopen(iFd, "r+");
+      else if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly)
+        gz = gzdopen(iFd, "w");
+      else
+        gz = gzdopen(iFd, "r");
+      filefd->Flags |= FileFd::Compressed;
+      return gz != nullptr;
+   }
+   virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) override
+   {
+      return gzread(gz, To, Size);
+   }
+   virtual bool InternalReadError() override
+   {
+      int err;
+      char const * const errmsg = gzerror(gz, &err);
+      if (err != Z_ERRNO)
+        return filefd->FileFdError("gzread: %s (%d: %s)", _("Read error"), err, errmsg);
+      return FileFdPrivate::InternalReadError();
+   }
+   virtual char * InternalReadLine(char * To, unsigned long long Size) override
+   {
+      return gzgets(gz, To, Size);
+   }
+   virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override
+   {
+      return gzwrite(gz,From,Size);
+   }
+   virtual bool InternalWriteError() override

    {
    {

-      d = new FileFdPrivate();
-      d->openmode = Mode;
-      d->compressor = compressor;
-#if defined HAVE_ZLIB || defined HAVE_BZ2 || defined HAVE_LZMA
-      if ((Flags & AutoClose) != AutoClose && compress_open != NULL)
+      int err;
+      char const * const errmsg = gzerror(gz, &err);
+      if (err != Z_ERRNO)
+        return filefd->FileFdError("gzwrite: %s (%d: %s)", _("Write error"), err, errmsg);
+      return FileFdPrivate::InternalWriteError();
+   }
+   virtual bool InternalSeek(unsigned long long const To) override
+   {
+      off_t const res = gzseek(gz, To, SEEK_SET);
+      if (res != (off_t)To)
+        return filefd->FileFdError("Unable to seek to %llu", To);
+      seekpos = To;
+      buffer.reset();
+      return true;
+   }
+   virtual bool InternalSkip(unsigned long long Over) override
+   {
+      if (Over >= buffer.size())

       {
       {

-        // Need to duplicate fd here or gz/bz2 close for cleanup will close the fd as well
-        int const internFd = dup(iFd);
-        if (internFd == -1)
-           return FileFdErrno("OpenInternDescriptor", _("Could not open file descriptor %d"), iFd);
-        iFd = internFd;
+        Over -= buffer.size();
+        buffer.reset();

       }
       }

-#endif
+      else
+      {
+        buffer.bufferstart += Over;
+        return true;
+      }
+      if (Over == 0)
+        return true;
+      off_t const res = gzseek(gz, Over, SEEK_CUR);
+      if (res < 0)
+        return filefd->FileFdError("Unable to seek ahead %llu",Over);
+      seekpos = res;
+      return true;
+   }
+   virtual unsigned long long InternalTell() override
+   {
+      return gztell(gz) - buffer.size();

    }
    }

+   virtual unsigned long long InternalSize() override
+   {
+      unsigned long long filesize = FileFdPrivate::InternalSize();
+      // only check gzsize if we are actually a gzip file, just checking for
+      // "gz" is not sufficient as uncompressed files could be opened with
+      // gzopen in "direct" mode as well
+      if (filesize == 0 || gzdirect(gz))
+        return filesize;
+
+      off_t const oldPos = lseek(filefd->iFd, 0, SEEK_CUR);
+      /* unfortunately zlib.h doesn't provide a gzsize(), so we have to do
+       * this ourselves; the original (uncompressed) file size is the last 32
+       * bits of the file */
+      // FIXME: Size for gz-files is limited by 32bit… no largefile support
+      if (lseek(filefd->iFd, -4, SEEK_END) < 0)
+      {
+        filefd->FileFdErrno("lseek","Unable to seek to end of gzipped file");
+        return 0;
+      }
+      uint32_t size = 0;
+      if (read(filefd->iFd, &size, 4) != 4)
+      {
+        filefd->FileFdErrno("read","Unable to read original size of gzipped file");
+        return 0;
+      }
+      size = le32toh(size);

 
 

-#if defined HAVE_ZLIB || defined HAVE_BZ2 || defined HAVE_LZMA
-   if (compress_open != NULL)
+      if (lseek(filefd->iFd, oldPos, SEEK_SET) < 0)
+      {
+        filefd->FileFdErrno("lseek","Unable to seek in gzipped file");
+        return 0;
+      }
+      return size;
+   }
+   virtual bool InternalClose(std::string const &FileName) override

    {
    {

-      void* compress_struct = NULL;
-      if ((Mode & ReadWrite) == ReadWrite)
-        compress_struct = compress_open(iFd, "r+");
-      else if ((Mode & WriteOnly) == WriteOnly)
-        compress_struct = compress_open(iFd, "w");
-      else
-        compress_struct = compress_open(iFd, "r");
-      if (compress_struct == NULL)
-        return false;
+      if (gz == nullptr)
+        return true;
+      int const e = gzclose(gz);
+      gz = nullptr;
+      // gzdclose() on empty files always fails with "buffer error" here, ignore that
+      if (e != 0 && e != Z_BUF_ERROR)
+        return _error->Errno("close",_("Problem closing the gzip file %s"), FileName.c_str());
+      return true;
+   }

 
 

-      if (false)
-        /* dummy so that the rest can be 'else if's */;
-#ifdef HAVE_ZLIB
-      else if (compressor.Name == "gzip")
-        d->gz = (gzFile) compress_struct;
+   explicit GzipFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), gz(nullptr) {}
+   virtual ~GzipFileFdPrivate() { InternalClose(""); }

 #endif
 #endif

+};
+                                                                       /*}}}*/
+class APT_HIDDEN Bz2FileFdPrivate: public FileFdPrivate {                              /*{{{*/

 #ifdef HAVE_BZ2
 #ifdef HAVE_BZ2

-      else if (compressor.Name == "bzip2")
-        d->bz2 = (BZFILE*) compress_struct;
+   BZFILE* bz2;
+public:
+   virtual bool InternalOpen(int const iFd, unsigned int const Mode) override
+   {
+      if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite)
+        bz2 = BZ2_bzdopen(iFd, "r+");
+      else if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly)
+        bz2 = BZ2_bzdopen(iFd, "w");
+      else
+        bz2 = BZ2_bzdopen(iFd, "r");
+      filefd->Flags |= FileFd::Compressed;
+      return bz2 != nullptr;
+   }
+   virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) override
+   {
+      return BZ2_bzread(bz2, To, Size);
+   }
+   virtual bool InternalReadError() override
+   {
+      int err;
+      char const * const errmsg = BZ2_bzerror(bz2, &err);
+      if (err != BZ_IO_ERROR)
+        return filefd->FileFdError("BZ2_bzread: %s %s (%d: %s)", filefd->FileName.c_str(), _("Read error"), err, errmsg);
+      return FileFdPrivate::InternalReadError();
+   }
+   virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override
+   {
+      return BZ2_bzwrite(bz2, (void*)From, Size);
+   }
+   virtual bool InternalWriteError() override
+   {
+      int err;
+      char const * const errmsg = BZ2_bzerror(bz2, &err);
+      if (err != BZ_IO_ERROR)
+        return filefd->FileFdError("BZ2_bzwrite: %s %s (%d: %s)", filefd->FileName.c_str(), _("Write error"), err, errmsg);
+      return FileFdPrivate::InternalWriteError();
+   }
+   virtual bool InternalStream() const override { return true; }
+   virtual bool InternalClose(std::string const &) override
+   {
+      if (bz2 == nullptr)
+        return true;
+      BZ2_bzclose(bz2);
+      bz2 = nullptr;
+      return true;
+   }
+
+   explicit Bz2FileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), bz2(nullptr) {}
+   virtual ~Bz2FileFdPrivate() { InternalClose(""); }

 #endif
 #endif

+};
+                                                                       /*}}}*/
+class APT_HIDDEN LzmaFileFdPrivate: public FileFdPrivate {                             /*{{{*/

 #ifdef HAVE_LZMA
 #ifdef HAVE_LZMA

-      else if (compressor.Name == "xz" || compressor.Name == "lzma")
+   struct LZMAFILE {
+      FILE* file;
+      uint8_t buffer[4096];
+      lzma_stream stream;
+      lzma_ret err;
+      bool eof;
+      bool compressing;
+
+      LZMAFILE() : file(nullptr), eof(false), compressing(false) { buffer[0] = '\0'; }
+      ~LZMAFILE()

       {
       {

-        uint32_t const xzlevel = 6;
-        uint64_t const memlimit = UINT64_MAX;
-        if (d->lzma == NULL)
-           d->lzma = new FileFdPrivate::LZMAFILE;
-        d->lzma->file = (FILE*) compress_struct;
-         lzma_stream tmp_stream = LZMA_STREAM_INIT;
-        d->lzma->stream = tmp_stream;
-
-        if ((Mode & ReadWrite) == ReadWrite)
-           return FileFdError("ReadWrite mode is not supported for file %s", FileName.c_str());
-
-        if ((Mode & WriteOnly) == WriteOnly)
+        if (compressing == true)

         {
         {

-           if (compressor.Name == "xz")
+           size_t constexpr buffersize = sizeof(buffer)/sizeof(buffer[0]);
+           while(true)

            {
            {

-              if (lzma_easy_encoder(&d->lzma->stream, xzlevel, LZMA_CHECK_CRC32) != LZMA_OK)
-                 return false;
+              stream.avail_out = buffersize;
+              stream.next_out = buffer;
+              err = lzma_code(&stream, LZMA_FINISH);
+              if (err != LZMA_OK && err != LZMA_STREAM_END)
+              {
+                 _error->Error("~LZMAFILE: Compress finalisation failed");
+                 break;
+              }
+              size_t const n =  buffersize - stream.avail_out;
+              if (n && fwrite(buffer, 1, n, file) != n)
+              {
+                 _error->Errno("~LZMAFILE",_("Write error"));
+                 break;
+              }
+              if (err == LZMA_STREAM_END)
+                 break;

            }
            }

-           else
+        }
+        lzma_end(&stream);
+        fclose(file);
+      }
+   };
+   LZMAFILE* lzma;
+   static uint32_t findXZlevel(std::vector<std::string> const &Args)
+   {
+      for (auto a = Args.rbegin(); a != Args.rend(); ++a)
+        if (a->empty() == false && (*a)[0] == '-' && (*a)[1] != '-')
+        {
+           auto const number = a->find_last_of("0123456789");
+           if (number == std::string::npos)
+              continue;
+           auto const extreme = a->find("e", number);
+           uint32_t level = (extreme != std::string::npos) ? LZMA_PRESET_EXTREME : 0;
+           switch ((*a)[number])

            {
            {

-              lzma_options_lzma options;
-              lzma_lzma_preset(&options, xzlevel);
-              if (lzma_alone_encoder(&d->lzma->stream, &options) != LZMA_OK)
-                 return false;
+              case '0': return level | 0;
+              case '1': return level | 1;
+              case '2': return level | 2;
+              case '3': return level | 3;
+              case '4': return level | 4;
+              case '5': return level | 5;
+              case '6': return level | 6;
+              case '7': return level | 7;
+              case '8': return level | 8;
+              case '9': return level | 9;

            }
            }

-           d->lzma->compressing = true;
+        }
+      return 6;
+   }
+public:
+   virtual bool InternalOpen(int const iFd, unsigned int const Mode) override
+   {
+      if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite)
+        return filefd->FileFdError("ReadWrite mode is not supported for lzma/xz files %s", filefd->FileName.c_str());
+
+      if (lzma == nullptr)
+        lzma = new LzmaFileFdPrivate::LZMAFILE;
+      if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly)
+        lzma->file = fdopen(iFd, "w");
+      else
+        lzma->file = fdopen(iFd, "r");
+      filefd->Flags |= FileFd::Compressed;
+      if (lzma->file == nullptr)
+        return false;
+
+      lzma_stream tmp_stream = LZMA_STREAM_INIT;
+      lzma->stream = tmp_stream;
+
+      if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly)
+      {
+        uint32_t const xzlevel = findXZlevel(compressor.CompressArgs);
+        if (compressor.Name == "xz")
+        {
+           if (lzma_easy_encoder(&lzma->stream, xzlevel, LZMA_CHECK_CRC64) != LZMA_OK)
+              return false;

         }
         else
         {
         }
         else
         {

-           if (compressor.Name == "xz")
-           {
-              if (lzma_auto_decoder(&d->lzma->stream, memlimit, 0) != LZMA_OK)
-                 return false;
-           }
-           else
-           {
-              if (lzma_alone_decoder(&d->lzma->stream, memlimit) != LZMA_OK)
-                 return false;
-           }
-           d->lzma->compressing = false;
+           lzma_options_lzma options;
+           lzma_lzma_preset(&options, xzlevel);
+           if (lzma_alone_encoder(&lzma->stream, &options) != LZMA_OK)
+              return false;

         }
         }

+        lzma->compressing = true;
+      }
+      else
+      {
+        uint64_t const memlimit = UINT64_MAX;
+        if (compressor.Name == "xz")
+        {
+           if (lzma_auto_decoder(&lzma->stream, memlimit, 0) != LZMA_OK)
+              return false;
+        }
+        else
+        {
+           if (lzma_alone_decoder(&lzma->stream, memlimit) != LZMA_OK)
+              return false;
+        }
+        lzma->compressing = false;

       }
       }

-#endif
-      Flags |= Compressed;

       return true;
    }
       return true;
    }

+   virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) override
+   {
+      ssize_t Res;
+      if (lzma->eof == true)
+        return 0;
+
+      lzma->stream.next_out = (uint8_t *) To;
+      lzma->stream.avail_out = Size;
+      if (lzma->stream.avail_in == 0)
+      {
+        lzma->stream.next_in = lzma->buffer;
+        lzma->stream.avail_in = fread(lzma->buffer, 1, sizeof(lzma->buffer)/sizeof(lzma->buffer[0]), lzma->file);
+      }
+      lzma->err = lzma_code(&lzma->stream, LZMA_RUN);
+      if (lzma->err == LZMA_STREAM_END)
+      {
+        lzma->eof = true;
+        Res = Size - lzma->stream.avail_out;
+      }
+      else if (lzma->err != LZMA_OK)
+      {
+        Res = -1;
+        errno = 0;
+      }
+      else
+      {
+        Res = Size - lzma->stream.avail_out;
+        if (Res == 0)
+        {
+           // lzma run was okay, but produced no output…
+           Res = -1;
+           errno = EINTR;
+        }
+      }
+      return Res;
+   }
+   virtual bool InternalReadError() override
+   {
+      return filefd->FileFdError("lzma_read: %s (%d)", _("Read error"), lzma->err);
+   }
+   virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override
+   {
+      lzma->stream.next_in = (uint8_t *)From;
+      lzma->stream.avail_in = Size;
+      lzma->stream.next_out = lzma->buffer;
+      lzma->stream.avail_out = sizeof(lzma->buffer)/sizeof(lzma->buffer[0]);
+      lzma->err = lzma_code(&lzma->stream, LZMA_RUN);
+      if (lzma->err != LZMA_OK)
+        return -1;
+      size_t const n = sizeof(lzma->buffer)/sizeof(lzma->buffer[0]) - lzma->stream.avail_out;
+      size_t const m = (n == 0) ? 0 : fwrite(lzma->buffer, 1, n, lzma->file);
+      if (m != n)
+        return -1;
+      else
+        return Size - lzma->stream.avail_in;
+   }
+   virtual bool InternalWriteError() override
+   {
+      return filefd->FileFdError("lzma_write: %s (%d)", _("Write error"), lzma->err);
+   }
+   virtual bool InternalStream() const override { return true; }
+   virtual bool InternalClose(std::string const &) override
+   {
+      delete lzma;
+      lzma = nullptr;
+      return true;
+   }
+
+   explicit LzmaFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), lzma(nullptr) {}
+   virtual ~LzmaFileFdPrivate() { InternalClose(""); }

 #endif
 #endif

+};
+                                                                       /*}}}*/
+class APT_HIDDEN PipedFileFdPrivate: public FileFdPrivate                              /*{{{*/
+/* if we don't have a specific class dealing with library calls, we (un)compress
+   by executing a specified binary and pipe in/out what we need */
+{
+public:
+   virtual bool InternalOpen(int const, unsigned int const Mode) override
+   {
+      // collect zombies here in case we reopen
+      if (compressor_pid > 0)
+        ExecWait(compressor_pid, "FileFdCompressor", true);
+
+      if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite)
+        return filefd->FileFdError("ReadWrite mode is not supported for file %s", filefd->FileName.c_str());

 
 

-   // collect zombies here in case we reopen
-   if (d->compressor_pid > 0)
-      ExecWait(d->compressor_pid, "FileFdCompressor", true);
+      bool const Comp = (Mode & FileFd::WriteOnly) == FileFd::WriteOnly;
+      if (Comp == false)
+      {
+        // Handle 'decompression' of empty files
+        struct stat Buf;
+        fstat(filefd->iFd, &Buf);
+        if (Buf.st_size == 0 && S_ISFIFO(Buf.st_mode) == false)
+           return true;
+
+        // We don't need the file open - instead let the compressor open it
+        // as he properly knows better how to efficiently read from 'his' file
+        if (filefd->FileName.empty() == false)
+        {
+           close(filefd->iFd);
+           filefd->iFd = -1;
+        }
+      }
+
+      // Create a data pipe
+      int Pipe[2] = {-1,-1};
+      if (pipe(Pipe) != 0)
+        return filefd->FileFdErrno("pipe",_("Failed to create subprocess IPC"));
+      for (int J = 0; J != 2; J++)
+        SetCloseExec(Pipe[J],true);
+
+      compressed_fd = filefd->iFd;
+      set_is_pipe(true);
+
+      if (Comp == true)
+        filefd->iFd = Pipe[1];
+      else
+        filefd->iFd = Pipe[0];
+
+      // The child..
+      compressor_pid = ExecFork();
+      if (compressor_pid == 0)
+      {
+        if (Comp == true)
+        {
+           dup2(compressed_fd,STDOUT_FILENO);
+           dup2(Pipe[0],STDIN_FILENO);
+        }
+        else
+        {
+           if (compressed_fd != -1)
+              dup2(compressed_fd,STDIN_FILENO);
+           dup2(Pipe[1],STDOUT_FILENO);
+        }
+        int const nullfd = open("/dev/null", O_WRONLY);
+        if (nullfd != -1)
+        {
+           dup2(nullfd,STDERR_FILENO);
+           close(nullfd);
+        }

 
 

-   if ((Mode & ReadWrite) == ReadWrite)
-      return FileFdError("ReadWrite mode is not supported for file %s", FileName.c_str());
+        SetCloseExec(STDOUT_FILENO,false);
+        SetCloseExec(STDIN_FILENO,false);
+
+        std::vector<char const*> Args;
+        Args.push_back(compressor.Binary.c_str());
+        std::vector<std::string> const * const addArgs =
+           (Comp == true) ? &(compressor.CompressArgs) : &(compressor.UncompressArgs);
+        for (std::vector<std::string>::const_iterator a = addArgs->begin();
+              a != addArgs->end(); ++a)
+           Args.push_back(a->c_str());
+        if (Comp == false && filefd->FileName.empty() == false)
+        {
+           // commands not needing arguments, do not need to be told about using standard output
+           // in reality, only testcases with tools like cat, rev, rot13, … are able to trigger this
+           if (compressor.CompressArgs.empty() == false && compressor.UncompressArgs.empty() == false)
+              Args.push_back("--stdout");
+           if (filefd->TemporaryFileName.empty() == false)
+              Args.push_back(filefd->TemporaryFileName.c_str());
+           else
+              Args.push_back(filefd->FileName.c_str());
+        }
+        Args.push_back(NULL);

 
 

-   bool const Comp = (Mode & WriteOnly) == WriteOnly;
-   if (Comp == false)
+        execvp(Args[0],(char **)&Args[0]);
+        cerr << _("Failed to exec compressor ") << Args[0] << endl;
+        _exit(100);
+      }
+      if (Comp == true)
+        close(Pipe[0]);
+      else
+        close(Pipe[1]);
+
+      return true;
+   }
+   virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) override

    {
    {

-      // Handle 'decompression' of empty files
-      struct stat Buf;
-      fstat(iFd, &Buf);
-      if (Buf.st_size == 0 && S_ISFIFO(Buf.st_mode) == false)
+      return read(filefd->iFd, To, Size);
+   }
+   virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override
+   {
+      return write(filefd->iFd, From, Size);
+   }
+   virtual bool InternalClose(std::string const &) override
+   {
+      bool Ret = true;
+      if (compressor_pid > 0)
+        Ret &= ExecWait(compressor_pid, "FileFdCompressor", true);
+      compressor_pid = -1;
+      return Ret;
+   }
+   explicit PipedFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd) {}
+   virtual ~PipedFileFdPrivate() { InternalClose(""); }
+};
+                                                                       /*}}}*/
+class APT_HIDDEN DirectFileFdPrivate: public FileFdPrivate                             /*{{{*/
+{
+public:
+   virtual bool InternalOpen(int const, unsigned int const) override { return true; }
+   virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) override
+   {
+      return read(filefd->iFd, To, Size);
+   }
+   virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override
+   {
+      // files opened read+write are strange and only really "supported" for direct files
+      if (buffer.size() != 0)
+      {
+        lseek(filefd->iFd, -buffer.size(), SEEK_CUR);
+        buffer.reset();
+      }
+      return write(filefd->iFd, From, Size);
+   }
+   virtual bool InternalSeek(unsigned long long const To) override
+   {
+      off_t const res = lseek(filefd->iFd, To, SEEK_SET);
+      if (res != (off_t)To)
+        return filefd->FileFdError("Unable to seek to %llu", To);
+      seekpos = To;
+      buffer.reset();
+      return true;
+   }
+   virtual bool InternalSkip(unsigned long long Over) override
+   {
+      if (Over >= buffer.size())
+      {
+        Over -= buffer.size();
+        buffer.reset();
+      }
+      else
+      {
+        buffer.bufferstart += Over;
+        return true;
+      }
+      if (Over == 0)

         return true;
         return true;

+      off_t const res = lseek(filefd->iFd, Over, SEEK_CUR);
+      if (res < 0)
+        return filefd->FileFdError("Unable to seek ahead %llu",Over);
+      seekpos = res;
+      return true;
+   }
+   virtual bool InternalTruncate(unsigned long long const To) override
+   {
+      if (buffer.size() != 0)
+      {
+        unsigned long long const seekpos = lseek(filefd->iFd, 0, SEEK_CUR);
+        if ((seekpos - buffer.size()) >= To)
+           buffer.reset();
+        else if (seekpos >= To)
+           buffer.bufferend = (To - seekpos) + buffer.bufferstart;
+        else
+           buffer.reset();
+      }
+      if (ftruncate(filefd->iFd, To) != 0)
+        return filefd->FileFdError("Unable to truncate to %llu",To);
+      return true;
+   }
+   virtual unsigned long long InternalTell() override
+   {
+      return lseek(filefd->iFd,0,SEEK_CUR) - buffer.size();
+   }
+   virtual unsigned long long InternalSize() override
+   {
+      return filefd->FileSize();
+   }
+   virtual bool InternalClose(std::string const &) override { return true; }
+   virtual bool InternalAlwaysAutoClose() const override { return false; }

 
 

-      // We don't need the file open - instead let the compressor open it
-      // as he properly knows better how to efficiently read from 'his' file
-      if (FileName.empty() == false)
+   explicit DirectFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd) {}
+   virtual ~DirectFileFdPrivate() { InternalClose(""); }
+};
+                                                                       /*}}}*/
+// FileFd Constructors                                                 /*{{{*/
+FileFd::FileFd(std::string FileName,unsigned int const Mode,unsigned long AccessMode) : iFd(-1), Flags(0), d(NULL)
+{
+   Open(FileName,Mode, None, AccessMode);
+}
+FileFd::FileFd(std::string FileName,unsigned int const Mode, CompressMode Compress, unsigned long AccessMode) : iFd(-1), Flags(0), d(NULL)
+{
+   Open(FileName,Mode, Compress, AccessMode);
+}
+FileFd::FileFd() : iFd(-1), Flags(AutoClose), d(NULL) {}
+FileFd::FileFd(int const Fd, unsigned int const Mode, CompressMode Compress) : iFd(-1), Flags(0), d(NULL)
+{
+   OpenDescriptor(Fd, Mode, Compress);
+}
+FileFd::FileFd(int const Fd, bool const AutoClose) : iFd(-1), Flags(0), d(NULL)
+{
+   OpenDescriptor(Fd, ReadWrite, None, AutoClose);
+}
+                                                                       /*}}}*/
+// FileFd::Open - Open a file                                          /*{{{*/
+// ---------------------------------------------------------------------
+/* The most commonly used open mode combinations are given with Mode */
+bool FileFd::Open(string FileName,unsigned int const Mode,CompressMode Compress, unsigned long const AccessMode)
+{
+   if (Mode == ReadOnlyGzip)
+      return Open(FileName, ReadOnly, Gzip, AccessMode);
+
+   if (Compress == Auto && (Mode & WriteOnly) == WriteOnly)
+      return FileFdError("Autodetection on %s only works in ReadOnly openmode!", FileName.c_str());
+
+   std::vector<APT::Configuration::Compressor> const compressors = APT::Configuration::getCompressors();
+   std::vector<APT::Configuration::Compressor>::const_iterator compressor = compressors.begin();
+   if (Compress == Auto)
+   {
+      for (; compressor != compressors.end(); ++compressor)
+      {
+        std::string file = FileName + compressor->Extension;
+        if (FileExists(file) == false)
+           continue;
+        FileName = file;
+        break;
+      }
+   }
+   else if (Compress == Extension)
+   {
+      std::string::size_type const found = FileName.find_last_of('.');
+      std::string ext;
+      if (found != std::string::npos)
+      {
+        ext = FileName.substr(found);
+        if (ext == ".new" || ext == ".bak")
+        {
+           std::string::size_type const found2 = FileName.find_last_of('.', found - 1);
+           if (found2 != std::string::npos)
+              ext = FileName.substr(found2, found - found2);
+           else
+              ext.clear();
+        }
+      }
+      for (; compressor != compressors.end(); ++compressor)
+        if (ext == compressor->Extension)
+           break;
+      // no matching extension - assume uncompressed (imagine files like 'example.org_Packages')
+      if (compressor == compressors.end())
+        for (compressor = compressors.begin(); compressor != compressors.end(); ++compressor)
+           if (compressor->Name == ".")
+              break;
+   }
+   else
+   {
+      std::string name;
+      switch (Compress)

       {
       {

-        close(iFd);
+      case None: name = "."; break;
+      case Gzip: name = "gzip"; break;
+      case Bzip2: name = "bzip2"; break;
+      case Lzma: name = "lzma"; break;
+      case Xz: name = "xz"; break;
+      case Auto:
+      case Extension:
+        // Unreachable
+        return FileFdError("Opening File %s in None, Auto or Extension should be already handled?!?", FileName.c_str());
+      }
+      for (; compressor != compressors.end(); ++compressor)
+        if (compressor->Name == name)
+           break;
+      if (compressor == compressors.end())
+        return FileFdError("Can't find a configured compressor %s for file %s", name.c_str(), FileName.c_str());
+   }
+
+   if (compressor == compressors.end())
+      return FileFdError("Can't find a match for specified compressor mode for file %s", FileName.c_str());
+   return Open(FileName, Mode, *compressor, AccessMode);
+}
+bool FileFd::Open(string FileName,unsigned int const Mode,APT::Configuration::Compressor const &compressor, unsigned long const AccessMode)
+{
+   Close();
+   Flags = AutoClose;
+
+   if ((Mode & WriteOnly) != WriteOnly && (Mode & (Atomic | Create | Empty | Exclusive)) != 0)
+      return FileFdError("ReadOnly mode for %s doesn't accept additional flags!", FileName.c_str());
+   if ((Mode & ReadWrite) == 0)
+      return FileFdError("No openmode provided in FileFd::Open for %s", FileName.c_str());
+
+   unsigned int OpenMode = Mode;
+   if (FileName == "/dev/null")
+      OpenMode = OpenMode & ~(Atomic | Exclusive | Create | Empty);
+
+   if ((OpenMode & Atomic) == Atomic)
+   {
+      Flags |= Replace;
+   }
+   else if ((OpenMode & (Exclusive | Create)) == (Exclusive | Create))
+   {
+      // for atomic, this will be done by rename in Close()
+      RemoveFile("FileFd::Open", FileName);
+   }
+   if ((OpenMode & Empty) == Empty)
+   {
+      struct stat Buf;
+      if (lstat(FileName.c_str(),&Buf) == 0 && S_ISLNK(Buf.st_mode))
+        RemoveFile("FileFd::Open", FileName);
+   }
+
+   int fileflags = 0;
+   #define if_FLAGGED_SET(FLAG, MODE) if ((OpenMode & FLAG) == FLAG) fileflags |= MODE
+   if_FLAGGED_SET(ReadWrite, O_RDWR);
+   else if_FLAGGED_SET(ReadOnly, O_RDONLY);
+   else if_FLAGGED_SET(WriteOnly, O_WRONLY);
+
+   if_FLAGGED_SET(Create, O_CREAT);
+   if_FLAGGED_SET(Empty, O_TRUNC);
+   if_FLAGGED_SET(Exclusive, O_EXCL);
+   #undef if_FLAGGED_SET
+
+   if ((OpenMode & Atomic) == Atomic)
+   {
+      char *name = strdup((FileName + ".XXXXXX").c_str());
+
+      if((iFd = mkstemp(name)) == -1)
+      {
+          free(name);
+          return FileFdErrno("mkstemp", "Could not create temporary file for %s", FileName.c_str());
+      }
+
+      TemporaryFileName = string(name);
+      free(name);
+
+      // umask() will always set the umask and return the previous value, so
+      // we first set the umask and then reset it to the old value
+      mode_t const CurrentUmask = umask(0);
+      umask(CurrentUmask);
+      // calculate the actual file permissions (just like open/creat)
+      mode_t const FilePermissions = (AccessMode & ~CurrentUmask);
+
+      if(fchmod(iFd, FilePermissions) == -1)
+          return FileFdErrno("fchmod", "Could not change permissions for temporary file %s", TemporaryFileName.c_str());
+   }
+   else
+      iFd = open(FileName.c_str(), fileflags, AccessMode);
+
+   this->FileName = FileName;
+   if (iFd == -1 || OpenInternDescriptor(OpenMode, compressor) == false)
+   {
+      if (iFd != -1)
+      {
+        close (iFd);

         iFd = -1;
       }
         iFd = -1;
       }

+      return FileFdErrno("open",_("Could not open file %s"), FileName.c_str());

    }
 
    }
 

-   // Create a data pipe
-   int Pipe[2] = {-1,-1};
-   if (pipe(Pipe) != 0)
-      return FileFdErrno("pipe",_("Failed to create subprocess IPC"));
-   for (int J = 0; J != 2; J++)
-      SetCloseExec(Pipe[J],true);
+   SetCloseExec(iFd,true);
+   return true;
+}
+                                                                       /*}}}*/
+// FileFd::OpenDescriptor - Open a filedescriptor                      /*{{{*/
+bool FileFd::OpenDescriptor(int Fd, unsigned int const Mode, CompressMode Compress, bool AutoClose)
+{
+   std::vector<APT::Configuration::Compressor> const compressors = APT::Configuration::getCompressors();
+   std::vector<APT::Configuration::Compressor>::const_iterator compressor = compressors.begin();
+   std::string name;
+
+   // compat with the old API
+   if (Mode == ReadOnlyGzip && Compress == None)
+      Compress = Gzip;

 
 

-   d->compressed_fd = iFd;
-   d->pipe = true;
+   switch (Compress)
+   {
+   case None: name = "."; break;
+   case Gzip: name = "gzip"; break;
+   case Bzip2: name = "bzip2"; break;
+   case Lzma: name = "lzma"; break;
+   case Xz: name = "xz"; break;
+   case Auto:
+   case Extension:
+      if (AutoClose == true && Fd != -1)
+        close(Fd);
+      return FileFdError("Opening Fd %d in Auto or Extension compression mode is not supported", Fd);
+   }
+   for (; compressor != compressors.end(); ++compressor)
+      if (compressor->Name == name)
+        break;
+   if (compressor == compressors.end())
+   {
+      if (AutoClose == true && Fd != -1)
+        close(Fd);
+      return FileFdError("Can't find a configured compressor %s for file %s", name.c_str(), FileName.c_str());
+   }
+   return OpenDescriptor(Fd, Mode, *compressor, AutoClose);
+}
+bool FileFd::OpenDescriptor(int Fd, unsigned int const Mode, APT::Configuration::Compressor const &compressor, bool AutoClose)
+{
+   Close();
+   Flags = (AutoClose) ? FileFd::AutoClose : 0;
+   iFd = Fd;
+   this->FileName = "";
+   if (OpenInternDescriptor(Mode, compressor) == false)
+   {
+      if (iFd != -1 && (
+       (Flags & Compressed) == Compressed ||
+       AutoClose == true))
+      {
+        close (iFd);
+        iFd = -1;
+      }
+      return FileFdError(_("Could not open file descriptor %d"), Fd);
+   }
+   return true;
+}
+bool FileFd::OpenInternDescriptor(unsigned int const Mode, APT::Configuration::Compressor const &compressor)
+{
+   if (iFd == -1)
+      return false;

 
 

-   if (Comp == true)
-      iFd = Pipe[1];
-   else
-      iFd = Pipe[0];
+   if (d != nullptr)
+      d->InternalClose(FileName);

 
 

-   // The child..
-   d->compressor_pid = ExecFork();
-   if (d->compressor_pid == 0)
+   if (d == nullptr)

    {
    {

-      if (Comp == true)
-      {
-        dup2(d->compressed_fd,STDOUT_FILENO);
-        dup2(Pipe[0],STDIN_FILENO);
-      }
+      if (false)
+        /* dummy so that the rest can be 'else if's */;
+#define APT_COMPRESS_INIT(NAME, CONSTRUCTOR) \
+      else if (compressor.Name == NAME) \
+        d = new CONSTRUCTOR(this)
+#ifdef HAVE_ZLIB
+      APT_COMPRESS_INIT("gzip", GzipFileFdPrivate);
+#endif
+#ifdef HAVE_BZ2
+      APT_COMPRESS_INIT("bzip2", Bz2FileFdPrivate);
+#endif
+#ifdef HAVE_LZMA
+      APT_COMPRESS_INIT("xz", LzmaFileFdPrivate);
+      APT_COMPRESS_INIT("lzma", LzmaFileFdPrivate);
+#endif
+#undef APT_COMPRESS_INIT
+      else if (compressor.Name == "." || compressor.Binary.empty() == true)
+        d = new DirectFileFdPrivate(this);

       else
       else

-      {
-        if (d->compressed_fd != -1)
-           dup2(d->compressed_fd,STDIN_FILENO);
-        dup2(Pipe[1],STDOUT_FILENO);
-      }
-      int const nullfd = open("/dev/null", O_WRONLY);
-      if (nullfd != -1)
-      {
-        dup2(nullfd,STDERR_FILENO);
-        close(nullfd);
-      }
+        d = new PipedFileFdPrivate(this);

 
 

-      SetCloseExec(STDOUT_FILENO,false);
-      SetCloseExec(STDIN_FILENO,false);
-
-      std::vector<char const*> Args;
-      Args.push_back(compressor.Binary.c_str());
-      std::vector<std::string> const * const addArgs =
-               (Comp == true) ? &(compressor.CompressArgs) : &(compressor.UncompressArgs);
-      for (std::vector<std::string>::const_iterator a = addArgs->begin();
-          a != addArgs->end(); ++a)
-        Args.push_back(a->c_str());
-      if (Comp == false && FileName.empty() == false)
+      if (Mode & BufferedWrite)
+        d = new BufferedWriteFileFdPrivate(d);
+
+      d->set_openmode(Mode);
+      d->set_compressor(compressor);
+      if ((Flags & AutoClose) != AutoClose && d->InternalAlwaysAutoClose())

       {
       {

-        // commands not needing arguments, do not need to be told about using standard output
-        // in reality, only testcases with tools like cat, rev, rot13, … are able to trigger this
-        if (compressor.CompressArgs.empty() == false && compressor.UncompressArgs.empty() == false)
-           Args.push_back("--stdout");
-        if (TemporaryFileName.empty() == false)
-           Args.push_back(TemporaryFileName.c_str());
-        else
-           Args.push_back(FileName.c_str());
+        // Need to duplicate fd here or gz/bz2 close for cleanup will close the fd as well
+        int const internFd = dup(iFd);
+        if (internFd == -1)
+           return FileFdErrno("OpenInternDescriptor", _("Could not open file descriptor %d"), iFd);
+        iFd = internFd;

       }
       }

-      Args.push_back(NULL);
-
-      execvp(Args[0],(char **)&Args[0]);
-      cerr << _("Failed to exec compressor ") << Args[0] << endl;
-      _exit(100);

    }
    }

-   if (Comp == true)
-      close(Pipe[0]);
-   else
-      close(Pipe[1]);
-
-   return true;
+   return d->InternalOpen(iFd, Mode);

 }
                                                                        /*}}}*/
 // FileFd::~File - Closes the file                                     /*{{{*/
 }
                                                                        /*}}}*/
 // FileFd::~File - Closes the file                                     /*{{{*/


@@ -1432,7 +2157,7 @@ FileFd::~FileFd()
 {
    Close();
    if (d != NULL)
 {
    Close();
    if (d != NULL)

-      d->CloseDown(FileName);
+      d->InternalClose(FileName);

    delete d;
    d = NULL;
 }
    delete d;
    d = NULL;
 }


@@ -1443,61 +2168,16 @@ FileFd::~FileFd()
    gracefully. */
 bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual)
 {
    gracefully. */
 bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual)
 {

-   ssize_t Res;
+   if (d == nullptr)
+      return false;
+   ssize_t Res = 1;

    errno = 0;
    if (Actual != 0)
       *Actual = 0;
    *((char *)To) = '\0';
    errno = 0;
    if (Actual != 0)
       *Actual = 0;
    *((char *)To) = '\0';

-   do
+   while (Res > 0 && Size > 0)

    {
    {

-      if (false)
-        /* dummy so that the rest can be 'else if's */;
-#ifdef HAVE_ZLIB
-      else if (d != NULL && d->gz != NULL)
-        Res = gzread(d->gz,To,Size);
-#endif
-#ifdef HAVE_BZ2
-      else if (d != NULL && d->bz2 != NULL)
-        Res = BZ2_bzread(d->bz2,To,Size);
-#endif
-#ifdef HAVE_LZMA
-      else if (d != NULL && d->lzma != NULL)
-      {
-        if (d->lzma->eof == true)
-           break;
-
-        d->lzma->stream.next_out = (uint8_t *) To;
-        d->lzma->stream.avail_out = Size;
-        if (d->lzma->stream.avail_in == 0)
-        {
-           d->lzma->stream.next_in = d->lzma->buffer;
-           d->lzma->stream.avail_in = fread(d->lzma->buffer, 1, sizeof(d->lzma->buffer)/sizeof(d->lzma->buffer[0]), d->lzma->file);
-        }
-        d->lzma->err = lzma_code(&d->lzma->stream, LZMA_RUN);
-        if (d->lzma->err == LZMA_STREAM_END)
-        {
-           d->lzma->eof = true;
-           Res = Size - d->lzma->stream.avail_out;
-        }
-        else if (d->lzma->err != LZMA_OK)
-        {
-           Res = -1;
-           errno = 0;
-        }
-        else
-        {
-           Res = Size - d->lzma->stream.avail_out;
-           if (Res == 0)
-           {
-              // lzma run was okay, but produced no output…
-              Res = -1;
-              errno = EINTR;
-           }
-        }
-      }
-#endif
-      else
-         Res = read(iFd,To,Size);
+      Res = d->InternalRead(To, Size);

 
       if (Res < 0)
       {
 
       if (Res < 0)
       {


@@ -1508,41 +2188,16 @@ bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual)
            errno = 0;
            continue;
         }
            errno = 0;
            continue;
         }

-        if (false)
-           /* dummy so that the rest can be 'else if's */;
-#ifdef HAVE_ZLIB
-        else if (d != NULL && d->gz != NULL)
-        {
-           int err;
-           char const * const errmsg = gzerror(d->gz, &err);
-           if (err != Z_ERRNO)
-              return FileFdError("gzread: %s (%d: %s)", _("Read error"), err, errmsg);
-        }
-#endif
-#ifdef HAVE_BZ2
-        else if (d != NULL && d->bz2 != NULL)
-        {
-           int err;
-           char const * const errmsg = BZ2_bzerror(d->bz2, &err);
-           if (err != BZ_IO_ERROR)
-              return FileFdError("BZ2_bzread: %s (%d: %s)", _("Read error"), err, errmsg);
-        }
-#endif
-#ifdef HAVE_LZMA
-        else if (d != NULL && d->lzma != NULL)
-           return FileFdError("lzma_read: %s (%d)", _("Read error"), d->lzma->err);
-#endif
-        return FileFdErrno("read",_("Read error"));
+        return d->InternalReadError();

       }
       
       To = (char *)To + Res;
       Size -= Res;
       if (d != NULL)
       }
       
       To = (char *)To + Res;
       Size -= Res;
       if (d != NULL)

-        d->seekpos += Res;
+        d->set_seekpos(d->get_seekpos() + Res);

       if (Actual != 0)
         *Actual += Res;
    }
       if (Actual != 0)
         *Actual += Res;
    }

-   while (Res > 0 && Size > 0);

    
    if (Size == 0)
       return true;
    
    if (Size == 0)
       return true;


@@ -1559,111 +2214,46 @@ bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual)
                                                                        /*}}}*/
 // FileFd::ReadLine - Read a complete line from the file               /*{{{*/
 // ---------------------------------------------------------------------
                                                                        /*}}}*/
 // FileFd::ReadLine - Read a complete line from the file               /*{{{*/
 // ---------------------------------------------------------------------

-/* Beware: This method can be quiet slow for big buffers on UNcompressed
+/* Beware: This method can be quite slow for big buffers on UNcompressed

    files because of the naive implementation! */
 char* FileFd::ReadLine(char *To, unsigned long long const Size)
 {
    *To = '\0';
    files because of the naive implementation! */
 char* FileFd::ReadLine(char *To, unsigned long long const Size)
 {
    *To = '\0';

-#ifdef HAVE_ZLIB
-   if (d != NULL && d->gz != NULL)
-      return gzgets(d->gz, To, Size);
-#endif
+   if (d == nullptr)
+      return nullptr;
+   return d->InternalReadLine(To, Size);
+}
+                                                                       /*}}}*/
+// FileFd::Flush - Flush the file                                      /*{{{*/
+bool FileFd::Flush()
+{
+   if (d == nullptr)
+      return true;

 
 

-   unsigned long long read = 0;
-   while ((Size - 1) != read)
-   {
-      unsigned long long done = 0;
-      if (Read(To + read, 1, &done) == false)
-        return NULL;
-      if (done == 0)
-        break;
-      if (To[read++] == '\n')
-        break;
-   }
-   if (read == 0)
-      return NULL;
-   To[read] = '\0';
-   return To;
+   return d->InternalFlush();

 }
                                                                        /*}}}*/
 // FileFd::Write - Write to the file                                   /*{{{*/
 }
                                                                        /*}}}*/
 // FileFd::Write - Write to the file                                   /*{{{*/

-// ---------------------------------------------------------------------
-/* */

 bool FileFd::Write(const void *From,unsigned long long Size)
 {
 bool FileFd::Write(const void *From,unsigned long long Size)
 {

-   ssize_t Res;
+   if (d == nullptr)
+      return false;
+   ssize_t Res = 1;

    errno = 0;
    errno = 0;

-   do
+   while (Res > 0 && Size > 0)

    {
    {

-      if (false)
-        /* dummy so that the rest can be 'else if's */;
-#ifdef HAVE_ZLIB
-      else if (d != NULL && d->gz != NULL)
-        Res = gzwrite(d->gz,From,Size);
-#endif
-#ifdef HAVE_BZ2
-      else if (d != NULL && d->bz2 != NULL)
-        Res = BZ2_bzwrite(d->bz2,(void*)From,Size);
-#endif
-#ifdef HAVE_LZMA
-      else if (d != NULL && d->lzma != NULL)
-      {
-        d->lzma->stream.next_in = (uint8_t *)From;
-        d->lzma->stream.avail_in = Size;
-        d->lzma->stream.next_out = d->lzma->buffer;
-        d->lzma->stream.avail_out = sizeof(d->lzma->buffer)/sizeof(d->lzma->buffer[0]);
-        d->lzma->err = lzma_code(&d->lzma->stream, LZMA_RUN);
-        if (d->lzma->err != LZMA_OK)
-           return false;
-        size_t const n = sizeof(d->lzma->buffer)/sizeof(d->lzma->buffer[0]) - d->lzma->stream.avail_out;
-        size_t const m = (n == 0) ? 0 : fwrite(d->lzma->buffer, 1, n, d->lzma->file);
-        if (m != n)
-           Res = -1;
-        else
-           Res = Size - d->lzma->stream.avail_in;
-      }
-#endif
-      else
-        Res = write(iFd,From,Size);
-
+      Res = d->InternalWrite(From, Size);

       if (Res < 0 && errno == EINTR)
         continue;
       if (Res < 0)
       if (Res < 0 && errno == EINTR)
         continue;
       if (Res < 0)

-      {
-        if (false)
-           /* dummy so that the rest can be 'else if's */;
-#ifdef HAVE_ZLIB
-        else if (d != NULL && d->gz != NULL)
-        {
-           int err;
-           char const * const errmsg = gzerror(d->gz, &err);
-           if (err != Z_ERRNO)
-              return FileFdError("gzwrite: %s (%d: %s)", _("Write error"), err, errmsg);
-        }
-#endif
-#ifdef HAVE_BZ2
-        else if (d != NULL && d->bz2 != NULL)
-        {
-           int err;
-           char const * const errmsg = BZ2_bzerror(d->bz2, &err);
-           if (err != BZ_IO_ERROR)
-              return FileFdError("BZ2_bzwrite: %s (%d: %s)", _("Write error"), err, errmsg);
-        }
-#endif
-#ifdef HAVE_LZMA
-        else if (d != NULL && d->lzma != NULL)
-           return FileFdErrno("lzma_fwrite", _("Write error"));
-#endif
-        return FileFdErrno("write",_("Write error"));
-      }
-      
+        return d->InternalWriteError();
+

       From = (char const *)From + Res;
       Size -= Res;
       if (d != NULL)
       From = (char const *)From + Res;
       Size -= Res;
       if (d != NULL)

-        d->seekpos += Res;
+        d->set_seekpos(d->get_seekpos() + Res);

    }
    }

-   while (Res > 0 && Size > 0);
-   
+

    if (Size == 0)
       return true;
 
    if (Size == 0)
       return true;
 


@@ -1671,9 +2261,9 @@ bool FileFd::Write(const void *From,unsigned long long Size)
 }
 bool FileFd::Write(int Fd, const void *From, unsigned long long Size)
 {
 }
 bool FileFd::Write(int Fd, const void *From, unsigned long long Size)
 {

-   ssize_t Res;
+   ssize_t Res = 1;

    errno = 0;
    errno = 0;

-   do
+   while (Res > 0 && Size > 0)

    {
       Res = write(Fd,From,Size);
       if (Res < 0 && errno == EINTR)
    {
       Res = write(Fd,From,Size);
       if (Res < 0 && errno == EINTR)


@@ -1684,7 +2274,6 @@ bool FileFd::Write(int Fd, const void *From, unsigned long long Size)
       From = (char const *)From + Res;
       Size -= Res;
    }
       From = (char const *)From + Res;
       Size -= Res;
    }

-   while (Res > 0 && Size > 0);

 
    if (Size == 0)
       return true;
 
    if (Size == 0)
       return true;


@@ -1693,117 +2282,31 @@ bool FileFd::Write(int Fd, const void *From, unsigned long long Size)
 }
                                                                        /*}}}*/
 // FileFd::Seek - Seek in the file                                     /*{{{*/
 }
                                                                        /*}}}*/
 // FileFd::Seek - Seek in the file                                     /*{{{*/

-// ---------------------------------------------------------------------
-/* */

 bool FileFd::Seek(unsigned long long To)
 {
 bool FileFd::Seek(unsigned long long To)
 {

+   if (d == nullptr)
+      return false;

    Flags &= ~HitEof;
    Flags &= ~HitEof;

-
-   if (d != NULL && (d->pipe == true || d->InternalStream() == true))
-   {
-      // Our poor man seeking in pipes is costly, so try to avoid it
-      unsigned long long seekpos = Tell();
-      if (seekpos == To)
-        return true;
-      else if (seekpos < To)
-        return Skip(To - seekpos);
-
-      if ((d->openmode & ReadOnly) != ReadOnly)
-        return FileFdError("Reopen is only implemented for read-only files!");
-      d->InternalClose(FileName);
-      if (iFd != -1)
-        close(iFd);
-      iFd = -1;
-      if (TemporaryFileName.empty() == false)
-        iFd = open(TemporaryFileName.c_str(), O_RDONLY);
-      else if (FileName.empty() == false)
-        iFd = open(FileName.c_str(), O_RDONLY);
-      else
-      {
-        if (d->compressed_fd > 0)
-           if (lseek(d->compressed_fd, 0, SEEK_SET) != 0)
-              iFd = d->compressed_fd;
-        if (iFd < 0)
-           return FileFdError("Reopen is not implemented for pipes opened with FileFd::OpenDescriptor()!");
-      }
-
-      if (OpenInternDescriptor(d->openmode, d->compressor) == false)
-        return FileFdError("Seek on file %s because it couldn't be reopened", FileName.c_str());
-
-      if (To != 0)
-        return Skip(To);
-
-      d->seekpos = To;
-      return true;
-   }
-   off_t res;
-#ifdef HAVE_ZLIB
-   if (d != NULL && d->gz)
-      res = gzseek(d->gz,To,SEEK_SET);
-   else
-#endif
-      res = lseek(iFd,To,SEEK_SET);
-   if (res != (off_t)To)
-      return FileFdError("Unable to seek to %llu", To);
-
-   if (d != NULL)
-      d->seekpos = To;
-   return true;
+   return d->InternalSeek(To);

 }
                                                                        /*}}}*/
 }
                                                                        /*}}}*/

-// FileFd::Skip - Seek in the file                                     /*{{{*/
-// ---------------------------------------------------------------------
-/* */
+// FileFd::Skip - Skip over data in the file                           /*{{{*/

 bool FileFd::Skip(unsigned long long Over)
 {
 bool FileFd::Skip(unsigned long long Over)
 {

-   if (d != NULL && (d->pipe == true || d->InternalStream() == true))
-   {
-      char buffer[1024];
-      while (Over != 0)
-      {
-        unsigned long long toread = std::min((unsigned long long) sizeof(buffer), Over);
-        if (Read(buffer, toread) == false)
-           return FileFdError("Unable to seek ahead %llu",Over);
-        Over -= toread;
-      }
-      return true;
-   }
-
-   off_t res;
-#ifdef HAVE_ZLIB
-   if (d != NULL && d->gz != NULL)
-      res = gzseek(d->gz,Over,SEEK_CUR);
-   else
-#endif
-      res = lseek(iFd,Over,SEEK_CUR);
-   if (res < 0)
-      return FileFdError("Unable to seek ahead %llu",Over);
-   if (d != NULL)
-      d->seekpos = res;
-
-   return true;
+   if (d == nullptr)
+      return false;
+   return d->InternalSkip(Over);

 }
                                                                        /*}}}*/
 }
                                                                        /*}}}*/

-// FileFd::Truncate - Truncate the file                                /*{{{*/
-// ---------------------------------------------------------------------
-/* */
+// FileFd::Truncate - Truncate the file                                        /*{{{*/

 bool FileFd::Truncate(unsigned long long To)
 {
 bool FileFd::Truncate(unsigned long long To)
 {

+   if (d == nullptr)
+      return false;

    // truncating /dev/null is always successful - as we get an error otherwise
    if (To == 0 && FileName == "/dev/null")
       return true;
    // truncating /dev/null is always successful - as we get an error otherwise
    if (To == 0 && FileName == "/dev/null")
       return true;

-#if defined HAVE_ZLIB || defined HAVE_BZ2 || defined HAVE_LZMA
-   if (d != NULL && (d->InternalStream() == true
-#ifdef HAVE_ZLIB
-           || d->gz != NULL
-#endif
-           ))
-      return FileFdError("Truncating compressed files is not implemented (%s)", FileName.c_str());
-#endif
-   if (ftruncate(iFd,To) != 0)
-      return FileFdError("Unable to truncate to %llu",To);
-
-   return true;
+   return d->InternalTruncate(To);

 }
                                                                        /*}}}*/
 // FileFd::Tell - Current seek position                                        /*{{{*/
 }
                                                                        /*}}}*/
 // FileFd::Tell - Current seek position                                        /*{{{*/


@@ -1811,30 +2314,18 @@ bool FileFd::Truncate(unsigned long long To)
 /* */
 unsigned long long FileFd::Tell()
 {
 /* */
 unsigned long long FileFd::Tell()
 {

-   // In theory, we could just return seekpos here always instead of
-   // seeking around, but not all users of FileFd use always Seek() and co
-   // so d->seekpos isn't always true and we can just use it as a hint if
-   // we have nothing else, but not always as an authority…
-   if (d != NULL && (d->pipe == true || d->InternalStream() == true))
-      return d->seekpos;
-
-   off_t Res;
-#ifdef HAVE_ZLIB
-   if (d != NULL && d->gz != NULL)
-     Res = gztell(d->gz);
-   else
-#endif
-     Res = lseek(iFd,0,SEEK_CUR);
+   if (d == nullptr)
+      return false;
+   off_t const Res = d->InternalTell();

    if (Res == (off_t)-1)
       FileFdErrno("lseek","Failed to determine the current file position");
    if (Res == (off_t)-1)
       FileFdErrno("lseek","Failed to determine the current file position");

-   if (d != NULL)
-      d->seekpos = Res;
+   d->set_seekpos(Res);

    return Res;
 }
                                                                        /*}}}*/
 static bool StatFileFd(char const * const msg, int const iFd, std::string const &FileName, struct stat &Buf, FileFdPrivate * const d) /*{{{*/
 {
    return Res;
 }
                                                                        /*}}}*/
 static bool StatFileFd(char const * const msg, int const iFd, std::string const &FileName, struct stat &Buf, FileFdPrivate * const d) /*{{{*/
 {

-   bool ispipe = (d != NULL && d->pipe == true);
+   bool ispipe = (d != NULL && d->get_is_pipe() == true);

    if (ispipe == false)
    {
       if (fstat(iFd,&Buf) != 0)
    if (ispipe == false)
    {
       if (fstat(iFd,&Buf) != 0)


@@ -1851,7 +2342,7 @@ static bool StatFileFd(char const * const msg, int const iFd, std::string const
       // we set it here, too, as we get the info here for free
       // in theory the Open-methods should take care of it already
       if (d != NULL)
       // we set it here, too, as we get the info here for free
       // in theory the Open-methods should take care of it already
       if (d != NULL)

-        d->pipe = true;
+        d->set_is_pipe(true);

       if (stat(FileName.c_str(), &Buf) != 0)
         return _error->Errno("fstat", "Unable to determine %s for file %s", msg, FileName.c_str());
    }
       if (stat(FileName.c_str(), &Buf) != 0)
         return _error->Errno("fstat", "Unable to determine %s for file %s", msg, FileName.c_str());
    }


@@ -1883,64 +2374,11 @@ time_t FileFd::ModificationTime()
 }
                                                                        /*}}}*/
 // FileFd::Size - Return the size of the content in the file           /*{{{*/
 }
                                                                        /*}}}*/
 // FileFd::Size - Return the size of the content in the file           /*{{{*/

-// ---------------------------------------------------------------------
-/* */

 unsigned long long FileFd::Size()
 {
 unsigned long long FileFd::Size()
 {

-   unsigned long long size = FileSize();
-
-   // for compressor pipes st_size is undefined and at 'best' zero,
-   // so we 'read' the content and 'seek' back - see there
-   if (d != NULL && (d->pipe == true || (d->InternalStream() == true && size > 0)))
-   {
-      unsigned long long const oldSeek = Tell();
-      char ignore[1000];
-      unsigned long long read = 0;
-      do {
-        if (Read(ignore, sizeof(ignore), &read) == false)
-        {
-           Seek(oldSeek);
-           return 0;
-        }
-      } while(read != 0);
-      size = Tell();
-      Seek(oldSeek);
-   }
-#ifdef HAVE_ZLIB
-   // only check gzsize if we are actually a gzip file, just checking for
-   // "gz" is not sufficient as uncompressed files could be opened with
-   // gzopen in "direct" mode as well
-   else if (d != NULL && d->gz && !gzdirect(d->gz) && size > 0)
-   {
-       off_t const oldPos = lseek(iFd,0,SEEK_CUR);
-       /* unfortunately zlib.h doesn't provide a gzsize(), so we have to do
-       * this ourselves; the original (uncompressed) file size is the last 32
-       * bits of the file */
-       // FIXME: Size for gz-files is limited by 32bit… no largefile support
-       if (lseek(iFd, -4, SEEK_END) < 0)
-       {
-         FileFdErrno("lseek","Unable to seek to end of gzipped file");
-         return 0;
-       }
-       uint32_t size = 0;
-       if (read(iFd, &size, 4) != 4)
-       {
-         FileFdErrno("read","Unable to read original size of gzipped file");
-         return 0;
-       }
-       size = le32toh(size);
-
-       if (lseek(iFd, oldPos, SEEK_SET) < 0)
-       {
-         FileFdErrno("lseek","Unable to seek in gzipped file");
-         return 0;
-       }
-
-       return size;
-   }
-#endif
-
-   return size;
+   if (d == nullptr)
+      return false;
+   return d->InternalSize();

 }
                                                                        /*}}}*/
 // FileFd::Close - Close the file if the close flag is set             /*{{{*/
 }
                                                                        /*}}}*/
 // FileFd::Close - Close the file if the close flag is set             /*{{{*/


@@ -1948,6 +2386,8 @@ unsigned long long FileFd::Size()
 /* */
 bool FileFd::Close()
 {
 /* */
 bool FileFd::Close()
 {

+   if (Flush() == false)
+      return false;

    if (iFd == -1)
       return true;
 
    if (iFd == -1)
       return true;
 


@@ -1956,12 +2396,13 @@ bool FileFd::Close()
    {
       if ((Flags & Compressed) != Compressed && iFd > 0 && close(iFd) != 0)
         Res &= _error->Errno("close",_("Problem closing the file %s"), FileName.c_str());
    {
       if ((Flags & Compressed) != Compressed && iFd > 0 && close(iFd) != 0)
         Res &= _error->Errno("close",_("Problem closing the file %s"), FileName.c_str());

-      if (d != NULL)
-      {
-        Res &= d->CloseDown(FileName);
-        delete d;
-        d = NULL;
-      }
+   }
+
+   if (d != NULL)
+   {
+      Res &= d->InternalClose(FileName);
+      delete d;
+      d = NULL;

    }
 
    if ((Flags & Replace) == Replace) {
    }
 
    if ((Flags & Replace) == Replace) {


@@ -1976,8 +2417,7 @@ bool FileFd::Close()
 
    if ((Flags & Fail) == Fail && (Flags & DelOnFail) == DelOnFail &&
        FileName.empty() == false)
 
    if ((Flags & Fail) == Fail && (Flags & DelOnFail) == DelOnFail &&
        FileName.empty() == false)

-      if (unlink(FileName.c_str()) != 0)
-        Res &= _error->WarningE("unlnk",_("Problem unlinking the file %s"), FileName.c_str());
+      Res &= RemoveFile("FileFd::Close", FileName);

 
    if (Res == false)
       Flags |= Fail;
 
    if (Res == false)
       Flags |= Fail;


@@ -2026,19 +2466,20 @@ bool FileFd::FileFdError(const char *Description,...) {
    return false;
 }
                                                                        /*}}}*/
    return false;
 }
                                                                        /*}}}*/

-
-APT_DEPRECATED gzFile FileFd::gzFd() {
+gzFile FileFd::gzFd() {                                                        /*{{{*/

 #ifdef HAVE_ZLIB
 #ifdef HAVE_ZLIB

-   return d->gz;
+   GzipFileFdPrivate * const gzipd = dynamic_cast<GzipFileFdPrivate*>(d);
+   if (gzipd == nullptr)
+      return nullptr;
+   else
+      return gzipd->gz;

 #else
 #else

-   return NULL;
+   return nullptr;

 #endif
 }
 #endif
 }

+                                                                       /*}}}*/

 
 

-
-// Glob - wrapper around "glob()"                                      /*{{{*/
-// ---------------------------------------------------------------------
-/* */
+// Glob - wrapper around "glob()"                                      /*{{{*/

 std::vector<std::string> Glob(std::string const &pattern, int flags)
 {
    std::vector<std::string> result;
 std::vector<std::string> Glob(std::string const &pattern, int flags)
 {
    std::vector<std::string> result;


@@ -2064,8 +2505,7 @@ std::vector<std::string> Glob(std::string const &pattern, int flags)
    return result;
 }
                                                                        /*}}}*/
    return result;
 }
                                                                        /*}}}*/

-
-std::string GetTempDir()
+std::string GetTempDir()                                               /*{{{*/

 {
    const char *tmpdir = getenv("TMPDIR");
 
 {
    const char *tmpdir = getenv("TMPDIR");
 


@@ -2074,51 +2514,79 @@ std::string GetTempDir()
       tmpdir = P_tmpdir;
 #endif
 
       tmpdir = P_tmpdir;
 #endif
 

-   // check that tmpdir is set and exists

    struct stat st;
    struct stat st;

-   if (!tmpdir || strlen(tmpdir) == 0 || stat(tmpdir, &st) != 0)
+   if (!tmpdir || strlen(tmpdir) == 0 || // tmpdir is set
+        stat(tmpdir, &st) != 0 || (st.st_mode & S_IFDIR) == 0) // exists and is directory
+      tmpdir = "/tmp";
+   else if (geteuid() != 0 && // root can do everything anyway
+        faccessat(-1, tmpdir, R_OK | W_OK | X_OK, AT_EACCESS | AT_SYMLINK_NOFOLLOW) != 0) // current user has rwx access to directory

       tmpdir = "/tmp";
 
    return string(tmpdir);
 }
       tmpdir = "/tmp";
 
    return string(tmpdir);
 }

+std::string GetTempDir(std::string const &User)
+{
+   // no need/possibility to drop privs
+   if(getuid() != 0 || User.empty() || User == "root")
+      return GetTempDir();
+
+   struct passwd const * const pw = getpwnam(User.c_str());
+   if (pw == NULL)
+      return GetTempDir();

 
 

-FileFd* GetTempFile(std::string const &Prefix, bool ImmediateUnlink)
+   gid_t const old_euid = geteuid();
+   gid_t const old_egid = getegid();
+   if (setegid(pw->pw_gid) != 0)
+      _error->Errno("setegid", "setegid %u failed", pw->pw_gid);
+   if (seteuid(pw->pw_uid) != 0)
+      _error->Errno("seteuid", "seteuid %u failed", pw->pw_uid);
+
+   std::string const tmp = GetTempDir();
+
+   if (seteuid(old_euid) != 0)
+      _error->Errno("seteuid", "seteuid %u failed", old_euid);
+   if (setegid(old_egid) != 0)
+      _error->Errno("setegid", "setegid %u failed", old_egid);
+
+   return tmp;
+}
+                                                                       /*}}}*/
+FileFd* GetTempFile(std::string const &Prefix, bool ImmediateUnlink, FileFd * const TmpFd)     /*{{{*/

 {
    char fn[512];
 {
    char fn[512];

-   FileFd *Fd = new FileFd();
+   FileFd * const Fd = TmpFd == NULL ? new FileFd() : TmpFd;

 
 

-   std::string tempdir = GetTempDir();
-   snprintf(fn, sizeof(fn), "%s/%s.XXXXXX", 
+   std::string const tempdir = GetTempDir();
+   snprintf(fn, sizeof(fn), "%s/%s.XXXXXX",

             tempdir.c_str(), Prefix.c_str());
             tempdir.c_str(), Prefix.c_str());

-   int fd = mkstemp(fn);
+   int const fd = mkstemp(fn);

    if(ImmediateUnlink)
       unlink(fn);
    if(ImmediateUnlink)
       unlink(fn);

-   if (fd < 0) 
+   if (fd < 0)

    {
       _error->Errno("GetTempFile",_("Unable to mkstemp %s"), fn);
       return NULL;
    }
    {
       _error->Errno("GetTempFile",_("Unable to mkstemp %s"), fn);
       return NULL;
    }

-   if (!Fd->OpenDescriptor(fd, FileFd::WriteOnly, FileFd::None, true))
+   if (!Fd->OpenDescriptor(fd, FileFd::ReadWrite, FileFd::None, true))

    {
       _error->Errno("GetTempFile",_("Unable to write to %s"),fn);
       return NULL;
    }
    {
       _error->Errno("GetTempFile",_("Unable to write to %s"),fn);
       return NULL;
    }

-

    return Fd;
 }
    return Fd;
 }

-
-bool Rename(std::string From, std::string To)
+                                                                       /*}}}*/
+bool Rename(std::string From, std::string To)                          /*{{{*/

 {
    if (rename(From.c_str(),To.c_str()) != 0)
    {
       _error->Error(_("rename failed, %s (%s -> %s)."),strerror(errno),
                     From.c_str(),To.c_str());
       return false;
 {
    if (rename(From.c_str(),To.c_str()) != 0)
    {
       _error->Error(_("rename failed, %s (%s -> %s)."),strerror(errno),
                     From.c_str(),To.c_str());
       return false;

-   }   
+   }

    return true;
 }
    return true;
 }

-
-bool Popen(const char* Args[], FileFd &Fd, pid_t &Child, FileFd::OpenMode Mode)
+                                                                       /*}}}*/
+bool Popen(const char* Args[], FileFd &Fd, pid_t &Child, FileFd::OpenMode Mode)/*{{{*/

 {
    int fd;
    if (Mode != FileFd::ReadOnly && Mode != FileFd::WriteOnly)
 {
    int fd;
    if (Mode != FileFd::ReadOnly && Mode != FileFd::WriteOnly)


@@ -2161,101 +2629,149 @@ bool Popen(const char* Args[], FileFd &Fd, pid_t &Child, FileFd::OpenMode Mode)
    {
       close(Pipe[1]);
       fd = Pipe[0];
    {
       close(Pipe[1]);
       fd = Pipe[0];

-   } else if(Mode == FileFd::WriteOnly)
+   }
+   else if(Mode == FileFd::WriteOnly)

    {
       close(Pipe[0]);
       fd = Pipe[1];
    }
    {
       close(Pipe[0]);
       fd = Pipe[1];
    }

+   else
+      return _error->Error("Popen supports ReadOnly (x)or WriteOnly mode only");

    Fd.OpenDescriptor(fd, Mode, FileFd::None, true);
 
    return true;
 }
    Fd.OpenDescriptor(fd, Mode, FileFd::None, true);
 
    return true;
 }

-
-bool DropPrivs()
+                                                                       /*}}}*/
+bool DropPrivileges()                                                  /*{{{*/

 {
 {

+   if(_config->FindB("Debug::NoDropPrivs", false) == true)
+      return true;
+
+#if __gnu_linux__
+#if defined(PR_SET_NO_NEW_PRIVS) && ( PR_SET_NO_NEW_PRIVS != 38 )
+#error "PR_SET_NO_NEW_PRIVS is defined, but with a different value than expected!"
+#endif
+   // see prctl(2), needs linux3.5 at runtime - magic constant to avoid it at buildtime
+   int ret = prctl(38, 1, 0, 0, 0);
+   // ignore EINVAL - kernel is too old to understand the option
+   if(ret < 0 && errno != EINVAL)
+      _error->Warning("PR_SET_NO_NEW_PRIVS failed with %i", ret);
+#endif
+
+   // empty setting disables privilege dropping - this also ensures
+   // backward compatibility, see bug #764506
+   const std::string toUser = _config->Find("APT::Sandbox::User");
+   if (toUser.empty() || toUser == "root")
+      return true;
+
+   // a lot can go wrong trying to drop privileges completely,
+   // so ideally we would like to verify that we have done it –
+   // but the verify asks for too much in case of fakeroot (and alike)
+   // [Specific checks can be overridden with dedicated options]
+   bool const VerifySandboxing = _config->FindB("APT::Sandbox::Verify", false);
+

    // uid will be 0 in the end, but gid might be different anyway
    // uid will be 0 in the end, but gid might be different anyway

-   uid_t old_uid = getuid();
-   gid_t old_gid = getgid();
+   uid_t const old_uid = getuid();
+   gid_t const old_gid = getgid();

 
    if (old_uid != 0)
       return true;
 
    if (old_uid != 0)
       return true;

-   if(_config->FindB("Debug::NoDropPrivs", false) == true)
-      return true;

 
 

-   const std::string toUser = _config->Find("APT::Sandbox::User", "_apt");

    struct passwd *pw = getpwnam(toUser.c_str());
    if (pw == NULL)
       return _error->Error("No user %s, can not drop rights", toUser.c_str());
 
    struct passwd *pw = getpwnam(toUser.c_str());
    if (pw == NULL)
       return _error->Error("No user %s, can not drop rights", toUser.c_str());
 

-#if __gnu_linux__
-   // see prctl(2), needs linux3.5
-   int ret = prctl(PR_SET_NO_NEW_PRIVS, 1, 0,0, 0);
-   if(ret < 0)
-      _error->Warning("PR_SET_NO_NEW_PRIVS failed with %i", ret);
-#endif

    // Do not change the order here, it might break things
    // Do not change the order here, it might break things

+   // Get rid of all our supplementary groups first

    if (setgroups(1, &pw->pw_gid))
       return _error->Errno("setgroups", "Failed to setgroups");
 
    if (setgroups(1, &pw->pw_gid))
       return _error->Errno("setgroups", "Failed to setgroups");
 

+   // Now change the group ids to the new user
+#ifdef HAVE_SETRESGID
+   if (setresgid(pw->pw_gid, pw->pw_gid, pw->pw_gid) != 0)
+      return _error->Errno("setresgid", "Failed to set new group ids");
+#else

    if (setegid(pw->pw_gid) != 0)
       return _error->Errno("setegid", "Failed to setegid");
 
    if (setgid(pw->pw_gid) != 0)
       return _error->Errno("setgid", "Failed to setgid");
    if (setegid(pw->pw_gid) != 0)
       return _error->Errno("setegid", "Failed to setegid");
 
    if (setgid(pw->pw_gid) != 0)
       return _error->Errno("setgid", "Failed to setgid");

+#endif

 
 

+   // Change the user ids to the new user
+#ifdef HAVE_SETRESUID
+   if (setresuid(pw->pw_uid, pw->pw_uid, pw->pw_uid) != 0)
+      return _error->Errno("setresuid", "Failed to set new user ids");
+#else

    if (setuid(pw->pw_uid) != 0)
       return _error->Errno("setuid", "Failed to setuid");
    if (setuid(pw->pw_uid) != 0)
       return _error->Errno("setuid", "Failed to setuid");

-
-   // the seteuid() is probably uneeded (at least thats what the linux
-   // man-page says about setuid(2)) but we cargo culted it anyway

    if (seteuid(pw->pw_uid) != 0)
       return _error->Errno("seteuid", "Failed to seteuid");
    if (seteuid(pw->pw_uid) != 0)
       return _error->Errno("seteuid", "Failed to seteuid");

+#endif

 
 

-   // Verify that the user has only a single group, and the correct one
-   gid_t groups[1];
-   if (getgroups(1, groups) != 1)
-      return _error->Errno("getgroups", "Could not get new groups");
-   if (groups[0] != pw->pw_gid)
-      return _error->Error("Could not switch group");
-
-   // Verify that gid, egid, uid, and euid changed
-   if (getgid() != pw->pw_gid)
-      return _error->Error("Could not switch group");
-   if (getegid() != pw->pw_gid)
-      return _error->Error("Could not switch effective group");
-   if (getuid() != pw->pw_uid)
-      return _error->Error("Could not switch user");
-   if (geteuid() != pw->pw_uid)
-      return _error->Error("Could not switch effective user");
+   // disabled by default as fakeroot doesn't implement getgroups currently (#806521)
+   if (VerifySandboxing == true || _config->FindB("APT::Sandbox::Verify::Groups", false) == true)
+   {
+      // Verify that the user isn't still in any supplementary groups
+      long const ngroups_max = sysconf(_SC_NGROUPS_MAX);
+      std::unique_ptr<gid_t[]> gidlist(new gid_t[ngroups_max]);
+      if (unlikely(gidlist == NULL))
+        return _error->Error("Allocation of a list of size %lu for getgroups failed", ngroups_max);
+      ssize_t gidlist_nr;
+      if ((gidlist_nr = getgroups(ngroups_max, gidlist.get())) < 0)
+        return _error->Errno("getgroups", "Could not get new groups (%lu)", ngroups_max);
+      for (ssize_t i = 0; i < gidlist_nr; ++i)
+        if (gidlist[i] != pw->pw_gid)
+           return _error->Error("Could not switch group, user %s is still in group %d", toUser.c_str(), gidlist[i]);
+   }
+
+   // enabled by default as all fakeroot-lookalikes should fake that accordingly
+   if (VerifySandboxing == true || _config->FindB("APT::Sandbox::Verify::IDs", true) == true)
+   {
+      // Verify that gid, egid, uid, and euid changed
+      if (getgid() != pw->pw_gid)
+        return _error->Error("Could not switch group");
+      if (getegid() != pw->pw_gid)
+        return _error->Error("Could not switch effective group");
+      if (getuid() != pw->pw_uid)
+        return _error->Error("Could not switch user");
+      if (geteuid() != pw->pw_uid)
+        return _error->Error("Could not switch effective user");

 
 #ifdef HAVE_GETRESUID
 
 #ifdef HAVE_GETRESUID

-   // verify that the saved set-user-id was changed as well
-   uid_t ruid = 0;
-   uid_t euid = 0;
-   uid_t suid = 0;
-   if (getresuid(&ruid, &euid, &suid))
-      return _error->Errno("getresuid", "Could not get saved set-user-ID");
-   if (suid != pw->pw_uid)
-      return _error->Error("Could not switch saved set-user-ID");
+      // verify that the saved set-user-id was changed as well
+      uid_t ruid = 0;
+      uid_t euid = 0;
+      uid_t suid = 0;
+      if (getresuid(&ruid, &euid, &suid))
+        return _error->Errno("getresuid", "Could not get saved set-user-ID");
+      if (suid != pw->pw_uid)
+        return _error->Error("Could not switch saved set-user-ID");

 #endif
 
 #ifdef HAVE_GETRESGID
 #endif
 
 #ifdef HAVE_GETRESGID

-   // verify that the saved set-group-id was changed as well
-   gid_t rgid = 0;
-   gid_t egid = 0;
-   gid_t sgid = 0;
-   if (getresgid(&rgid, &egid, &sgid))
-      return _error->Errno("getresuid", "Could not get saved set-group-ID");
-   if (sgid != pw->pw_gid)
-      return _error->Error("Could not switch saved set-group-ID");
+      // verify that the saved set-group-id was changed as well
+      gid_t rgid = 0;
+      gid_t egid = 0;
+      gid_t sgid = 0;
+      if (getresgid(&rgid, &egid, &sgid))
+        return _error->Errno("getresuid", "Could not get saved set-group-ID");
+      if (sgid != pw->pw_gid)
+        return _error->Error("Could not switch saved set-group-ID");

 #endif
 #endif

+   }

 
 

-   // Check that uid and gid changes do not work anymore
-   if (pw->pw_gid != old_gid && (setgid(old_gid) != -1 || setegid(old_gid) != -1))
-      return _error->Error("Could restore a gid to root, privilege dropping did not work");
+   // disabled as fakeroot doesn't forbid (by design) (re)gaining root from unprivileged
+   if (VerifySandboxing == true || _config->FindB("APT::Sandbox::Verify::Regain", false) == true)
+   {
+      // Check that uid and gid changes do not work anymore
+      if (pw->pw_gid != old_gid && (setgid(old_gid) != -1 || setegid(old_gid) != -1))
+        return _error->Error("Could restore a gid to root, privilege dropping did not work");

 
 

-   if (pw->pw_uid != old_uid && (setuid(old_uid) != -1 || seteuid(old_uid) != -1))
-      return _error->Error("Could restore a uid to root, privilege dropping did not work");
+      if (pw->pw_uid != old_uid && (setuid(old_uid) != -1 || seteuid(old_uid) != -1))
+        return _error->Error("Could restore a uid to root, privilege dropping did not work");
+   }

 
    return true;
 }
 
    return true;
 }

+                                                                       /*}}}*/