X-Git-Url: https://git.saurik.com/apt.git/blobdiff_plain/c36db2b56a08bb7542d0c1be900ee90808efa770..47fcfff8a50caebe3cd1f2ee19875a2ca8bbeadc:/apt-pkg/contrib/fileutl.cc?ds=sidebyside diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index c5eb56f0e..004df3fe8 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -52,6 +52,7 @@ #include #include +#include #ifdef HAVE_ZLIB #include @@ -62,6 +63,9 @@ #ifdef HAVE_LZMA #include #endif +#ifdef HAVE_LZ4 + #include +#endif #include #include @@ -74,6 +78,9 @@ using namespace std; +/* Should be a multiple of the common page size (4096) */ +static constexpr unsigned long long APT_BUFFER_SIZE = 64 * 1024; + // RunScripts - Run a set of scripts from a configuration subtree /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -157,24 +164,33 @@ bool CopyFile(FileFd &From,FileFd &To) if (From.IsOpen() == false || To.IsOpen() == false || From.Failed() == true || To.Failed() == true) return false; - + // Buffered copy between fds - SPtrArray Buf = new unsigned char[64000]; - unsigned long long Size = From.Size(); - while (Size != 0) - { - unsigned long long ToRead = Size; - if (Size > 64000) - ToRead = 64000; - - if (From.Read(Buf,ToRead) == false || - To.Write(Buf,ToRead) == false) + constexpr size_t BufSize = APT_BUFFER_SIZE; + std::unique_ptr Buf(new unsigned char[BufSize]); + unsigned long long ToRead = 0; + do { + if (From.Read(Buf.get(),BufSize, &ToRead) == false || + To.Write(Buf.get(),ToRead) == false) return false; - - Size -= ToRead; - } + } while (ToRead != 0); + + return true; +} + /*}}}*/ +bool RemoveFile(char const * const Function, std::string const &FileName)/*{{{*/ +{ + if (FileName == "/dev/null") + return true; + errno = 0; + if (unlink(FileName.c_str()) != 0) + { + if (errno == ENOENT) + return true; - return true; + return _error->WarningE(Function,_("Problem unlinking the file %s"), FileName.c_str()); + } + return true; } /*}}}*/ // GetLock - Gets a lock file /*{{{*/ @@ -670,7 +686,7 @@ string flAbsPath(string File) char *p = realpath(File.c_str(), NULL); if (p == NULL) { - _error->Errno("realpath", "flAbsPath failed"); + _error->Errno("realpath", "flAbsPath on %s failed", File.c_str()); return ""; } std::string AbsPath(p); @@ -800,11 +816,26 @@ pid_t ExecFork(std::set KeepFDs) signal(SIGCONT,SIG_DFL); signal(SIGTSTP,SIG_DFL); - // Close all of our FDs - just in case - for (int K = 3; K != sysconf(_SC_OPEN_MAX); K++) + DIR *dir = opendir("/proc/self/fd"); + if (dir != NULL) { - if(KeepFDs.find(K) == KeepFDs.end()) - fcntl(K,F_SETFD,FD_CLOEXEC); + struct dirent *ent; + while ((ent = readdir(dir))) + { + int fd = atoi(ent->d_name); + // If fd > 0, it was a fd number and not . or .. + if (fd >= 3 && KeepFDs.find(fd) == KeepFDs.end()) + fcntl(fd,F_SETFD,FD_CLOEXEC); + } + closedir(dir); + } else { + long ScOpenMax = sysconf(_SC_OPEN_MAX); + // Close all of our FDs - just in case + for (int K = 3; K != ScOpenMax; K++) + { + if(KeepFDs.find(K) == KeepFDs.end()) + fcntl(K,F_SETFD,FD_CLOEXEC); + } } } @@ -857,11 +888,7 @@ bool ExecWait(pid_t Pid,const char *Name,bool Reap) return true; } /*}}}*/ - - -// StartsWithGPGClearTextSignature - Check if a file is Pgp/GPG clearsigned /*{{{*/ -// --------------------------------------------------------------------- -/* */ +// StartsWithGPGClearTextSignature - Check if a file is Pgp/GPG clearsigned /*{{{*/ bool StartsWithGPGClearTextSignature(string const &FileName) { static const char* SIGMSG = "-----BEGIN PGP SIGNED MESSAGE-----\n"; @@ -877,551 +904,1430 @@ bool StartsWithGPGClearTextSignature(string const &FileName) return true; } - - -class FileFdPrivate { /*{{{*/ - public: -#ifdef HAVE_ZLIB - gzFile gz; -#endif -#ifdef HAVE_BZ2 - BZFILE* bz2; -#endif -#ifdef HAVE_LZMA - struct LZMAFILE { - FILE* file; - uint8_t buffer[4096]; - lzma_stream stream; - lzma_ret err; - bool eof; - bool compressing; - - LZMAFILE() : file(NULL), eof(false), compressing(false) { buffer[0] = '\0'; } - ~LZMAFILE() { - if (compressing == true) - { - for (;;) { - stream.avail_out = sizeof(buffer)/sizeof(buffer[0]); - stream.next_out = buffer; - err = lzma_code(&stream, LZMA_FINISH); - if (err != LZMA_OK && err != LZMA_STREAM_END) - { - _error->Error("~LZMAFILE: Compress finalisation failed"); - break; - } - size_t const n = sizeof(buffer)/sizeof(buffer[0]) - stream.avail_out; - if (n && fwrite(buffer, 1, n, file) != n) - { - _error->Errno("~LZMAFILE",_("Write error")); - break; - } - if (err == LZMA_STREAM_END) - break; - } - } - lzma_end(&stream); - fclose(file); - } - }; - LZMAFILE* lzma; -#endif - int compressed_fd; - pid_t compressor_pid; - bool pipe; - APT::Configuration::Compressor compressor; - unsigned int openmode; - unsigned long long seekpos; - FileFdPrivate() : -#ifdef HAVE_ZLIB - gz(NULL), -#endif -#ifdef HAVE_BZ2 - bz2(NULL), -#endif -#ifdef HAVE_LZMA - lzma(NULL), -#endif - compressed_fd(-1), compressor_pid(-1), pipe(false), - openmode(0), seekpos(0) {}; - bool InternalClose(std::string const &FileName) - { - if (false) - /* dummy so that the rest can be 'else if's */; -#ifdef HAVE_ZLIB - else if (gz != NULL) { - int const e = gzclose(gz); - gz = NULL; - // gzdclose() on empty files always fails with "buffer error" here, ignore that - if (e != 0 && e != Z_BUF_ERROR) - return _error->Errno("close",_("Problem closing the gzip file %s"), FileName.c_str()); - } -#endif -#ifdef HAVE_BZ2 - else if (bz2 != NULL) { - BZ2_bzclose(bz2); - bz2 = NULL; - } -#endif -#ifdef HAVE_LZMA - else if (lzma != NULL) { - delete lzma; - lzma = NULL; - } -#endif - return true; - } - bool CloseDown(std::string const &FileName) - { - bool const Res = InternalClose(FileName); - - if (compressor_pid > 0) - ExecWait(compressor_pid, "FileFdCompressor", true); - compressor_pid = -1; - - return Res; - } - bool InternalStream() const { - return false -#ifdef HAVE_BZ2 - || bz2 != NULL -#endif -#ifdef HAVE_LZMA - || lzma != NULL -#endif - ; - } - - - ~FileFdPrivate() { CloseDown(""); } -}; /*}}}*/ -// FileFd::Open - Open a file /*{{{*/ -// --------------------------------------------------------------------- -/* The most commonly used open mode combinations are given with Mode */ -bool FileFd::Open(string FileName,unsigned int const Mode,CompressMode Compress, unsigned long const AccessMode) +// ChangeOwnerAndPermissionOfFile - set file attributes to requested values /*{{{*/ +bool ChangeOwnerAndPermissionOfFile(char const * const requester, char const * const file, char const * const user, char const * const group, mode_t const mode) { - if (Mode == ReadOnlyGzip) - return Open(FileName, ReadOnly, Gzip, AccessMode); + if (strcmp(file, "/dev/null") == 0) + return true; + bool Res = true; + if (getuid() == 0 && strlen(user) != 0 && strlen(group) != 0) // if we aren't root, we can't chown, so don't try it + { + // ensure the file is owned by root and has good permissions + struct passwd const * const pw = getpwnam(user); + struct group const * const gr = getgrnam(group); + if (pw != NULL && gr != NULL && chown(file, pw->pw_uid, gr->gr_gid) != 0) + Res &= _error->WarningE(requester, "chown to %s:%s of file %s failed", user, group, file); + } + if (chmod(file, mode) != 0) + Res &= _error->WarningE(requester, "chmod 0%o of file %s failed", mode, file); + return Res; +} + /*}}}*/ - if (Compress == Auto && (Mode & WriteOnly) == WriteOnly) - return FileFdError("Autodetection on %s only works in ReadOnly openmode!", FileName.c_str()); +struct APT_HIDDEN simple_buffer { /*{{{*/ + size_t buffersize_max = 0; + unsigned long long bufferstart = 0; + unsigned long long bufferend = 0; + char *buffer = nullptr; - std::vector const compressors = APT::Configuration::getCompressors(); - std::vector::const_iterator compressor = compressors.begin(); - if (Compress == Auto) + simple_buffer() { + reset(4096); + } + ~simple_buffer() { + delete[] buffer; + } + + const char *get() const { return buffer + bufferstart; } + char *get() { return buffer + bufferstart; } + const char *getend() const { return buffer + bufferend; } + char *getend() { return buffer + bufferend; } + bool empty() const { return bufferend <= bufferstart; } + bool full() const { return bufferend == buffersize_max; } + unsigned long long free() const { return buffersize_max - bufferend; } + unsigned long long size() const { return bufferend-bufferstart; } + void reset(size_t size) { - for (; compressor != compressors.end(); ++compressor) - { - std::string file = FileName + compressor->Extension; - if (FileExists(file) == false) - continue; - FileName = file; - break; + if (size > buffersize_max) { + delete[] buffer; + buffersize_max = size; + buffer = new char[size]; } + reset(); } - else if (Compress == Extension) + void reset() { bufferend = bufferstart = 0; } + ssize_t read(void *to, unsigned long long requested_size) APT_MUSTCHECK { - std::string::size_type const found = FileName.find_last_of('.'); - std::string ext; - if (found != std::string::npos) - { - ext = FileName.substr(found); - if (ext == ".new" || ext == ".bak") - { - std::string::size_type const found2 = FileName.find_last_of('.', found - 1); - if (found2 != std::string::npos) - ext = FileName.substr(found2, found - found2); - else - ext.clear(); - } - } - for (; compressor != compressors.end(); ++compressor) - if (ext == compressor->Extension) - break; - // no matching extension - assume uncompressed (imagine files like 'example.org_Packages') - if (compressor == compressors.end()) - for (compressor = compressors.begin(); compressor != compressors.end(); ++compressor) - if (compressor->Name == ".") - break; + if (size() < requested_size) + requested_size = size(); + memcpy(to, buffer + bufferstart, requested_size); + bufferstart += requested_size; + if (bufferstart == bufferend) + bufferstart = bufferend = 0; + return requested_size; } - else + ssize_t write(const void *from, unsigned long long requested_size) APT_MUSTCHECK { - std::string name; - switch (Compress) - { - case None: name = "."; break; - case Gzip: name = "gzip"; break; - case Bzip2: name = "bzip2"; break; - case Lzma: name = "lzma"; break; - case Xz: name = "xz"; break; - case Auto: - case Extension: - // Unreachable - return FileFdError("Opening File %s in None, Auto or Extension should be already handled?!?", FileName.c_str()); - } - for (; compressor != compressors.end(); ++compressor) - if (compressor->Name == name) - break; - if (compressor == compressors.end()) - return FileFdError("Can't find a configured compressor %s for file %s", name.c_str(), FileName.c_str()); + if (free() < requested_size) + requested_size = free(); + memcpy(getend(), from, requested_size); + bufferend += requested_size; + if (bufferstart == bufferend) + bufferstart = bufferend = 0; + return requested_size; } +}; + /*}}}*/ - if (compressor == compressors.end()) - return FileFdError("Can't find a match for specified compressor mode for file %s", FileName.c_str()); - return Open(FileName, Mode, *compressor, AccessMode); -} -bool FileFd::Open(string FileName,unsigned int const Mode,APT::Configuration::Compressor const &compressor, unsigned long const AccessMode) -{ - Close(); - Flags = AutoClose; - - if ((Mode & WriteOnly) != WriteOnly && (Mode & (Atomic | Create | Empty | Exclusive)) != 0) - return FileFdError("ReadOnly mode for %s doesn't accept additional flags!", FileName.c_str()); - if ((Mode & ReadWrite) == 0) - return FileFdError("No openmode provided in FileFd::Open for %s", FileName.c_str()); - - if ((Mode & Atomic) == Atomic) +class APT_HIDDEN FileFdPrivate { /*{{{*/ + friend class BufferedWriteFileFdPrivate; +protected: + FileFd * const filefd; + simple_buffer buffer; + int compressed_fd; + pid_t compressor_pid; + bool is_pipe; + APT::Configuration::Compressor compressor; + unsigned int openmode; + unsigned long long seekpos; +public: + + explicit FileFdPrivate(FileFd * const pfilefd) : filefd(pfilefd), + compressed_fd(-1), compressor_pid(-1), is_pipe(false), + openmode(0), seekpos(0) {}; + virtual APT::Configuration::Compressor get_compressor() const { - Flags |= Replace; + return compressor; } - else if ((Mode & (Exclusive | Create)) == (Exclusive | Create)) + virtual void set_compressor(APT::Configuration::Compressor const &compressor) { - // for atomic, this will be done by rename in Close() - unlink(FileName.c_str()); + this->compressor = compressor; } - if ((Mode & Empty) == Empty) + virtual unsigned int get_openmode() const { - struct stat Buf; - if (lstat(FileName.c_str(),&Buf) == 0 && S_ISLNK(Buf.st_mode)) - unlink(FileName.c_str()); + return openmode; + } + virtual void set_openmode(unsigned int openmode) + { + this->openmode = openmode; + } + virtual bool get_is_pipe() const + { + return is_pipe; + } + virtual void set_is_pipe(bool is_pipe) + { + this->is_pipe = is_pipe; + } + virtual unsigned long long get_seekpos() const + { + return seekpos; + } + virtual void set_seekpos(unsigned long long seekpos) + { + this->seekpos = seekpos; } - int fileflags = 0; - #define if_FLAGGED_SET(FLAG, MODE) if ((Mode & FLAG) == FLAG) fileflags |= MODE - if_FLAGGED_SET(ReadWrite, O_RDWR); - else if_FLAGGED_SET(ReadOnly, O_RDONLY); - else if_FLAGGED_SET(WriteOnly, O_WRONLY); + virtual bool InternalOpen(int const iFd, unsigned int const Mode) = 0; + ssize_t InternalRead(void * To, unsigned long long Size) + { + // Drain the buffer if needed. + if (buffer.empty() == false) + { + return buffer.read(To, Size); + } + return InternalUnbufferedRead(To, Size); + } + virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) = 0; + virtual bool InternalReadError() { return filefd->FileFdErrno("read",_("Read error")); } + virtual char * InternalReadLine(char * To, unsigned long long Size) + { + if (unlikely(Size == 0)) + return nullptr; + // Read one byte less than buffer size to have space for trailing 0. + --Size; - if_FLAGGED_SET(Create, O_CREAT); - if_FLAGGED_SET(Empty, O_TRUNC); - if_FLAGGED_SET(Exclusive, O_EXCL); - #undef if_FLAGGED_SET + char * const InitialTo = To; - if ((Mode & Atomic) == Atomic) - { - char *name = strdup((FileName + ".XXXXXX").c_str()); + while (Size > 0) { + if (buffer.empty() == true) + { + buffer.reset(); + unsigned long long actualread = 0; + if (filefd->Read(buffer.get(), buffer.buffersize_max, &actualread) == false) + return nullptr; + buffer.bufferend = actualread; + if (buffer.size() == 0) + { + if (To == InitialTo) + return nullptr; + break; + } + filefd->Flags &= ~FileFd::HitEof; + } - if((iFd = mkstemp(name)) == -1) + unsigned long long const OutputSize = std::min(Size, buffer.size()); + char const * const newline = static_cast(memchr(buffer.get(), '\n', OutputSize)); + // Read until end of line or up to Size bytes from the buffer. + unsigned long long actualread = buffer.read(To, + (newline != nullptr) + ? (newline - buffer.get()) + 1 + : OutputSize); + To += actualread; + Size -= actualread; + if (newline != nullptr) + break; + } + *To = '\0'; + return InitialTo; + } + virtual bool InternalFlush() + { + return true; + } + virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) = 0; + virtual bool InternalWriteError() { return filefd->FileFdErrno("write",_("Write error")); } + virtual bool InternalSeek(unsigned long long const To) + { + // Our poor man seeking is costly, so try to avoid it + unsigned long long const iseekpos = filefd->Tell(); + if (iseekpos == To) + return true; + else if (iseekpos < To) + return filefd->Skip(To - iseekpos); + + if ((openmode & FileFd::ReadOnly) != FileFd::ReadOnly) + return filefd->FileFdError("Reopen is only implemented for read-only files!"); + InternalClose(filefd->FileName); + if (filefd->iFd != -1) + close(filefd->iFd); + filefd->iFd = -1; + if (filefd->TemporaryFileName.empty() == false) + filefd->iFd = open(filefd->TemporaryFileName.c_str(), O_RDONLY); + else if (filefd->FileName.empty() == false) + filefd->iFd = open(filefd->FileName.c_str(), O_RDONLY); + else { - free(name); - return FileFdErrno("mkstemp", "Could not create temporary file for %s", FileName.c_str()); + if (compressed_fd > 0) + if (lseek(compressed_fd, 0, SEEK_SET) != 0) + filefd->iFd = compressed_fd; + if (filefd->iFd < 0) + return filefd->FileFdError("Reopen is not implemented for pipes opened with FileFd::OpenDescriptor()!"); } - TemporaryFileName = string(name); - free(name); + if (filefd->OpenInternDescriptor(openmode, compressor) == false) + return filefd->FileFdError("Seek on file %s because it couldn't be reopened", filefd->FileName.c_str()); - // umask() will always set the umask and return the previous value, so - // we first set the umask and then reset it to the old value - mode_t const CurrentUmask = umask(0); - umask(CurrentUmask); - // calculate the actual file permissions (just like open/creat) - mode_t const FilePermissions = (AccessMode & ~CurrentUmask); + buffer.reset(); + if (To != 0) + return filefd->Skip(To); - if(fchmod(iFd, FilePermissions) == -1) - return FileFdErrno("fchmod", "Could not change permissions for temporary file %s", TemporaryFileName.c_str()); + seekpos = To; + return true; } - else - iFd = open(FileName.c_str(), fileflags, AccessMode); - - this->FileName = FileName; - if (iFd == -1 || OpenInternDescriptor(Mode, compressor) == false) + virtual bool InternalSkip(unsigned long long Over) { - if (iFd != -1) + unsigned long long constexpr buffersize = 1024; + char buffer[buffersize]; + while (Over != 0) { - close (iFd); - iFd = -1; + unsigned long long toread = std::min(buffersize, Over); + if (filefd->Read(buffer, toread) == false) + return filefd->FileFdError("Unable to seek ahead %llu",Over); + Over -= toread; } - return FileFdErrno("open",_("Could not open file %s"), FileName.c_str()); + return true; + } + virtual bool InternalTruncate(unsigned long long const) + { + return filefd->FileFdError("Truncating compressed files is not implemented (%s)", filefd->FileName.c_str()); + } + virtual unsigned long long InternalTell() + { + // In theory, we could just return seekpos here always instead of + // seeking around, but not all users of FileFd use always Seek() and co + // so d->seekpos isn't always true and we can just use it as a hint if + // we have nothing else, but not always as an authority… + return seekpos - buffer.size(); + } + virtual unsigned long long InternalSize() + { + unsigned long long size = 0; + unsigned long long const oldSeek = filefd->Tell(); + unsigned long long constexpr ignoresize = 1024; + char ignore[ignoresize]; + unsigned long long read = 0; + do { + if (filefd->Read(ignore, ignoresize, &read) == false) + { + filefd->Seek(oldSeek); + return 0; + } + } while(read != 0); + size = filefd->Tell(); + filefd->Seek(oldSeek); + return size; } + virtual bool InternalClose(std::string const &FileName) = 0; + virtual bool InternalStream() const { return false; } + virtual bool InternalAlwaysAutoClose() const { return true; } - SetCloseExec(iFd,true); - return true; -} + virtual ~FileFdPrivate() {} +}; /*}}}*/ -// FileFd::OpenDescriptor - Open a filedescriptor /*{{{*/ -// --------------------------------------------------------------------- -/* */ -bool FileFd::OpenDescriptor(int Fd, unsigned int const Mode, CompressMode Compress, bool AutoClose) -{ - std::vector const compressors = APT::Configuration::getCompressors(); - std::vector::const_iterator compressor = compressors.begin(); - std::string name; +class APT_HIDDEN BufferedWriteFileFdPrivate : public FileFdPrivate { /*{{{*/ +protected: + FileFdPrivate *wrapped; + simple_buffer writebuffer; - // compat with the old API - if (Mode == ReadOnlyGzip && Compress == None) - Compress = Gzip; +public: - switch (Compress) + explicit BufferedWriteFileFdPrivate(FileFdPrivate *Priv) : + FileFdPrivate(Priv->filefd), wrapped(Priv) {}; + + virtual APT::Configuration::Compressor get_compressor() const override { - case None: name = "."; break; - case Gzip: name = "gzip"; break; - case Bzip2: name = "bzip2"; break; - case Lzma: name = "lzma"; break; - case Xz: name = "xz"; break; - case Auto: - case Extension: - if (AutoClose == true && Fd != -1) - close(Fd); - return FileFdError("Opening Fd %d in Auto or Extension compression mode is not supported", Fd); + return wrapped->get_compressor(); } - for (; compressor != compressors.end(); ++compressor) - if (compressor->Name == name) - break; - if (compressor == compressors.end()) + virtual void set_compressor(APT::Configuration::Compressor const &compressor) override { - if (AutoClose == true && Fd != -1) - close(Fd); - return FileFdError("Can't find a configured compressor %s for file %s", name.c_str(), FileName.c_str()); + return wrapped->set_compressor(compressor); } - return OpenDescriptor(Fd, Mode, *compressor, AutoClose); -} -bool FileFd::OpenDescriptor(int Fd, unsigned int const Mode, APT::Configuration::Compressor const &compressor, bool AutoClose) -{ - Close(); - Flags = (AutoClose) ? FileFd::AutoClose : 0; - iFd = Fd; - this->FileName = ""; - if (OpenInternDescriptor(Mode, compressor) == false) + virtual unsigned int get_openmode() const override { - if (iFd != -1 && ( - (Flags & Compressed) == Compressed || - AutoClose == true)) - { - close (iFd); - iFd = -1; - } - return FileFdError(_("Could not open file descriptor %d"), Fd); + return wrapped->get_openmode(); } - return true; -} -bool FileFd::OpenInternDescriptor(unsigned int const Mode, APT::Configuration::Compressor const &compressor) -{ - if (iFd == -1) - return false; - if (compressor.Name == "." || compressor.Binary.empty() == true) + virtual void set_openmode(unsigned int openmode) override + { + return wrapped->set_openmode(openmode); + } + virtual bool get_is_pipe() const override + { + return wrapped->get_is_pipe(); + } + virtual void set_is_pipe(bool is_pipe) override + { + FileFdPrivate::set_is_pipe(is_pipe); + wrapped->set_is_pipe(is_pipe); + } + virtual unsigned long long get_seekpos() const override + { + return wrapped->get_seekpos(); + } + virtual void set_seekpos(unsigned long long seekpos) override + { + return wrapped->set_seekpos(seekpos); + } + virtual bool InternalOpen(int const iFd, unsigned int const Mode) override + { + if (InternalFlush() == false) + return false; + return wrapped->InternalOpen(iFd, Mode); + } + virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) override + { + if (InternalFlush() == false) + return -1; + return wrapped->InternalUnbufferedRead(To, Size); + + } + virtual bool InternalReadError() override + { + return wrapped->InternalReadError(); + } + virtual char * InternalReadLine(char * To, unsigned long long Size) override + { + if (InternalFlush() == false) + return nullptr; + return wrapped->InternalReadLine(To, Size); + } + virtual bool InternalFlush() override + { + while (writebuffer.empty() == false) { + auto written = wrapped->InternalWrite(writebuffer.get(), + writebuffer.size()); + // Ignore interrupted syscalls + if (written < 0 && errno == EINTR) + continue; + if (written < 0) + return false; + + writebuffer.bufferstart += written; + } + + writebuffer.reset(); return true; + } + virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override + { + size_t written = 0; + + while (written < Size) { + auto buffered = writebuffer.write(static_cast(From) + written, Size - written); + + written += buffered; -#if defined HAVE_ZLIB || defined HAVE_BZ2 || defined HAVE_LZMA - // the API to open files is similar, so setup to avoid code duplicates later - // and while at it ensure that we close before opening (if its a reopen) - void* (*compress_open)(int, const char *) = NULL; - if (false) - /* dummy so that the rest can be 'else if's */; -#define APT_COMPRESS_INIT(NAME,OPEN) \ - else if (compressor.Name == NAME) \ - { \ - compress_open = (void*(*)(int, const char *)) OPEN; \ - if (d != NULL) d->InternalClose(FileName); \ + if (writebuffer.full() && InternalFlush() == false) + return -1; + } + + return written; + } + virtual bool InternalWriteError() + { + return wrapped->InternalWriteError(); + } + virtual bool InternalSeek(unsigned long long const To) + { + if (InternalFlush() == false) + return false; + return wrapped->InternalSeek(To); + } + virtual bool InternalSkip(unsigned long long Over) + { + if (InternalFlush() == false) + return false; + return wrapped->InternalSkip(Over); + } + virtual bool InternalTruncate(unsigned long long const Size) + { + if (InternalFlush() == false) + return false; + return wrapped->InternalTruncate(Size); + } + virtual unsigned long long InternalTell() + { + if (InternalFlush() == false) + return -1; + return wrapped->InternalTell(); + } + virtual unsigned long long InternalSize() + { + if (InternalFlush() == false) + return -1; + return wrapped->InternalSize(); + } + virtual bool InternalClose(std::string const &FileName) + { + return wrapped->InternalClose(FileName); + } + virtual bool InternalAlwaysAutoClose() const + { + return wrapped->InternalAlwaysAutoClose(); + } + virtual ~BufferedWriteFileFdPrivate() + { + delete wrapped; } +}; + /*}}}*/ +class APT_HIDDEN GzipFileFdPrivate: public FileFdPrivate { /*{{{*/ #ifdef HAVE_ZLIB - APT_COMPRESS_INIT("gzip", gzdopen) -#endif -#ifdef HAVE_BZ2 - APT_COMPRESS_INIT("bzip2", BZ2_bzdopen) -#endif -#ifdef HAVE_LZMA - APT_COMPRESS_INIT("xz", fdopen) - APT_COMPRESS_INIT("lzma", fdopen) -#endif -#undef APT_COMPRESS_INIT -#endif - - if (d == NULL) +public: + gzFile gz; + virtual bool InternalOpen(int const iFd, unsigned int const Mode) override + { + if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite) + gz = gzdopen(iFd, "r+"); + else if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly) + gz = gzdopen(iFd, "w"); + else + gz = gzdopen(iFd, "r"); + filefd->Flags |= FileFd::Compressed; + return gz != nullptr; + } + virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) override + { + return gzread(gz, To, Size); + } + virtual bool InternalReadError() override + { + int err; + char const * const errmsg = gzerror(gz, &err); + if (err != Z_ERRNO) + return filefd->FileFdError("gzread: %s (%d: %s)", _("Read error"), err, errmsg); + return FileFdPrivate::InternalReadError(); + } + virtual char * InternalReadLine(char * To, unsigned long long Size) override + { + return gzgets(gz, To, Size); + } + virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override + { + return gzwrite(gz,From,Size); + } + virtual bool InternalWriteError() override { - d = new FileFdPrivate(); - d->openmode = Mode; - d->compressor = compressor; -#if defined HAVE_ZLIB || defined HAVE_BZ2 || defined HAVE_LZMA - if ((Flags & AutoClose) != AutoClose && compress_open != NULL) + int err; + char const * const errmsg = gzerror(gz, &err); + if (err != Z_ERRNO) + return filefd->FileFdError("gzwrite: %s (%d: %s)", _("Write error"), err, errmsg); + return FileFdPrivate::InternalWriteError(); + } + virtual bool InternalSeek(unsigned long long const To) override + { + off_t const res = gzseek(gz, To, SEEK_SET); + if (res != (off_t)To) + return filefd->FileFdError("Unable to seek to %llu", To); + seekpos = To; + buffer.reset(); + return true; + } + virtual bool InternalSkip(unsigned long long Over) override + { + if (Over >= buffer.size()) { - // Need to duplicate fd here or gz/bz2 close for cleanup will close the fd as well - int const internFd = dup(iFd); - if (internFd == -1) - return FileFdErrno("OpenInternDescriptor", _("Could not open file descriptor %d"), iFd); - iFd = internFd; + Over -= buffer.size(); + buffer.reset(); } -#endif + else + { + buffer.bufferstart += Over; + return true; + } + if (Over == 0) + return true; + off_t const res = gzseek(gz, Over, SEEK_CUR); + if (res < 0) + return filefd->FileFdError("Unable to seek ahead %llu",Over); + seekpos = res; + return true; + } + virtual unsigned long long InternalTell() override + { + return gztell(gz) - buffer.size(); } + virtual unsigned long long InternalSize() override + { + unsigned long long filesize = FileFdPrivate::InternalSize(); + // only check gzsize if we are actually a gzip file, just checking for + // "gz" is not sufficient as uncompressed files could be opened with + // gzopen in "direct" mode as well + if (filesize == 0 || gzdirect(gz)) + return filesize; + + off_t const oldPos = lseek(filefd->iFd, 0, SEEK_CUR); + /* unfortunately zlib.h doesn't provide a gzsize(), so we have to do + * this ourselves; the original (uncompressed) file size is the last 32 + * bits of the file */ + // FIXME: Size for gz-files is limited by 32bit… no largefile support + if (lseek(filefd->iFd, -4, SEEK_END) < 0) + { + filefd->FileFdErrno("lseek","Unable to seek to end of gzipped file"); + return 0; + } + uint32_t size = 0; + if (read(filefd->iFd, &size, 4) != 4) + { + filefd->FileFdErrno("read","Unable to read original size of gzipped file"); + return 0; + } + size = le32toh(size); -#if defined HAVE_ZLIB || defined HAVE_BZ2 || defined HAVE_LZMA - if (compress_open != NULL) + if (lseek(filefd->iFd, oldPos, SEEK_SET) < 0) + { + filefd->FileFdErrno("lseek","Unable to seek in gzipped file"); + return 0; + } + return size; + } + virtual bool InternalClose(std::string const &FileName) override { - void* compress_struct = NULL; - if ((Mode & ReadWrite) == ReadWrite) - compress_struct = compress_open(iFd, "r+"); - else if ((Mode & WriteOnly) == WriteOnly) - compress_struct = compress_open(iFd, "w"); - else - compress_struct = compress_open(iFd, "r"); - if (compress_struct == NULL) - return false; + if (gz == nullptr) + return true; + int const e = gzclose(gz); + gz = nullptr; + // gzdclose() on empty files always fails with "buffer error" here, ignore that + if (e != 0 && e != Z_BUF_ERROR) + return _error->Errno("close",_("Problem closing the gzip file %s"), FileName.c_str()); + return true; + } - if (false) - /* dummy so that the rest can be 'else if's */; -#ifdef HAVE_ZLIB - else if (compressor.Name == "gzip") - d->gz = (gzFile) compress_struct; + explicit GzipFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), gz(nullptr) {} + virtual ~GzipFileFdPrivate() { InternalClose(""); } #endif +}; + /*}}}*/ +class APT_HIDDEN Bz2FileFdPrivate: public FileFdPrivate { /*{{{*/ #ifdef HAVE_BZ2 - else if (compressor.Name == "bzip2") - d->bz2 = (BZFILE*) compress_struct; + BZFILE* bz2; +public: + virtual bool InternalOpen(int const iFd, unsigned int const Mode) override + { + if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite) + bz2 = BZ2_bzdopen(iFd, "r+"); + else if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly) + bz2 = BZ2_bzdopen(iFd, "w"); + else + bz2 = BZ2_bzdopen(iFd, "r"); + filefd->Flags |= FileFd::Compressed; + return bz2 != nullptr; + } + virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) override + { + return BZ2_bzread(bz2, To, Size); + } + virtual bool InternalReadError() override + { + int err; + char const * const errmsg = BZ2_bzerror(bz2, &err); + if (err != BZ_IO_ERROR) + return filefd->FileFdError("BZ2_bzread: %s %s (%d: %s)", filefd->FileName.c_str(), _("Read error"), err, errmsg); + return FileFdPrivate::InternalReadError(); + } + virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override + { + return BZ2_bzwrite(bz2, (void*)From, Size); + } + virtual bool InternalWriteError() override + { + int err; + char const * const errmsg = BZ2_bzerror(bz2, &err); + if (err != BZ_IO_ERROR) + return filefd->FileFdError("BZ2_bzwrite: %s %s (%d: %s)", filefd->FileName.c_str(), _("Write error"), err, errmsg); + return FileFdPrivate::InternalWriteError(); + } + virtual bool InternalStream() const override { return true; } + virtual bool InternalClose(std::string const &) override + { + if (bz2 == nullptr) + return true; + BZ2_bzclose(bz2); + bz2 = nullptr; + return true; + } + + explicit Bz2FileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), bz2(nullptr) {} + virtual ~Bz2FileFdPrivate() { InternalClose(""); } #endif -#ifdef HAVE_LZMA - else if (compressor.Name == "xz" || compressor.Name == "lzma") +}; + /*}}}*/ +class APT_HIDDEN Lz4FileFdPrivate: public FileFdPrivate { /*{{{*/ + static constexpr unsigned long long LZ4_HEADER_SIZE = 19; + static constexpr unsigned long long LZ4_FOOTER_SIZE = 4; +#ifdef HAVE_LZ4 + LZ4F_decompressionContext_t dctx; + LZ4F_compressionContext_t cctx; + LZ4F_errorCode_t res; + FileFd backend; + simple_buffer lz4_buffer; + // Count of bytes that the decompressor expects to read next, or buffer size. + size_t next_to_load = APT_BUFFER_SIZE; +public: + virtual bool InternalOpen(int const iFd, unsigned int const Mode) override + { + if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite) + return _error->Error("lz4 only supports write or read mode"); + + if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly) { + res = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION); + lz4_buffer.reset(LZ4F_compressBound(APT_BUFFER_SIZE, nullptr) + + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE); + } else { + res = LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION); + lz4_buffer.reset(APT_BUFFER_SIZE); + } + + filefd->Flags |= FileFd::Compressed; + + if (LZ4F_isError(res)) + return false; + + unsigned int flags = (Mode & (FileFd::WriteOnly|FileFd::ReadOnly)); + if (backend.OpenDescriptor(iFd, flags) == false) + return false; + + // Write the file header + if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly) { - uint32_t const xzlevel = 6; - uint64_t const memlimit = UINT64_MAX; - if (d->lzma == NULL) - d->lzma = new FileFdPrivate::LZMAFILE; - d->lzma->file = (FILE*) compress_struct; - lzma_stream tmp_stream = LZMA_STREAM_INIT; - d->lzma->stream = tmp_stream; + res = LZ4F_compressBegin(cctx, lz4_buffer.buffer, lz4_buffer.buffersize_max, nullptr); + if (LZ4F_isError(res) || backend.Write(lz4_buffer.buffer, res) == false) + return false; + } + + return true; + } + virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) override + { + /* Keep reading as long as the compressor still wants to read */ + while (next_to_load) { + // Fill compressed buffer; + if (lz4_buffer.empty()) { + unsigned long long read; + /* Reset - if LZ4 decompressor wants to read more, allocate more */ + lz4_buffer.reset(next_to_load); + if (backend.Read(lz4_buffer.getend(), lz4_buffer.free(), &read) == false) + return -1; + lz4_buffer.bufferend += read; + + /* Expected EOF */ + if (read == 0) { + res = -1; + return filefd->FileFdError("LZ4F: %s %s", + filefd->FileName.c_str(), + _("Unexpected end of file")), -1; + } + } + // Drain compressed buffer as far as possible. + size_t in = lz4_buffer.size(); + size_t out = Size; + + res = LZ4F_decompress(dctx, To, &out, lz4_buffer.get(), &in, nullptr); + if (LZ4F_isError(res)) + return -1; + + next_to_load = res; + lz4_buffer.bufferstart += in; + + if (out != 0) + return out; + } + + return 0; + } + virtual bool InternalReadError() override + { + char const * const errmsg = LZ4F_getErrorName(res); + + return filefd->FileFdError("LZ4F: %s %s (%zu: %s)", filefd->FileName.c_str(), _("Read error"), res, errmsg); + } + virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override + { + unsigned long long const towrite = std::min(APT_BUFFER_SIZE, Size); + + res = LZ4F_compressUpdate(cctx, + lz4_buffer.buffer, lz4_buffer.buffersize_max, + From, towrite, nullptr); + + if (LZ4F_isError(res) || backend.Write(lz4_buffer.buffer, res) == false) + return -1; + + return towrite; + } + virtual bool InternalWriteError() override + { + char const * const errmsg = LZ4F_getErrorName(res); + + return filefd->FileFdError("LZ4F: %s %s (%zu: %s)", filefd->FileName.c_str(), _("Write error"), res, errmsg); + } + virtual bool InternalStream() const override { return true; } + + virtual bool InternalFlush() override + { + return backend.Flush(); + } + + virtual bool InternalClose(std::string const &) override + { + /* Reset variables */ + res = 0; + next_to_load = APT_BUFFER_SIZE; - if ((Mode & ReadWrite) == ReadWrite) - return FileFdError("ReadWrite mode is not supported for file %s", FileName.c_str()); + if (cctx != nullptr) + { + res = LZ4F_compressEnd(cctx, lz4_buffer.buffer, lz4_buffer.buffersize_max, nullptr); + if (LZ4F_isError(res) || backend.Write(lz4_buffer.buffer, res) == false) + return false; + if (!backend.Flush()) + return false; + if (!backend.Close()) + return false; + + res = LZ4F_freeCompressionContext(cctx); + cctx = nullptr; + } + + if (dctx != nullptr) + { + res = LZ4F_freeDecompressionContext(dctx); + dctx = nullptr; + } + + return LZ4F_isError(res) == false; + } - if ((Mode & WriteOnly) == WriteOnly) + explicit Lz4FileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), dctx(nullptr), cctx(nullptr) {} + virtual ~Lz4FileFdPrivate() { + InternalClose(""); + } +#endif +}; + /*}}}*/ +class APT_HIDDEN LzmaFileFdPrivate: public FileFdPrivate { /*{{{*/ +#ifdef HAVE_LZMA + struct LZMAFILE { + FILE* file; + uint8_t buffer[4096]; + lzma_stream stream; + lzma_ret err; + bool eof; + bool compressing; + + LZMAFILE() : file(nullptr), eof(false), compressing(false) { buffer[0] = '\0'; } + ~LZMAFILE() + { + if (compressing == true) { - if (compressor.Name == "xz") - { - if (lzma_easy_encoder(&d->lzma->stream, xzlevel, LZMA_CHECK_CRC32) != LZMA_OK) - return false; - } - else + size_t constexpr buffersize = sizeof(buffer)/sizeof(buffer[0]); + while(true) { - lzma_options_lzma options; - lzma_lzma_preset(&options, xzlevel); - if (lzma_alone_encoder(&d->lzma->stream, &options) != LZMA_OK) - return false; + stream.avail_out = buffersize; + stream.next_out = buffer; + err = lzma_code(&stream, LZMA_FINISH); + if (err != LZMA_OK && err != LZMA_STREAM_END) + { + _error->Error("~LZMAFILE: Compress finalisation failed"); + break; + } + size_t const n = buffersize - stream.avail_out; + if (n && fwrite(buffer, 1, n, file) != n) + { + _error->Errno("~LZMAFILE",_("Write error")); + break; + } + if (err == LZMA_STREAM_END) + break; } - d->lzma->compressing = true; } - else + lzma_end(&stream); + fclose(file); + } + }; + LZMAFILE* lzma; + static uint32_t findXZlevel(std::vector const &Args) + { + for (auto a = Args.rbegin(); a != Args.rend(); ++a) + if (a->empty() == false && (*a)[0] == '-' && (*a)[1] != '-') { - if (compressor.Name == "xz") - { - if (lzma_auto_decoder(&d->lzma->stream, memlimit, 0) != LZMA_OK) - return false; - } - else + auto const number = a->find_last_of("0123456789"); + if (number == std::string::npos) + continue; + auto const extreme = a->find("e", number); + uint32_t level = (extreme != std::string::npos) ? LZMA_PRESET_EXTREME : 0; + switch ((*a)[number]) { - if (lzma_alone_decoder(&d->lzma->stream, memlimit) != LZMA_OK) - return false; + case '0': return level | 0; + case '1': return level | 1; + case '2': return level | 2; + case '3': return level | 3; + case '4': return level | 4; + case '5': return level | 5; + case '6': return level | 6; + case '7': return level | 7; + case '8': return level | 8; + case '9': return level | 9; } - d->lzma->compressing = false; } - } -#endif - Flags |= Compressed; - return true; + return 6; } -#endif - - // collect zombies here in case we reopen - if (d->compressor_pid > 0) - ExecWait(d->compressor_pid, "FileFdCompressor", true); +public: + virtual bool InternalOpen(int const iFd, unsigned int const Mode) override + { + if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite) + return filefd->FileFdError("ReadWrite mode is not supported for lzma/xz files %s", filefd->FileName.c_str()); - if ((Mode & ReadWrite) == ReadWrite) - return FileFdError("ReadWrite mode is not supported for file %s", FileName.c_str()); + if (lzma == nullptr) + lzma = new LzmaFileFdPrivate::LZMAFILE; + if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly) + lzma->file = fdopen(iFd, "w"); + else + lzma->file = fdopen(iFd, "r"); + filefd->Flags |= FileFd::Compressed; + if (lzma->file == nullptr) + return false; - bool const Comp = (Mode & WriteOnly) == WriteOnly; - if (Comp == false) - { - // Handle 'decompression' of empty files - struct stat Buf; - fstat(iFd, &Buf); - if (Buf.st_size == 0 && S_ISFIFO(Buf.st_mode) == false) - return true; + lzma_stream tmp_stream = LZMA_STREAM_INIT; + lzma->stream = tmp_stream; - // We don't need the file open - instead let the compressor open it - // as he properly knows better how to efficiently read from 'his' file - if (FileName.empty() == false) + if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly) { - close(iFd); - iFd = -1; + uint32_t const xzlevel = findXZlevel(compressor.CompressArgs); + if (compressor.Name == "xz") + { + if (lzma_easy_encoder(&lzma->stream, xzlevel, LZMA_CHECK_CRC64) != LZMA_OK) + return false; + } + else + { + lzma_options_lzma options; + lzma_lzma_preset(&options, xzlevel); + if (lzma_alone_encoder(&lzma->stream, &options) != LZMA_OK) + return false; + } + lzma->compressing = true; + } + else + { + uint64_t const memlimit = UINT64_MAX; + if (compressor.Name == "xz") + { + if (lzma_auto_decoder(&lzma->stream, memlimit, 0) != LZMA_OK) + return false; + } + else + { + if (lzma_alone_decoder(&lzma->stream, memlimit) != LZMA_OK) + return false; + } + lzma->compressing = false; } + return true; } + virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) override + { + ssize_t Res; + if (lzma->eof == true) + return 0; - // Create a data pipe - int Pipe[2] = {-1,-1}; - if (pipe(Pipe) != 0) - return FileFdErrno("pipe",_("Failed to create subprocess IPC")); - for (int J = 0; J != 2; J++) - SetCloseExec(Pipe[J],true); - - d->compressed_fd = iFd; - d->pipe = true; + lzma->stream.next_out = (uint8_t *) To; + lzma->stream.avail_out = Size; + if (lzma->stream.avail_in == 0) + { + lzma->stream.next_in = lzma->buffer; + lzma->stream.avail_in = fread(lzma->buffer, 1, sizeof(lzma->buffer)/sizeof(lzma->buffer[0]), lzma->file); + } + lzma->err = lzma_code(&lzma->stream, LZMA_RUN); + if (lzma->err == LZMA_STREAM_END) + { + lzma->eof = true; + Res = Size - lzma->stream.avail_out; + } + else if (lzma->err != LZMA_OK) + { + Res = -1; + errno = 0; + } + else + { + Res = Size - lzma->stream.avail_out; + if (Res == 0) + { + // lzma run was okay, but produced no output… + Res = -1; + errno = EINTR; + } + } + return Res; + } + virtual bool InternalReadError() override + { + return filefd->FileFdError("lzma_read: %s (%d)", _("Read error"), lzma->err); + } + virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override + { + lzma->stream.next_in = (uint8_t *)From; + lzma->stream.avail_in = Size; + lzma->stream.next_out = lzma->buffer; + lzma->stream.avail_out = sizeof(lzma->buffer)/sizeof(lzma->buffer[0]); + lzma->err = lzma_code(&lzma->stream, LZMA_RUN); + if (lzma->err != LZMA_OK) + return -1; + size_t const n = sizeof(lzma->buffer)/sizeof(lzma->buffer[0]) - lzma->stream.avail_out; + size_t const m = (n == 0) ? 0 : fwrite(lzma->buffer, 1, n, lzma->file); + if (m != n) + return -1; + else + return Size - lzma->stream.avail_in; + } + virtual bool InternalWriteError() override + { + return filefd->FileFdError("lzma_write: %s (%d)", _("Write error"), lzma->err); + } + virtual bool InternalStream() const override { return true; } + virtual bool InternalClose(std::string const &) override + { + delete lzma; + lzma = nullptr; + return true; + } + + explicit LzmaFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), lzma(nullptr) {} + virtual ~LzmaFileFdPrivate() { InternalClose(""); } +#endif +}; + /*}}}*/ +class APT_HIDDEN PipedFileFdPrivate: public FileFdPrivate /*{{{*/ +/* if we don't have a specific class dealing with library calls, we (un)compress + by executing a specified binary and pipe in/out what we need */ +{ +public: + virtual bool InternalOpen(int const, unsigned int const Mode) override + { + // collect zombies here in case we reopen + if (compressor_pid > 0) + ExecWait(compressor_pid, "FileFdCompressor", true); + + if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite) + return filefd->FileFdError("ReadWrite mode is not supported for file %s", filefd->FileName.c_str()); + + bool const Comp = (Mode & FileFd::WriteOnly) == FileFd::WriteOnly; + if (Comp == false) + { + // Handle 'decompression' of empty files + struct stat Buf; + fstat(filefd->iFd, &Buf); + if (Buf.st_size == 0 && S_ISFIFO(Buf.st_mode) == false) + return true; + + // We don't need the file open - instead let the compressor open it + // as he properly knows better how to efficiently read from 'his' file + if (filefd->FileName.empty() == false) + { + close(filefd->iFd); + filefd->iFd = -1; + } + } + + // Create a data pipe + int Pipe[2] = {-1,-1}; + if (pipe(Pipe) != 0) + return filefd->FileFdErrno("pipe",_("Failed to create subprocess IPC")); + for (int J = 0; J != 2; J++) + SetCloseExec(Pipe[J],true); + + compressed_fd = filefd->iFd; + set_is_pipe(true); + + if (Comp == true) + filefd->iFd = Pipe[1]; + else + filefd->iFd = Pipe[0]; + + // The child.. + compressor_pid = ExecFork(); + if (compressor_pid == 0) + { + if (Comp == true) + { + dup2(compressed_fd,STDOUT_FILENO); + dup2(Pipe[0],STDIN_FILENO); + } + else + { + if (compressed_fd != -1) + dup2(compressed_fd,STDIN_FILENO); + dup2(Pipe[1],STDOUT_FILENO); + } + int const nullfd = open("/dev/null", O_WRONLY); + if (nullfd != -1) + { + dup2(nullfd,STDERR_FILENO); + close(nullfd); + } + + SetCloseExec(STDOUT_FILENO,false); + SetCloseExec(STDIN_FILENO,false); + + std::vector Args; + Args.push_back(compressor.Binary.c_str()); + std::vector const * const addArgs = + (Comp == true) ? &(compressor.CompressArgs) : &(compressor.UncompressArgs); + for (std::vector::const_iterator a = addArgs->begin(); + a != addArgs->end(); ++a) + Args.push_back(a->c_str()); + if (Comp == false && filefd->FileName.empty() == false) + { + // commands not needing arguments, do not need to be told about using standard output + // in reality, only testcases with tools like cat, rev, rot13, … are able to trigger this + if (compressor.CompressArgs.empty() == false && compressor.UncompressArgs.empty() == false) + Args.push_back("--stdout"); + if (filefd->TemporaryFileName.empty() == false) + Args.push_back(filefd->TemporaryFileName.c_str()); + else + Args.push_back(filefd->FileName.c_str()); + } + Args.push_back(NULL); + + execvp(Args[0],(char **)&Args[0]); + cerr << _("Failed to exec compressor ") << Args[0] << endl; + _exit(100); + } + if (Comp == true) + close(Pipe[0]); + else + close(Pipe[1]); + + return true; + } + virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) override + { + return read(filefd->iFd, To, Size); + } + virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override + { + return write(filefd->iFd, From, Size); + } + virtual bool InternalClose(std::string const &) override + { + bool Ret = true; + if (compressor_pid > 0) + Ret &= ExecWait(compressor_pid, "FileFdCompressor", true); + compressor_pid = -1; + return Ret; + } + explicit PipedFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd) {} + virtual ~PipedFileFdPrivate() { InternalClose(""); } +}; + /*}}}*/ +class APT_HIDDEN DirectFileFdPrivate: public FileFdPrivate /*{{{*/ +{ +public: + virtual bool InternalOpen(int const, unsigned int const) override { return true; } + virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) override + { + return read(filefd->iFd, To, Size); + } + virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override + { + // files opened read+write are strange and only really "supported" for direct files + if (buffer.size() != 0) + { + lseek(filefd->iFd, -buffer.size(), SEEK_CUR); + buffer.reset(); + } + return write(filefd->iFd, From, Size); + } + virtual bool InternalSeek(unsigned long long const To) override + { + off_t const res = lseek(filefd->iFd, To, SEEK_SET); + if (res != (off_t)To) + return filefd->FileFdError("Unable to seek to %llu", To); + seekpos = To; + buffer.reset(); + return true; + } + virtual bool InternalSkip(unsigned long long Over) override + { + if (Over >= buffer.size()) + { + Over -= buffer.size(); + buffer.reset(); + } + else + { + buffer.bufferstart += Over; + return true; + } + if (Over == 0) + return true; + off_t const res = lseek(filefd->iFd, Over, SEEK_CUR); + if (res < 0) + return filefd->FileFdError("Unable to seek ahead %llu",Over); + seekpos = res; + return true; + } + virtual bool InternalTruncate(unsigned long long const To) override + { + if (buffer.size() != 0) + { + unsigned long long const seekpos = lseek(filefd->iFd, 0, SEEK_CUR); + if ((seekpos - buffer.size()) >= To) + buffer.reset(); + else if (seekpos >= To) + buffer.bufferend = (To - seekpos) + buffer.bufferstart; + else + buffer.reset(); + } + if (ftruncate(filefd->iFd, To) != 0) + return filefd->FileFdError("Unable to truncate to %llu",To); + return true; + } + virtual unsigned long long InternalTell() override + { + return lseek(filefd->iFd,0,SEEK_CUR) - buffer.size(); + } + virtual unsigned long long InternalSize() override + { + return filefd->FileSize(); + } + virtual bool InternalClose(std::string const &) override { return true; } + virtual bool InternalAlwaysAutoClose() const override { return false; } + + explicit DirectFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd) {} + virtual ~DirectFileFdPrivate() { InternalClose(""); } +}; + /*}}}*/ +// FileFd Constructors /*{{{*/ +FileFd::FileFd(std::string FileName,unsigned int const Mode,unsigned long AccessMode) : iFd(-1), Flags(0), d(NULL) +{ + Open(FileName,Mode, None, AccessMode); +} +FileFd::FileFd(std::string FileName,unsigned int const Mode, CompressMode Compress, unsigned long AccessMode) : iFd(-1), Flags(0), d(NULL) +{ + Open(FileName,Mode, Compress, AccessMode); +} +FileFd::FileFd() : iFd(-1), Flags(AutoClose), d(NULL) {} +FileFd::FileFd(int const Fd, unsigned int const Mode, CompressMode Compress) : iFd(-1), Flags(0), d(NULL) +{ + OpenDescriptor(Fd, Mode, Compress); +} +FileFd::FileFd(int const Fd, bool const AutoClose) : iFd(-1), Flags(0), d(NULL) +{ + OpenDescriptor(Fd, ReadWrite, None, AutoClose); +} + /*}}}*/ +// FileFd::Open - Open a file /*{{{*/ +// --------------------------------------------------------------------- +/* The most commonly used open mode combinations are given with Mode */ +bool FileFd::Open(string FileName,unsigned int const Mode,CompressMode Compress, unsigned long const AccessMode) +{ + if (Mode == ReadOnlyGzip) + return Open(FileName, ReadOnly, Gzip, AccessMode); + + if (Compress == Auto && (Mode & WriteOnly) == WriteOnly) + return FileFdError("Autodetection on %s only works in ReadOnly openmode!", FileName.c_str()); + + std::vector const compressors = APT::Configuration::getCompressors(); + std::vector::const_iterator compressor = compressors.begin(); + if (Compress == Auto) + { + for (; compressor != compressors.end(); ++compressor) + { + std::string file = FileName + compressor->Extension; + if (FileExists(file) == false) + continue; + FileName = file; + break; + } + } + else if (Compress == Extension) + { + std::string::size_type const found = FileName.find_last_of('.'); + std::string ext; + if (found != std::string::npos) + { + ext = FileName.substr(found); + if (ext == ".new" || ext == ".bak") + { + std::string::size_type const found2 = FileName.find_last_of('.', found - 1); + if (found2 != std::string::npos) + ext = FileName.substr(found2, found - found2); + else + ext.clear(); + } + } + for (; compressor != compressors.end(); ++compressor) + if (ext == compressor->Extension) + break; + // no matching extension - assume uncompressed (imagine files like 'example.org_Packages') + if (compressor == compressors.end()) + for (compressor = compressors.begin(); compressor != compressors.end(); ++compressor) + if (compressor->Name == ".") + break; + } + else + { + std::string name; + switch (Compress) + { + case None: name = "."; break; + case Gzip: name = "gzip"; break; + case Bzip2: name = "bzip2"; break; + case Lzma: name = "lzma"; break; + case Xz: name = "xz"; break; + case Lz4: name = "lz4"; break; + case Auto: + case Extension: + // Unreachable + return FileFdError("Opening File %s in None, Auto or Extension should be already handled?!?", FileName.c_str()); + } + for (; compressor != compressors.end(); ++compressor) + if (compressor->Name == name) + break; + if (compressor == compressors.end()) + return FileFdError("Can't find a configured compressor %s for file %s", name.c_str(), FileName.c_str()); + } + + if (compressor == compressors.end()) + return FileFdError("Can't find a match for specified compressor mode for file %s", FileName.c_str()); + return Open(FileName, Mode, *compressor, AccessMode); +} +bool FileFd::Open(string FileName,unsigned int const Mode,APT::Configuration::Compressor const &compressor, unsigned long const AccessMode) +{ + Close(); + Flags = AutoClose; + + if ((Mode & WriteOnly) != WriteOnly && (Mode & (Atomic | Create | Empty | Exclusive)) != 0) + return FileFdError("ReadOnly mode for %s doesn't accept additional flags!", FileName.c_str()); + if ((Mode & ReadWrite) == 0) + return FileFdError("No openmode provided in FileFd::Open for %s", FileName.c_str()); + + unsigned int OpenMode = Mode; + if (FileName == "/dev/null") + OpenMode = OpenMode & ~(Atomic | Exclusive | Create | Empty); + + if ((OpenMode & Atomic) == Atomic) + { + Flags |= Replace; + } + else if ((OpenMode & (Exclusive | Create)) == (Exclusive | Create)) + { + // for atomic, this will be done by rename in Close() + RemoveFile("FileFd::Open", FileName); + } + if ((OpenMode & Empty) == Empty) + { + struct stat Buf; + if (lstat(FileName.c_str(),&Buf) == 0 && S_ISLNK(Buf.st_mode)) + RemoveFile("FileFd::Open", FileName); + } + + int fileflags = 0; + #define if_FLAGGED_SET(FLAG, MODE) if ((OpenMode & FLAG) == FLAG) fileflags |= MODE + if_FLAGGED_SET(ReadWrite, O_RDWR); + else if_FLAGGED_SET(ReadOnly, O_RDONLY); + else if_FLAGGED_SET(WriteOnly, O_WRONLY); + + if_FLAGGED_SET(Create, O_CREAT); + if_FLAGGED_SET(Empty, O_TRUNC); + if_FLAGGED_SET(Exclusive, O_EXCL); + #undef if_FLAGGED_SET + + if ((OpenMode & Atomic) == Atomic) + { + char *name = strdup((FileName + ".XXXXXX").c_str()); + + if((iFd = mkstemp(name)) == -1) + { + free(name); + return FileFdErrno("mkstemp", "Could not create temporary file for %s", FileName.c_str()); + } + + TemporaryFileName = string(name); + free(name); + + // umask() will always set the umask and return the previous value, so + // we first set the umask and then reset it to the old value + mode_t const CurrentUmask = umask(0); + umask(CurrentUmask); + // calculate the actual file permissions (just like open/creat) + mode_t const FilePermissions = (AccessMode & ~CurrentUmask); + + if(fchmod(iFd, FilePermissions) == -1) + return FileFdErrno("fchmod", "Could not change permissions for temporary file %s", TemporaryFileName.c_str()); + } + else + iFd = open(FileName.c_str(), fileflags, AccessMode); + + this->FileName = FileName; + if (iFd == -1 || OpenInternDescriptor(OpenMode, compressor) == false) + { + if (iFd != -1) + { + close (iFd); + iFd = -1; + } + return FileFdErrno("open",_("Could not open file %s"), FileName.c_str()); + } + + SetCloseExec(iFd,true); + return true; +} + /*}}}*/ +// FileFd::OpenDescriptor - Open a filedescriptor /*{{{*/ +bool FileFd::OpenDescriptor(int Fd, unsigned int const Mode, CompressMode Compress, bool AutoClose) +{ + std::vector const compressors = APT::Configuration::getCompressors(); + std::vector::const_iterator compressor = compressors.begin(); + std::string name; + + // compat with the old API + if (Mode == ReadOnlyGzip && Compress == None) + Compress = Gzip; + + switch (Compress) + { + case None: name = "."; break; + case Gzip: name = "gzip"; break; + case Bzip2: name = "bzip2"; break; + case Lzma: name = "lzma"; break; + case Xz: name = "xz"; break; + case Lz4: name = "lz4"; break; + case Auto: + case Extension: + if (AutoClose == true && Fd != -1) + close(Fd); + return FileFdError("Opening Fd %d in Auto or Extension compression mode is not supported", Fd); + } + for (; compressor != compressors.end(); ++compressor) + if (compressor->Name == name) + break; + if (compressor == compressors.end()) + { + if (AutoClose == true && Fd != -1) + close(Fd); + return FileFdError("Can't find a configured compressor %s for file %s", name.c_str(), FileName.c_str()); + } + return OpenDescriptor(Fd, Mode, *compressor, AutoClose); +} +bool FileFd::OpenDescriptor(int Fd, unsigned int const Mode, APT::Configuration::Compressor const &compressor, bool AutoClose) +{ + Close(); + Flags = (AutoClose) ? FileFd::AutoClose : 0; + iFd = Fd; + this->FileName = ""; + if (OpenInternDescriptor(Mode, compressor) == false) + { + if (iFd != -1 && ( + (Flags & Compressed) == Compressed || + AutoClose == true)) + { + close (iFd); + iFd = -1; + } + return FileFdError(_("Could not open file descriptor %d"), Fd); + } + return true; +} +bool FileFd::OpenInternDescriptor(unsigned int const Mode, APT::Configuration::Compressor const &compressor) +{ + if (iFd == -1) + return false; - if (Comp == true) - iFd = Pipe[1]; - else - iFd = Pipe[0]; + if (d != nullptr) + d->InternalClose(FileName); - // The child.. - d->compressor_pid = ExecFork(); - if (d->compressor_pid == 0) + if (d == nullptr) { - if (Comp == true) - { - dup2(d->compressed_fd,STDOUT_FILENO); - dup2(Pipe[0],STDIN_FILENO); - } + if (false) + /* dummy so that the rest can be 'else if's */; +#define APT_COMPRESS_INIT(NAME, CONSTRUCTOR) \ + else if (compressor.Name == NAME) \ + d = new CONSTRUCTOR(this) +#ifdef HAVE_ZLIB + APT_COMPRESS_INIT("gzip", GzipFileFdPrivate); +#endif +#ifdef HAVE_BZ2 + APT_COMPRESS_INIT("bzip2", Bz2FileFdPrivate); +#endif +#ifdef HAVE_LZMA + APT_COMPRESS_INIT("xz", LzmaFileFdPrivate); + APT_COMPRESS_INIT("lzma", LzmaFileFdPrivate); +#endif +#ifdef HAVE_LZ4 + APT_COMPRESS_INIT("lz4", Lz4FileFdPrivate); +#endif +#undef APT_COMPRESS_INIT + else if (compressor.Name == "." || compressor.Binary.empty() == true) + d = new DirectFileFdPrivate(this); else - { - if (d->compressed_fd != -1) - dup2(d->compressed_fd,STDIN_FILENO); - dup2(Pipe[1],STDOUT_FILENO); - } - int const nullfd = open("/dev/null", O_WRONLY); - if (nullfd != -1) - { - dup2(nullfd,STDERR_FILENO); - close(nullfd); - } + d = new PipedFileFdPrivate(this); - SetCloseExec(STDOUT_FILENO,false); - SetCloseExec(STDIN_FILENO,false); - - std::vector Args; - Args.push_back(compressor.Binary.c_str()); - std::vector const * const addArgs = - (Comp == true) ? &(compressor.CompressArgs) : &(compressor.UncompressArgs); - for (std::vector::const_iterator a = addArgs->begin(); - a != addArgs->end(); ++a) - Args.push_back(a->c_str()); - if (Comp == false && FileName.empty() == false) + if (Mode & BufferedWrite) + d = new BufferedWriteFileFdPrivate(d); + + d->set_openmode(Mode); + d->set_compressor(compressor); + if ((Flags & AutoClose) != AutoClose && d->InternalAlwaysAutoClose()) { - // commands not needing arguments, do not need to be told about using standard output - // in reality, only testcases with tools like cat, rev, rot13, … are able to trigger this - if (compressor.CompressArgs.empty() == false && compressor.UncompressArgs.empty() == false) - Args.push_back("--stdout"); - if (TemporaryFileName.empty() == false) - Args.push_back(TemporaryFileName.c_str()); - else - Args.push_back(FileName.c_str()); + // Need to duplicate fd here or gz/bz2 close for cleanup will close the fd as well + int const internFd = dup(iFd); + if (internFd == -1) + return FileFdErrno("OpenInternDescriptor", _("Could not open file descriptor %d"), iFd); + iFd = internFd; } - Args.push_back(NULL); - - execvp(Args[0],(char **)&Args[0]); - cerr << _("Failed to exec compressor ") << Args[0] << endl; - _exit(100); } - if (Comp == true) - close(Pipe[0]); - else - close(Pipe[1]); - - return true; + return d->InternalOpen(iFd, Mode); } /*}}}*/ // FileFd::~File - Closes the file /*{{{*/ @@ -1432,7 +2338,7 @@ FileFd::~FileFd() { Close(); if (d != NULL) - d->CloseDown(FileName); + d->InternalClose(FileName); delete d; d = NULL; } @@ -1443,61 +2349,16 @@ FileFd::~FileFd() gracefully. */ bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual) { - ssize_t Res; + if (d == nullptr) + return false; + ssize_t Res = 1; errno = 0; if (Actual != 0) *Actual = 0; *((char *)To) = '\0'; - do + while (Res > 0 && Size > 0) { - if (false) - /* dummy so that the rest can be 'else if's */; -#ifdef HAVE_ZLIB - else if (d != NULL && d->gz != NULL) - Res = gzread(d->gz,To,Size); -#endif -#ifdef HAVE_BZ2 - else if (d != NULL && d->bz2 != NULL) - Res = BZ2_bzread(d->bz2,To,Size); -#endif -#ifdef HAVE_LZMA - else if (d != NULL && d->lzma != NULL) - { - if (d->lzma->eof == true) - break; - - d->lzma->stream.next_out = (uint8_t *) To; - d->lzma->stream.avail_out = Size; - if (d->lzma->stream.avail_in == 0) - { - d->lzma->stream.next_in = d->lzma->buffer; - d->lzma->stream.avail_in = fread(d->lzma->buffer, 1, sizeof(d->lzma->buffer)/sizeof(d->lzma->buffer[0]), d->lzma->file); - } - d->lzma->err = lzma_code(&d->lzma->stream, LZMA_RUN); - if (d->lzma->err == LZMA_STREAM_END) - { - d->lzma->eof = true; - Res = Size - d->lzma->stream.avail_out; - } - else if (d->lzma->err != LZMA_OK) - { - Res = -1; - errno = 0; - } - else - { - Res = Size - d->lzma->stream.avail_out; - if (Res == 0) - { - // lzma run was okay, but produced no output… - Res = -1; - errno = EINTR; - } - } - } -#endif - else - Res = read(iFd,To,Size); + Res = d->InternalRead(To, Size); if (Res < 0) { @@ -1508,41 +2369,16 @@ bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual) errno = 0; continue; } - if (false) - /* dummy so that the rest can be 'else if's */; -#ifdef HAVE_ZLIB - else if (d != NULL && d->gz != NULL) - { - int err; - char const * const errmsg = gzerror(d->gz, &err); - if (err != Z_ERRNO) - return FileFdError("gzread: %s (%d: %s)", _("Read error"), err, errmsg); - } -#endif -#ifdef HAVE_BZ2 - else if (d != NULL && d->bz2 != NULL) - { - int err; - char const * const errmsg = BZ2_bzerror(d->bz2, &err); - if (err != BZ_IO_ERROR) - return FileFdError("BZ2_bzread: %s %s (%d: %s)", FileName.c_str(), _("Read error"), err, errmsg); - } -#endif -#ifdef HAVE_LZMA - else if (d != NULL && d->lzma != NULL) - return FileFdError("lzma_read: %s (%d)", _("Read error"), d->lzma->err); -#endif - return FileFdErrno("read",_("Read error")); + return d->InternalReadError(); } To = (char *)To + Res; Size -= Res; if (d != NULL) - d->seekpos += Res; + d->set_seekpos(d->get_seekpos() + Res); if (Actual != 0) *Actual += Res; } - while (Res > 0 && Size > 0); if (Size == 0) return true; @@ -1559,111 +2395,46 @@ bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual) /*}}}*/ // FileFd::ReadLine - Read a complete line from the file /*{{{*/ // --------------------------------------------------------------------- -/* Beware: This method can be quiet slow for big buffers on UNcompressed +/* Beware: This method can be quite slow for big buffers on UNcompressed files because of the naive implementation! */ char* FileFd::ReadLine(char *To, unsigned long long const Size) { *To = '\0'; -#ifdef HAVE_ZLIB - if (d != NULL && d->gz != NULL) - return gzgets(d->gz, To, Size); -#endif + if (d == nullptr) + return nullptr; + return d->InternalReadLine(To, Size); +} + /*}}}*/ +// FileFd::Flush - Flush the file /*{{{*/ +bool FileFd::Flush() +{ + if (d == nullptr) + return true; - unsigned long long read = 0; - while ((Size - 1) != read) - { - unsigned long long done = 0; - if (Read(To + read, 1, &done) == false) - return NULL; - if (done == 0) - break; - if (To[read++] == '\n') - break; - } - if (read == 0) - return NULL; - To[read] = '\0'; - return To; + return d->InternalFlush(); } /*}}}*/ // FileFd::Write - Write to the file /*{{{*/ -// --------------------------------------------------------------------- -/* */ bool FileFd::Write(const void *From,unsigned long long Size) { - ssize_t Res; + if (d == nullptr) + return false; + ssize_t Res = 1; errno = 0; - do + while (Res > 0 && Size > 0) { - if (false) - /* dummy so that the rest can be 'else if's */; -#ifdef HAVE_ZLIB - else if (d != NULL && d->gz != NULL) - Res = gzwrite(d->gz,From,Size); -#endif -#ifdef HAVE_BZ2 - else if (d != NULL && d->bz2 != NULL) - Res = BZ2_bzwrite(d->bz2,(void*)From,Size); -#endif -#ifdef HAVE_LZMA - else if (d != NULL && d->lzma != NULL) - { - d->lzma->stream.next_in = (uint8_t *)From; - d->lzma->stream.avail_in = Size; - d->lzma->stream.next_out = d->lzma->buffer; - d->lzma->stream.avail_out = sizeof(d->lzma->buffer)/sizeof(d->lzma->buffer[0]); - d->lzma->err = lzma_code(&d->lzma->stream, LZMA_RUN); - if (d->lzma->err != LZMA_OK) - return false; - size_t const n = sizeof(d->lzma->buffer)/sizeof(d->lzma->buffer[0]) - d->lzma->stream.avail_out; - size_t const m = (n == 0) ? 0 : fwrite(d->lzma->buffer, 1, n, d->lzma->file); - if (m != n) - Res = -1; - else - Res = Size - d->lzma->stream.avail_in; - } -#endif - else - Res = write(iFd,From,Size); - + Res = d->InternalWrite(From, Size); if (Res < 0 && errno == EINTR) continue; if (Res < 0) - { - if (false) - /* dummy so that the rest can be 'else if's */; -#ifdef HAVE_ZLIB - else if (d != NULL && d->gz != NULL) - { - int err; - char const * const errmsg = gzerror(d->gz, &err); - if (err != Z_ERRNO) - return FileFdError("gzwrite: %s (%d: %s)", _("Write error"), err, errmsg); - } -#endif -#ifdef HAVE_BZ2 - else if (d != NULL && d->bz2 != NULL) - { - int err; - char const * const errmsg = BZ2_bzerror(d->bz2, &err); - if (err != BZ_IO_ERROR) - return FileFdError("BZ2_bzwrite: %s (%d: %s)", _("Write error"), err, errmsg); - } -#endif -#ifdef HAVE_LZMA - else if (d != NULL && d->lzma != NULL) - return FileFdErrno("lzma_fwrite", _("Write error")); -#endif - return FileFdErrno("write",_("Write error")); - } - + return d->InternalWriteError(); + From = (char const *)From + Res; Size -= Res; if (d != NULL) - d->seekpos += Res; + d->set_seekpos(d->get_seekpos() + Res); } - while (Res > 0 && Size > 0); - + if (Size == 0) return true; @@ -1671,9 +2442,9 @@ bool FileFd::Write(const void *From,unsigned long long Size) } bool FileFd::Write(int Fd, const void *From, unsigned long long Size) { - ssize_t Res; + ssize_t Res = 1; errno = 0; - do + while (Res > 0 && Size > 0) { Res = write(Fd,From,Size); if (Res < 0 && errno == EINTR) @@ -1684,7 +2455,6 @@ bool FileFd::Write(int Fd, const void *From, unsigned long long Size) From = (char const *)From + Res; Size -= Res; } - while (Res > 0 && Size > 0); if (Size == 0) return true; @@ -1693,117 +2463,31 @@ bool FileFd::Write(int Fd, const void *From, unsigned long long Size) } /*}}}*/ // FileFd::Seek - Seek in the file /*{{{*/ -// --------------------------------------------------------------------- -/* */ bool FileFd::Seek(unsigned long long To) { + if (d == nullptr) + return false; Flags &= ~HitEof; - - if (d != NULL && (d->pipe == true || d->InternalStream() == true)) - { - // Our poor man seeking in pipes is costly, so try to avoid it - unsigned long long seekpos = Tell(); - if (seekpos == To) - return true; - else if (seekpos < To) - return Skip(To - seekpos); - - if ((d->openmode & ReadOnly) != ReadOnly) - return FileFdError("Reopen is only implemented for read-only files!"); - d->InternalClose(FileName); - if (iFd != -1) - close(iFd); - iFd = -1; - if (TemporaryFileName.empty() == false) - iFd = open(TemporaryFileName.c_str(), O_RDONLY); - else if (FileName.empty() == false) - iFd = open(FileName.c_str(), O_RDONLY); - else - { - if (d->compressed_fd > 0) - if (lseek(d->compressed_fd, 0, SEEK_SET) != 0) - iFd = d->compressed_fd; - if (iFd < 0) - return FileFdError("Reopen is not implemented for pipes opened with FileFd::OpenDescriptor()!"); - } - - if (OpenInternDescriptor(d->openmode, d->compressor) == false) - return FileFdError("Seek on file %s because it couldn't be reopened", FileName.c_str()); - - if (To != 0) - return Skip(To); - - d->seekpos = To; - return true; - } - off_t res; -#ifdef HAVE_ZLIB - if (d != NULL && d->gz) - res = gzseek(d->gz,To,SEEK_SET); - else -#endif - res = lseek(iFd,To,SEEK_SET); - if (res != (off_t)To) - return FileFdError("Unable to seek to %llu", To); - - if (d != NULL) - d->seekpos = To; - return true; + return d->InternalSeek(To); } /*}}}*/ -// FileFd::Skip - Seek in the file /*{{{*/ -// --------------------------------------------------------------------- -/* */ +// FileFd::Skip - Skip over data in the file /*{{{*/ bool FileFd::Skip(unsigned long long Over) { - if (d != NULL && (d->pipe == true || d->InternalStream() == true)) - { - char buffer[1024]; - while (Over != 0) - { - unsigned long long toread = std::min((unsigned long long) sizeof(buffer), Over); - if (Read(buffer, toread) == false) - return FileFdError("Unable to seek ahead %llu",Over); - Over -= toread; - } - return true; - } - - off_t res; -#ifdef HAVE_ZLIB - if (d != NULL && d->gz != NULL) - res = gzseek(d->gz,Over,SEEK_CUR); - else -#endif - res = lseek(iFd,Over,SEEK_CUR); - if (res < 0) - return FileFdError("Unable to seek ahead %llu",Over); - if (d != NULL) - d->seekpos = res; - - return true; + if (d == nullptr) + return false; + return d->InternalSkip(Over); } /*}}}*/ -// FileFd::Truncate - Truncate the file /*{{{*/ -// --------------------------------------------------------------------- -/* */ +// FileFd::Truncate - Truncate the file /*{{{*/ bool FileFd::Truncate(unsigned long long To) { + if (d == nullptr) + return false; // truncating /dev/null is always successful - as we get an error otherwise if (To == 0 && FileName == "/dev/null") return true; -#if defined HAVE_ZLIB || defined HAVE_BZ2 || defined HAVE_LZMA - if (d != NULL && (d->InternalStream() == true -#ifdef HAVE_ZLIB - || d->gz != NULL -#endif - )) - return FileFdError("Truncating compressed files is not implemented (%s)", FileName.c_str()); -#endif - if (ftruncate(iFd,To) != 0) - return FileFdError("Unable to truncate to %llu",To); - - return true; + return d->InternalTruncate(To); } /*}}}*/ // FileFd::Tell - Current seek position /*{{{*/ @@ -1811,30 +2495,18 @@ bool FileFd::Truncate(unsigned long long To) /* */ unsigned long long FileFd::Tell() { - // In theory, we could just return seekpos here always instead of - // seeking around, but not all users of FileFd use always Seek() and co - // so d->seekpos isn't always true and we can just use it as a hint if - // we have nothing else, but not always as an authority… - if (d != NULL && (d->pipe == true || d->InternalStream() == true)) - return d->seekpos; - - off_t Res; -#ifdef HAVE_ZLIB - if (d != NULL && d->gz != NULL) - Res = gztell(d->gz); - else -#endif - Res = lseek(iFd,0,SEEK_CUR); + if (d == nullptr) + return false; + off_t const Res = d->InternalTell(); if (Res == (off_t)-1) FileFdErrno("lseek","Failed to determine the current file position"); - if (d != NULL) - d->seekpos = Res; + d->set_seekpos(Res); return Res; } /*}}}*/ static bool StatFileFd(char const * const msg, int const iFd, std::string const &FileName, struct stat &Buf, FileFdPrivate * const d) /*{{{*/ { - bool ispipe = (d != NULL && d->pipe == true); + bool ispipe = (d != NULL && d->get_is_pipe() == true); if (ispipe == false) { if (fstat(iFd,&Buf) != 0) @@ -1851,7 +2523,7 @@ static bool StatFileFd(char const * const msg, int const iFd, std::string const // we set it here, too, as we get the info here for free // in theory the Open-methods should take care of it already if (d != NULL) - d->pipe = true; + d->set_is_pipe(true); if (stat(FileName.c_str(), &Buf) != 0) return _error->Errno("fstat", "Unable to determine %s for file %s", msg, FileName.c_str()); } @@ -1883,64 +2555,11 @@ time_t FileFd::ModificationTime() } /*}}}*/ // FileFd::Size - Return the size of the content in the file /*{{{*/ -// --------------------------------------------------------------------- -/* */ unsigned long long FileFd::Size() { - unsigned long long size = FileSize(); - - // for compressor pipes st_size is undefined and at 'best' zero, - // so we 'read' the content and 'seek' back - see there - if (d != NULL && (d->pipe == true || (d->InternalStream() == true && size > 0))) - { - unsigned long long const oldSeek = Tell(); - char ignore[1000]; - unsigned long long read = 0; - do { - if (Read(ignore, sizeof(ignore), &read) == false) - { - Seek(oldSeek); - return 0; - } - } while(read != 0); - size = Tell(); - Seek(oldSeek); - } -#ifdef HAVE_ZLIB - // only check gzsize if we are actually a gzip file, just checking for - // "gz" is not sufficient as uncompressed files could be opened with - // gzopen in "direct" mode as well - else if (d != NULL && d->gz && !gzdirect(d->gz) && size > 0) - { - off_t const oldPos = lseek(iFd,0,SEEK_CUR); - /* unfortunately zlib.h doesn't provide a gzsize(), so we have to do - * this ourselves; the original (uncompressed) file size is the last 32 - * bits of the file */ - // FIXME: Size for gz-files is limited by 32bit… no largefile support - if (lseek(iFd, -4, SEEK_END) < 0) - { - FileFdErrno("lseek","Unable to seek to end of gzipped file"); - return 0; - } - uint32_t size = 0; - if (read(iFd, &size, 4) != 4) - { - FileFdErrno("read","Unable to read original size of gzipped file"); - return 0; - } - size = le32toh(size); - - if (lseek(iFd, oldPos, SEEK_SET) < 0) - { - FileFdErrno("lseek","Unable to seek in gzipped file"); - return 0; - } - - return size; - } -#endif - - return size; + if (d == nullptr) + return false; + return d->InternalSize(); } /*}}}*/ // FileFd::Close - Close the file if the close flag is set /*{{{*/ @@ -1948,6 +2567,8 @@ unsigned long long FileFd::Size() /* */ bool FileFd::Close() { + if (Flush() == false) + return false; if (iFd == -1) return true; @@ -1956,12 +2577,13 @@ bool FileFd::Close() { if ((Flags & Compressed) != Compressed && iFd > 0 && close(iFd) != 0) Res &= _error->Errno("close",_("Problem closing the file %s"), FileName.c_str()); - if (d != NULL) - { - Res &= d->CloseDown(FileName); - delete d; - d = NULL; - } + } + + if (d != NULL) + { + Res &= d->InternalClose(FileName); + delete d; + d = NULL; } if ((Flags & Replace) == Replace) { @@ -1976,8 +2598,7 @@ bool FileFd::Close() if ((Flags & Fail) == Fail && (Flags & DelOnFail) == DelOnFail && FileName.empty() == false) - if (unlink(FileName.c_str()) != 0) - Res &= _error->WarningE("unlnk",_("Problem unlinking the file %s"), FileName.c_str()); + Res &= RemoveFile("FileFd::Close", FileName); if (Res == false) Flags |= Fail; @@ -2026,19 +2647,20 @@ bool FileFd::FileFdError(const char *Description,...) { return false; } /*}}}*/ - -APT_DEPRECATED gzFile FileFd::gzFd() { +gzFile FileFd::gzFd() { /*{{{*/ #ifdef HAVE_ZLIB - return d->gz; + GzipFileFdPrivate * const gzipd = dynamic_cast(d); + if (gzipd == nullptr) + return nullptr; + else + return gzipd->gz; #else - return NULL; + return nullptr; #endif } + /*}}}*/ - -// Glob - wrapper around "glob()" /*{{{*/ -// --------------------------------------------------------------------- -/* */ +// Glob - wrapper around "glob()" /*{{{*/ std::vector Glob(std::string const &pattern, int flags) { std::vector result; @@ -2064,8 +2686,7 @@ std::vector Glob(std::string const &pattern, int flags) return result; } /*}}}*/ - -std::string GetTempDir() +std::string GetTempDir() /*{{{*/ { const char *tmpdir = getenv("TMPDIR"); @@ -2074,51 +2695,79 @@ std::string GetTempDir() tmpdir = P_tmpdir; #endif - // check that tmpdir is set and exists struct stat st; - if (!tmpdir || strlen(tmpdir) == 0 || stat(tmpdir, &st) != 0) + if (!tmpdir || strlen(tmpdir) == 0 || // tmpdir is set + stat(tmpdir, &st) != 0 || (st.st_mode & S_IFDIR) == 0) // exists and is directory + tmpdir = "/tmp"; + else if (geteuid() != 0 && // root can do everything anyway + faccessat(-1, tmpdir, R_OK | W_OK | X_OK, AT_EACCESS | AT_SYMLINK_NOFOLLOW) != 0) // current user has rwx access to directory tmpdir = "/tmp"; return string(tmpdir); } +std::string GetTempDir(std::string const &User) +{ + // no need/possibility to drop privs + if(getuid() != 0 || User.empty() || User == "root") + return GetTempDir(); + + struct passwd const * const pw = getpwnam(User.c_str()); + if (pw == NULL) + return GetTempDir(); -FileFd* GetTempFile(std::string const &Prefix, bool ImmediateUnlink) + gid_t const old_euid = geteuid(); + gid_t const old_egid = getegid(); + if (setegid(pw->pw_gid) != 0) + _error->Errno("setegid", "setegid %u failed", pw->pw_gid); + if (seteuid(pw->pw_uid) != 0) + _error->Errno("seteuid", "seteuid %u failed", pw->pw_uid); + + std::string const tmp = GetTempDir(); + + if (seteuid(old_euid) != 0) + _error->Errno("seteuid", "seteuid %u failed", old_euid); + if (setegid(old_egid) != 0) + _error->Errno("setegid", "setegid %u failed", old_egid); + + return tmp; +} + /*}}}*/ +FileFd* GetTempFile(std::string const &Prefix, bool ImmediateUnlink, FileFd * const TmpFd) /*{{{*/ { char fn[512]; - FileFd *Fd = new FileFd(); + FileFd * const Fd = TmpFd == NULL ? new FileFd() : TmpFd; - std::string tempdir = GetTempDir(); - snprintf(fn, sizeof(fn), "%s/%s.XXXXXX", + std::string const tempdir = GetTempDir(); + snprintf(fn, sizeof(fn), "%s/%s.XXXXXX", tempdir.c_str(), Prefix.c_str()); - int fd = mkstemp(fn); + int const fd = mkstemp(fn); if(ImmediateUnlink) unlink(fn); - if (fd < 0) + if (fd < 0) { _error->Errno("GetTempFile",_("Unable to mkstemp %s"), fn); return NULL; } - if (!Fd->OpenDescriptor(fd, FileFd::WriteOnly, FileFd::None, true)) + if (!Fd->OpenDescriptor(fd, FileFd::ReadWrite, FileFd::None, true)) { _error->Errno("GetTempFile",_("Unable to write to %s"),fn); return NULL; } - return Fd; } - -bool Rename(std::string From, std::string To) + /*}}}*/ +bool Rename(std::string From, std::string To) /*{{{*/ { if (rename(From.c_str(),To.c_str()) != 0) { _error->Error(_("rename failed, %s (%s -> %s)."),strerror(errno), From.c_str(),To.c_str()); return false; - } + } return true; } - -bool Popen(const char* Args[], FileFd &Fd, pid_t &Child, FileFd::OpenMode Mode) + /*}}}*/ +bool Popen(const char* Args[], FileFd &Fd, pid_t &Child, FileFd::OpenMode Mode)/*{{{*/ { int fd; if (Mode != FileFd::ReadOnly && Mode != FileFd::WriteOnly) @@ -2161,101 +2810,149 @@ bool Popen(const char* Args[], FileFd &Fd, pid_t &Child, FileFd::OpenMode Mode) { close(Pipe[1]); fd = Pipe[0]; - } else if(Mode == FileFd::WriteOnly) + } + else if(Mode == FileFd::WriteOnly) { close(Pipe[0]); fd = Pipe[1]; } + else + return _error->Error("Popen supports ReadOnly (x)or WriteOnly mode only"); Fd.OpenDescriptor(fd, Mode, FileFd::None, true); return true; } - -bool DropPrivs() + /*}}}*/ +bool DropPrivileges() /*{{{*/ { + if(_config->FindB("Debug::NoDropPrivs", false) == true) + return true; + +#if __gnu_linux__ +#if defined(PR_SET_NO_NEW_PRIVS) && ( PR_SET_NO_NEW_PRIVS != 38 ) +#error "PR_SET_NO_NEW_PRIVS is defined, but with a different value than expected!" +#endif + // see prctl(2), needs linux3.5 at runtime - magic constant to avoid it at buildtime + int ret = prctl(38, 1, 0, 0, 0); + // ignore EINVAL - kernel is too old to understand the option + if(ret < 0 && errno != EINVAL) + _error->Warning("PR_SET_NO_NEW_PRIVS failed with %i", ret); +#endif + + // empty setting disables privilege dropping - this also ensures + // backward compatibility, see bug #764506 + const std::string toUser = _config->Find("APT::Sandbox::User"); + if (toUser.empty() || toUser == "root") + return true; + + // a lot can go wrong trying to drop privileges completely, + // so ideally we would like to verify that we have done it – + // but the verify asks for too much in case of fakeroot (and alike) + // [Specific checks can be overridden with dedicated options] + bool const VerifySandboxing = _config->FindB("APT::Sandbox::Verify", false); + // uid will be 0 in the end, but gid might be different anyway - uid_t old_uid = getuid(); - gid_t old_gid = getgid(); + uid_t const old_uid = getuid(); + gid_t const old_gid = getgid(); if (old_uid != 0) return true; - if(_config->FindB("Debug::NoDropPrivs", false) == true) - return true; - const std::string toUser = _config->Find("APT::Sandbox::User", "_apt"); struct passwd *pw = getpwnam(toUser.c_str()); if (pw == NULL) return _error->Error("No user %s, can not drop rights", toUser.c_str()); -#if __gnu_linux__ - // see prctl(2), needs linux3.5 - int ret = prctl(PR_SET_NO_NEW_PRIVS, 1, 0,0, 0); - if(ret < 0) - _error->Warning("PR_SET_NO_NEW_PRIVS failed with %i", ret); -#endif // Do not change the order here, it might break things + // Get rid of all our supplementary groups first if (setgroups(1, &pw->pw_gid)) return _error->Errno("setgroups", "Failed to setgroups"); + // Now change the group ids to the new user +#ifdef HAVE_SETRESGID + if (setresgid(pw->pw_gid, pw->pw_gid, pw->pw_gid) != 0) + return _error->Errno("setresgid", "Failed to set new group ids"); +#else if (setegid(pw->pw_gid) != 0) return _error->Errno("setegid", "Failed to setegid"); if (setgid(pw->pw_gid) != 0) return _error->Errno("setgid", "Failed to setgid"); +#endif + // Change the user ids to the new user +#ifdef HAVE_SETRESUID + if (setresuid(pw->pw_uid, pw->pw_uid, pw->pw_uid) != 0) + return _error->Errno("setresuid", "Failed to set new user ids"); +#else if (setuid(pw->pw_uid) != 0) return _error->Errno("setuid", "Failed to setuid"); - - // the seteuid() is probably uneeded (at least thats what the linux - // man-page says about setuid(2)) but we cargo culted it anyway if (seteuid(pw->pw_uid) != 0) return _error->Errno("seteuid", "Failed to seteuid"); +#endif - // Verify that the user has only a single group, and the correct one - gid_t groups[1]; - if (getgroups(1, groups) != 1) - return _error->Errno("getgroups", "Could not get new groups"); - if (groups[0] != pw->pw_gid) - return _error->Error("Could not switch group"); - - // Verify that gid, egid, uid, and euid changed - if (getgid() != pw->pw_gid) - return _error->Error("Could not switch group"); - if (getegid() != pw->pw_gid) - return _error->Error("Could not switch effective group"); - if (getuid() != pw->pw_uid) - return _error->Error("Could not switch user"); - if (geteuid() != pw->pw_uid) - return _error->Error("Could not switch effective user"); + // disabled by default as fakeroot doesn't implement getgroups currently (#806521) + if (VerifySandboxing == true || _config->FindB("APT::Sandbox::Verify::Groups", false) == true) + { + // Verify that the user isn't still in any supplementary groups + long const ngroups_max = sysconf(_SC_NGROUPS_MAX); + std::unique_ptr gidlist(new gid_t[ngroups_max]); + if (unlikely(gidlist == NULL)) + return _error->Error("Allocation of a list of size %lu for getgroups failed", ngroups_max); + ssize_t gidlist_nr; + if ((gidlist_nr = getgroups(ngroups_max, gidlist.get())) < 0) + return _error->Errno("getgroups", "Could not get new groups (%lu)", ngroups_max); + for (ssize_t i = 0; i < gidlist_nr; ++i) + if (gidlist[i] != pw->pw_gid) + return _error->Error("Could not switch group, user %s is still in group %d", toUser.c_str(), gidlist[i]); + } + + // enabled by default as all fakeroot-lookalikes should fake that accordingly + if (VerifySandboxing == true || _config->FindB("APT::Sandbox::Verify::IDs", true) == true) + { + // Verify that gid, egid, uid, and euid changed + if (getgid() != pw->pw_gid) + return _error->Error("Could not switch group"); + if (getegid() != pw->pw_gid) + return _error->Error("Could not switch effective group"); + if (getuid() != pw->pw_uid) + return _error->Error("Could not switch user"); + if (geteuid() != pw->pw_uid) + return _error->Error("Could not switch effective user"); #ifdef HAVE_GETRESUID - // verify that the saved set-user-id was changed as well - uid_t ruid = 0; - uid_t euid = 0; - uid_t suid = 0; - if (getresuid(&ruid, &euid, &suid)) - return _error->Errno("getresuid", "Could not get saved set-user-ID"); - if (suid != pw->pw_uid) - return _error->Error("Could not switch saved set-user-ID"); + // verify that the saved set-user-id was changed as well + uid_t ruid = 0; + uid_t euid = 0; + uid_t suid = 0; + if (getresuid(&ruid, &euid, &suid)) + return _error->Errno("getresuid", "Could not get saved set-user-ID"); + if (suid != pw->pw_uid) + return _error->Error("Could not switch saved set-user-ID"); #endif #ifdef HAVE_GETRESGID - // verify that the saved set-group-id was changed as well - gid_t rgid = 0; - gid_t egid = 0; - gid_t sgid = 0; - if (getresgid(&rgid, &egid, &sgid)) - return _error->Errno("getresuid", "Could not get saved set-group-ID"); - if (sgid != pw->pw_gid) - return _error->Error("Could not switch saved set-group-ID"); + // verify that the saved set-group-id was changed as well + gid_t rgid = 0; + gid_t egid = 0; + gid_t sgid = 0; + if (getresgid(&rgid, &egid, &sgid)) + return _error->Errno("getresuid", "Could not get saved set-group-ID"); + if (sgid != pw->pw_gid) + return _error->Error("Could not switch saved set-group-ID"); #endif + } - // Check that uid and gid changes do not work anymore - if (pw->pw_gid != old_gid && (setgid(old_gid) != -1 || setegid(old_gid) != -1)) - return _error->Error("Could restore a gid to root, privilege dropping did not work"); + // disabled as fakeroot doesn't forbid (by design) (re)gaining root from unprivileged + if (VerifySandboxing == true || _config->FindB("APT::Sandbox::Verify::Regain", false) == true) + { + // Check that uid and gid changes do not work anymore + if (pw->pw_gid != old_gid && (setgid(old_gid) != -1 || setegid(old_gid) != -1)) + return _error->Error("Could restore a gid to root, privilege dropping did not work"); - if (pw->pw_uid != old_uid && (setuid(old_uid) != -1 || seteuid(old_uid) != -1)) - return _error->Error("Could restore a uid to root, privilege dropping did not work"); + if (pw->pw_uid != old_uid && (setuid(old_uid) != -1 || seteuid(old_uid) != -1)) + return _error->Error("Could restore a uid to root, privilege dropping did not work"); + } return true; } + /*}}}*/