#ifdef HAVE_LZMA
#include <lzma.h>
#endif
+#ifdef HAVE_LZ4
+ #include <lz4frame.h>
+#endif
#include <endian.h>
#include <stdint.h>
using namespace std;
+/* Should be a multiple of the common page size (4096) */
+static constexpr unsigned long long APT_BUFFER_SIZE = 64 * 1024;
+
// RunScripts - Run a set of scripts from a configuration subtree /*{{{*/
// ---------------------------------------------------------------------
/* */
if (From.IsOpen() == false || To.IsOpen() == false ||
From.Failed() == true || To.Failed() == true)
return false;
-
+
// Buffered copy between fds
- std::unique_ptr<unsigned char[]> Buf(new unsigned char[64000]);
- unsigned long long Size = From.Size();
- while (Size != 0)
- {
- unsigned long long ToRead = Size;
- if (Size > 64000)
- ToRead = 64000;
-
- if (From.Read(Buf.get(),ToRead) == false ||
+ constexpr size_t BufSize = APT_BUFFER_SIZE;
+ std::unique_ptr<unsigned char[]> Buf(new unsigned char[BufSize]);
+ unsigned long long ToRead = 0;
+ do {
+ if (From.Read(Buf.get(),BufSize, &ToRead) == false ||
To.Write(Buf.get(),ToRead) == false)
return false;
-
- Size -= ToRead;
- }
+ } while (ToRead != 0);
+
+ return true;
+}
+ /*}}}*/
+bool RemoveFile(char const * const Function, std::string const &FileName)/*{{{*/
+{
+ if (FileName == "/dev/null")
+ return true;
+ errno = 0;
+ if (unlink(FileName.c_str()) != 0)
+ {
+ if (errno == ENOENT)
+ return true;
- return true;
+ return _error->WarningE(Function,_("Problem unlinking the file %s"), FileName.c_str());
+ }
+ return true;
}
/*}}}*/
// GetLock - Gets a lock file /*{{{*/
char *p = realpath(File.c_str(), NULL);
if (p == NULL)
{
- _error->Errno("realpath", "flAbsPath failed");
+ _error->Errno("realpath", "flAbsPath on %s failed", File.c_str());
return "";
}
std::string AbsPath(p);
}
/*}}}*/
-class FileFdPrivate { /*{{{*/
- public:
-#ifdef HAVE_ZLIB
- gzFile gz;
-#endif
-#ifdef HAVE_BZ2
- BZFILE* bz2;
-#endif
-#ifdef HAVE_LZMA
- struct LZMAFILE {
- FILE* file;
- uint8_t buffer[4096];
- lzma_stream stream;
- lzma_ret err;
- bool eof;
- bool compressing;
-
- LZMAFILE() : file(NULL), eof(false), compressing(false) { buffer[0] = '\0'; }
- ~LZMAFILE() {
- if (compressing == true)
- {
- for (;;) {
- stream.avail_out = sizeof(buffer)/sizeof(buffer[0]);
- stream.next_out = buffer;
- err = lzma_code(&stream, LZMA_FINISH);
- if (err != LZMA_OK && err != LZMA_STREAM_END)
- {
- _error->Error("~LZMAFILE: Compress finalisation failed");
- break;
- }
- size_t const n = sizeof(buffer)/sizeof(buffer[0]) - stream.avail_out;
- if (n && fwrite(buffer, 1, n, file) != n)
- {
- _error->Errno("~LZMAFILE",_("Write error"));
- break;
- }
- if (err == LZMA_STREAM_END)
- break;
- }
- }
- lzma_end(&stream);
- fclose(file);
- }
- };
- LZMAFILE* lzma;
-#endif
- int compressed_fd;
- pid_t compressor_pid;
- bool pipe;
- APT::Configuration::Compressor compressor;
- unsigned int openmode;
- unsigned long long seekpos;
- FileFdPrivate() :
-#ifdef HAVE_ZLIB
- gz(NULL),
-#endif
-#ifdef HAVE_BZ2
- bz2(NULL),
-#endif
-#ifdef HAVE_LZMA
- lzma(NULL),
-#endif
- compressed_fd(-1), compressor_pid(-1), pipe(false),
- openmode(0), seekpos(0) {};
- bool InternalClose(std::string const &FileName)
- {
- if (false)
- /* dummy so that the rest can be 'else if's */;
-#ifdef HAVE_ZLIB
- else if (gz != NULL) {
- int const e = gzclose(gz);
- gz = NULL;
- // gzdclose() on empty files always fails with "buffer error" here, ignore that
- if (e != 0 && e != Z_BUF_ERROR)
- return _error->Errno("close",_("Problem closing the gzip file %s"), FileName.c_str());
- }
-#endif
-#ifdef HAVE_BZ2
- else if (bz2 != NULL) {
- BZ2_bzclose(bz2);
- bz2 = NULL;
- }
-#endif
-#ifdef HAVE_LZMA
- else if (lzma != NULL) {
- delete lzma;
- lzma = NULL;
- }
-#endif
- return true;
- }
- bool CloseDown(std::string const &FileName)
- {
- bool const Res = InternalClose(FileName);
-
- if (compressor_pid > 0)
- ExecWait(compressor_pid, "FileFdCompressor", true);
- compressor_pid = -1;
-
- return Res;
- }
- bool InternalStream() const {
- return false
-#ifdef HAVE_BZ2
- || bz2 != NULL
-#endif
-#ifdef HAVE_LZMA
- || lzma != NULL
-#endif
- ;
- }
+struct APT_HIDDEN simple_buffer { /*{{{*/
+ size_t buffersize_max = 0;
+ unsigned long long bufferstart = 0;
+ unsigned long long bufferend = 0;
+ char *buffer = nullptr;
+ simple_buffer() {
+ reset(4096);
+ }
+ ~simple_buffer() {
+ delete[] buffer;
+ }
- ~FileFdPrivate() { CloseDown(""); }
+ const char *get() const { return buffer + bufferstart; }
+ char *get() { return buffer + bufferstart; }
+ const char *getend() const { return buffer + bufferend; }
+ char *getend() { return buffer + bufferend; }
+ bool empty() const { return bufferend <= bufferstart; }
+ bool full() const { return bufferend == buffersize_max; }
+ unsigned long long free() const { return buffersize_max - bufferend; }
+ unsigned long long size() const { return bufferend-bufferstart; }
+ void reset(size_t size)
+ {
+ if (size > buffersize_max) {
+ delete[] buffer;
+ buffersize_max = size;
+ buffer = new char[size];
+ }
+ reset();
+ }
+ void reset() { bufferend = bufferstart = 0; }
+ ssize_t read(void *to, unsigned long long requested_size) APT_MUSTCHECK
+ {
+ if (size() < requested_size)
+ requested_size = size();
+ memcpy(to, buffer + bufferstart, requested_size);
+ bufferstart += requested_size;
+ if (bufferstart == bufferend)
+ bufferstart = bufferend = 0;
+ return requested_size;
+ }
+ ssize_t write(const void *from, unsigned long long requested_size) APT_MUSTCHECK
+ {
+ if (free() < requested_size)
+ requested_size = free();
+ memcpy(getend(), from, requested_size);
+ bufferend += requested_size;
+ if (bufferstart == bufferend)
+ bufferstart = bufferend = 0;
+ return requested_size;
+ }
};
/*}}}*/
-// FileFd Constructors /*{{{*/
-FileFd::FileFd(std::string FileName,unsigned int const Mode,unsigned long AccessMode) : iFd(-1), Flags(0), d(NULL)
-{
- Open(FileName,Mode, None, AccessMode);
-}
-FileFd::FileFd(std::string FileName,unsigned int const Mode, CompressMode Compress, unsigned long AccessMode) : iFd(-1), Flags(0), d(NULL)
-{
- Open(FileName,Mode, Compress, AccessMode);
-}
-FileFd::FileFd() : iFd(-1), Flags(AutoClose), d(NULL) {}
-FileFd::FileFd(int const Fd, unsigned int const Mode, CompressMode Compress) : iFd(-1), Flags(0), d(NULL)
-{
- OpenDescriptor(Fd, Mode, Compress);
-}
-FileFd::FileFd(int const Fd, bool const AutoClose) : iFd(-1), Flags(0), d(NULL)
-{
- OpenDescriptor(Fd, ReadWrite, None, AutoClose);
-}
- /*}}}*/
-// FileFd::Open - Open a file /*{{{*/
-// ---------------------------------------------------------------------
-/* The most commonly used open mode combinations are given with Mode */
-bool FileFd::Open(string FileName,unsigned int const Mode,CompressMode Compress, unsigned long const AccessMode)
-{
- if (Mode == ReadOnlyGzip)
- return Open(FileName, ReadOnly, Gzip, AccessMode);
- if (Compress == Auto && (Mode & WriteOnly) == WriteOnly)
- return FileFdError("Autodetection on %s only works in ReadOnly openmode!", FileName.c_str());
-
- std::vector<APT::Configuration::Compressor> const compressors = APT::Configuration::getCompressors();
- std::vector<APT::Configuration::Compressor>::const_iterator compressor = compressors.begin();
- if (Compress == Auto)
+class APT_HIDDEN FileFdPrivate { /*{{{*/
+ friend class BufferedWriteFileFdPrivate;
+protected:
+ FileFd * const filefd;
+ simple_buffer buffer;
+ int compressed_fd;
+ pid_t compressor_pid;
+ bool is_pipe;
+ APT::Configuration::Compressor compressor;
+ unsigned int openmode;
+ unsigned long long seekpos;
+public:
+
+ explicit FileFdPrivate(FileFd * const pfilefd) : filefd(pfilefd),
+ compressed_fd(-1), compressor_pid(-1), is_pipe(false),
+ openmode(0), seekpos(0) {};
+ virtual APT::Configuration::Compressor get_compressor() const
{
- for (; compressor != compressors.end(); ++compressor)
- {
- std::string file = FileName + compressor->Extension;
- if (FileExists(file) == false)
- continue;
- FileName = file;
- break;
- }
+ return compressor;
}
- else if (Compress == Extension)
+ virtual void set_compressor(APT::Configuration::Compressor const &compressor)
{
- std::string::size_type const found = FileName.find_last_of('.');
- std::string ext;
- if (found != std::string::npos)
- {
- ext = FileName.substr(found);
- if (ext == ".new" || ext == ".bak")
- {
- std::string::size_type const found2 = FileName.find_last_of('.', found - 1);
- if (found2 != std::string::npos)
- ext = FileName.substr(found2, found - found2);
- else
- ext.clear();
- }
- }
- for (; compressor != compressors.end(); ++compressor)
- if (ext == compressor->Extension)
- break;
- // no matching extension - assume uncompressed (imagine files like 'example.org_Packages')
- if (compressor == compressors.end())
- for (compressor = compressors.begin(); compressor != compressors.end(); ++compressor)
- if (compressor->Name == ".")
- break;
+ this->compressor = compressor;
}
- else
+ virtual unsigned int get_openmode() const
{
- std::string name;
- switch (Compress)
- {
- case None: name = "."; break;
- case Gzip: name = "gzip"; break;
- case Bzip2: name = "bzip2"; break;
- case Lzma: name = "lzma"; break;
- case Xz: name = "xz"; break;
- case Auto:
- case Extension:
- // Unreachable
- return FileFdError("Opening File %s in None, Auto or Extension should be already handled?!?", FileName.c_str());
- }
- for (; compressor != compressors.end(); ++compressor)
- if (compressor->Name == name)
- break;
- if (compressor == compressors.end())
- return FileFdError("Can't find a configured compressor %s for file %s", name.c_str(), FileName.c_str());
+ return openmode;
}
-
- if (compressor == compressors.end())
- return FileFdError("Can't find a match for specified compressor mode for file %s", FileName.c_str());
- return Open(FileName, Mode, *compressor, AccessMode);
-}
-bool FileFd::Open(string FileName,unsigned int const Mode,APT::Configuration::Compressor const &compressor, unsigned long const AccessMode)
-{
- Close();
- Flags = AutoClose;
-
- if ((Mode & WriteOnly) != WriteOnly && (Mode & (Atomic | Create | Empty | Exclusive)) != 0)
- return FileFdError("ReadOnly mode for %s doesn't accept additional flags!", FileName.c_str());
- if ((Mode & ReadWrite) == 0)
- return FileFdError("No openmode provided in FileFd::Open for %s", FileName.c_str());
-
- if ((Mode & Atomic) == Atomic)
+ virtual void set_openmode(unsigned int openmode)
{
- Flags |= Replace;
+ this->openmode = openmode;
}
- else if ((Mode & (Exclusive | Create)) == (Exclusive | Create))
+ virtual bool get_is_pipe() const
{
- // for atomic, this will be done by rename in Close()
- unlink(FileName.c_str());
+ return is_pipe;
}
- if ((Mode & Empty) == Empty)
+ virtual void set_is_pipe(bool is_pipe)
{
- struct stat Buf;
- if (lstat(FileName.c_str(),&Buf) == 0 && S_ISLNK(Buf.st_mode))
- unlink(FileName.c_str());
+ this->is_pipe = is_pipe;
+ }
+ virtual unsigned long long get_seekpos() const
+ {
+ return seekpos;
+ }
+ virtual void set_seekpos(unsigned long long seekpos)
+ {
+ this->seekpos = seekpos;
}
- int fileflags = 0;
- #define if_FLAGGED_SET(FLAG, MODE) if ((Mode & FLAG) == FLAG) fileflags |= MODE
- if_FLAGGED_SET(ReadWrite, O_RDWR);
- else if_FLAGGED_SET(ReadOnly, O_RDONLY);
- else if_FLAGGED_SET(WriteOnly, O_WRONLY);
+ virtual bool InternalOpen(int const iFd, unsigned int const Mode) = 0;
+ ssize_t InternalRead(void * To, unsigned long long Size)
+ {
+ // Drain the buffer if needed.
+ if (buffer.empty() == false)
+ {
+ return buffer.read(To, Size);
+ }
+ return InternalUnbufferedRead(To, Size);
+ }
+ virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) = 0;
+ virtual bool InternalReadError() { return filefd->FileFdErrno("read",_("Read error")); }
+ virtual char * InternalReadLine(char * To, unsigned long long Size)
+ {
+ if (unlikely(Size == 0))
+ return nullptr;
+ // Read one byte less than buffer size to have space for trailing 0.
+ --Size;
- if_FLAGGED_SET(Create, O_CREAT);
- if_FLAGGED_SET(Empty, O_TRUNC);
- if_FLAGGED_SET(Exclusive, O_EXCL);
- #undef if_FLAGGED_SET
+ char * const InitialTo = To;
- if ((Mode & Atomic) == Atomic)
- {
- char *name = strdup((FileName + ".XXXXXX").c_str());
+ while (Size > 0) {
+ if (buffer.empty() == true)
+ {
+ buffer.reset();
+ unsigned long long actualread = 0;
+ if (filefd->Read(buffer.getend(), buffer.free(), &actualread) == false)
+ return nullptr;
+ buffer.bufferend = actualread;
+ if (buffer.size() == 0)
+ {
+ if (To == InitialTo)
+ return nullptr;
+ break;
+ }
+ filefd->Flags &= ~FileFd::HitEof;
+ }
- if((iFd = mkstemp(name)) == -1)
+ unsigned long long const OutputSize = std::min(Size, buffer.size());
+ char const * const newline = static_cast<char const * const>(memchr(buffer.get(), '\n', OutputSize));
+ // Read until end of line or up to Size bytes from the buffer.
+ unsigned long long actualread = buffer.read(To,
+ (newline != nullptr)
+ ? (newline - buffer.get()) + 1
+ : OutputSize);
+ To += actualread;
+ Size -= actualread;
+ if (newline != nullptr)
+ break;
+ }
+ *To = '\0';
+ return InitialTo;
+ }
+ virtual bool InternalFlush()
+ {
+ return true;
+ }
+ virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) = 0;
+ virtual bool InternalWriteError() { return filefd->FileFdErrno("write",_("Write error")); }
+ virtual bool InternalSeek(unsigned long long const To)
+ {
+ // Our poor man seeking is costly, so try to avoid it
+ unsigned long long const iseekpos = filefd->Tell();
+ if (iseekpos == To)
+ return true;
+ else if (iseekpos < To)
+ return filefd->Skip(To - iseekpos);
+
+ if ((openmode & FileFd::ReadOnly) != FileFd::ReadOnly)
+ return filefd->FileFdError("Reopen is only implemented for read-only files!");
+ InternalClose(filefd->FileName);
+ if (filefd->iFd != -1)
+ close(filefd->iFd);
+ filefd->iFd = -1;
+ if (filefd->TemporaryFileName.empty() == false)
+ filefd->iFd = open(filefd->TemporaryFileName.c_str(), O_RDONLY);
+ else if (filefd->FileName.empty() == false)
+ filefd->iFd = open(filefd->FileName.c_str(), O_RDONLY);
+ else
{
- free(name);
- return FileFdErrno("mkstemp", "Could not create temporary file for %s", FileName.c_str());
+ if (compressed_fd > 0)
+ if (lseek(compressed_fd, 0, SEEK_SET) != 0)
+ filefd->iFd = compressed_fd;
+ if (filefd->iFd < 0)
+ return filefd->FileFdError("Reopen is not implemented for pipes opened with FileFd::OpenDescriptor()!");
}
- TemporaryFileName = string(name);
- free(name);
+ if (filefd->OpenInternDescriptor(openmode, compressor) == false)
+ return filefd->FileFdError("Seek on file %s because it couldn't be reopened", filefd->FileName.c_str());
- // umask() will always set the umask and return the previous value, so
- // we first set the umask and then reset it to the old value
- mode_t const CurrentUmask = umask(0);
- umask(CurrentUmask);
- // calculate the actual file permissions (just like open/creat)
- mode_t const FilePermissions = (AccessMode & ~CurrentUmask);
+ buffer.reset();
+ set_seekpos(0);
+ if (To != 0)
+ return filefd->Skip(To);
- if(fchmod(iFd, FilePermissions) == -1)
- return FileFdErrno("fchmod", "Could not change permissions for temporary file %s", TemporaryFileName.c_str());
+ seekpos = To;
+ return true;
}
- else
- iFd = open(FileName.c_str(), fileflags, AccessMode);
-
- this->FileName = FileName;
- if (iFd == -1 || OpenInternDescriptor(Mode, compressor) == false)
+ virtual bool InternalSkip(unsigned long long Over)
{
- if (iFd != -1)
+ unsigned long long constexpr buffersize = 1024;
+ char buffer[buffersize];
+ while (Over != 0)
{
- close (iFd);
- iFd = -1;
+ unsigned long long toread = std::min(buffersize, Over);
+ if (filefd->Read(buffer, toread) == false)
+ return filefd->FileFdError("Unable to seek ahead %llu",Over);
+ Over -= toread;
}
- return FileFdErrno("open",_("Could not open file %s"), FileName.c_str());
+ return true;
+ }
+ virtual bool InternalTruncate(unsigned long long const)
+ {
+ return filefd->FileFdError("Truncating compressed files is not implemented (%s)", filefd->FileName.c_str());
+ }
+ virtual unsigned long long InternalTell()
+ {
+ // In theory, we could just return seekpos here always instead of
+ // seeking around, but not all users of FileFd use always Seek() and co
+ // so d->seekpos isn't always true and we can just use it as a hint if
+ // we have nothing else, but not always as an authority…
+ return seekpos - buffer.size();
+ }
+ virtual unsigned long long InternalSize()
+ {
+ unsigned long long size = 0;
+ unsigned long long const oldSeek = filefd->Tell();
+ unsigned long long constexpr ignoresize = 1024;
+ char ignore[ignoresize];
+ unsigned long long read = 0;
+ do {
+ if (filefd->Read(ignore, ignoresize, &read) == false)
+ {
+ filefd->Seek(oldSeek);
+ return 0;
+ }
+ } while(read != 0);
+ size = filefd->Tell();
+ filefd->Seek(oldSeek);
+ return size;
}
+ virtual bool InternalClose(std::string const &FileName) = 0;
+ virtual bool InternalStream() const { return false; }
+ virtual bool InternalAlwaysAutoClose() const { return true; }
- SetCloseExec(iFd,true);
- return true;
-}
+ virtual ~FileFdPrivate() {}
+};
/*}}}*/
-// FileFd::OpenDescriptor - Open a filedescriptor /*{{{*/
-// ---------------------------------------------------------------------
-/* */
-bool FileFd::OpenDescriptor(int Fd, unsigned int const Mode, CompressMode Compress, bool AutoClose)
-{
- std::vector<APT::Configuration::Compressor> const compressors = APT::Configuration::getCompressors();
- std::vector<APT::Configuration::Compressor>::const_iterator compressor = compressors.begin();
- std::string name;
+class APT_HIDDEN BufferedWriteFileFdPrivate : public FileFdPrivate { /*{{{*/
+protected:
+ FileFdPrivate *wrapped;
+ simple_buffer writebuffer;
- // compat with the old API
- if (Mode == ReadOnlyGzip && Compress == None)
- Compress = Gzip;
+public:
- switch (Compress)
+ explicit BufferedWriteFileFdPrivate(FileFdPrivate *Priv) :
+ FileFdPrivate(Priv->filefd), wrapped(Priv) {};
+
+ virtual APT::Configuration::Compressor get_compressor() const override
{
- case None: name = "."; break;
- case Gzip: name = "gzip"; break;
- case Bzip2: name = "bzip2"; break;
- case Lzma: name = "lzma"; break;
- case Xz: name = "xz"; break;
- case Auto:
- case Extension:
- if (AutoClose == true && Fd != -1)
- close(Fd);
- return FileFdError("Opening Fd %d in Auto or Extension compression mode is not supported", Fd);
+ return wrapped->get_compressor();
}
- for (; compressor != compressors.end(); ++compressor)
- if (compressor->Name == name)
- break;
- if (compressor == compressors.end())
+ virtual void set_compressor(APT::Configuration::Compressor const &compressor) override
{
- if (AutoClose == true && Fd != -1)
- close(Fd);
- return FileFdError("Can't find a configured compressor %s for file %s", name.c_str(), FileName.c_str());
+ return wrapped->set_compressor(compressor);
+ }
+ virtual unsigned int get_openmode() const override
+ {
+ return wrapped->get_openmode();
+ }
+ virtual void set_openmode(unsigned int openmode) override
+ {
+ return wrapped->set_openmode(openmode);
+ }
+ virtual bool get_is_pipe() const override
+ {
+ return wrapped->get_is_pipe();
+ }
+ virtual void set_is_pipe(bool is_pipe) override
+ {
+ FileFdPrivate::set_is_pipe(is_pipe);
+ wrapped->set_is_pipe(is_pipe);
+ }
+ virtual unsigned long long get_seekpos() const override
+ {
+ return wrapped->get_seekpos();
+ }
+ virtual void set_seekpos(unsigned long long seekpos) override
+ {
+ return wrapped->set_seekpos(seekpos);
+ }
+ virtual bool InternalOpen(int const iFd, unsigned int const Mode) override
+ {
+ if (InternalFlush() == false)
+ return false;
+ return wrapped->InternalOpen(iFd, Mode);
+ }
+ virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) override
+ {
+ if (InternalFlush() == false)
+ return -1;
+ return wrapped->InternalUnbufferedRead(To, Size);
+
+ }
+ virtual bool InternalReadError() override
+ {
+ return wrapped->InternalReadError();
+ }
+ virtual char * InternalReadLine(char * To, unsigned long long Size) override
+ {
+ if (InternalFlush() == false)
+ return nullptr;
+ return wrapped->InternalReadLine(To, Size);
+ }
+ virtual bool InternalFlush() override
+ {
+ while (writebuffer.empty() == false) {
+ auto written = wrapped->InternalWrite(writebuffer.get(),
+ writebuffer.size());
+ // Ignore interrupted syscalls
+ if (written < 0 && errno == EINTR)
+ continue;
+ if (written < 0)
+ return wrapped->InternalWriteError();
+
+ writebuffer.bufferstart += written;
+ }
+
+ writebuffer.reset();
+ return true;
+ }
+ virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override
+ {
+ // Optimisation: If the buffer is empty and we have more to write than
+ // would fit in the buffer (or equal number of bytes), write directly.
+ if (writebuffer.empty() == true && Size >= writebuffer.free())
+ return wrapped->InternalWrite(From, Size);
+
+ // Write as much into the buffer as possible and then flush if needed
+ auto written = writebuffer.write(From, Size);
+
+ if (writebuffer.full() && InternalFlush() == false)
+ return -1;
+
+ return written;
+ }
+ virtual bool InternalWriteError()
+ {
+ return wrapped->InternalWriteError();
+ }
+ virtual bool InternalSeek(unsigned long long const To)
+ {
+ if (InternalFlush() == false)
+ return false;
+ return wrapped->InternalSeek(To);
+ }
+ virtual bool InternalSkip(unsigned long long Over)
+ {
+ if (InternalFlush() == false)
+ return false;
+ return wrapped->InternalSkip(Over);
+ }
+ virtual bool InternalTruncate(unsigned long long const Size)
+ {
+ if (InternalFlush() == false)
+ return false;
+ return wrapped->InternalTruncate(Size);
+ }
+ virtual unsigned long long InternalTell()
+ {
+ if (InternalFlush() == false)
+ return -1;
+ return wrapped->InternalTell();
+ }
+ virtual unsigned long long InternalSize()
+ {
+ if (InternalFlush() == false)
+ return -1;
+ return wrapped->InternalSize();
+ }
+ virtual bool InternalClose(std::string const &FileName)
+ {
+ return wrapped->InternalClose(FileName);
+ }
+ virtual bool InternalAlwaysAutoClose() const
+ {
+ return wrapped->InternalAlwaysAutoClose();
+ }
+ virtual ~BufferedWriteFileFdPrivate()
+ {
+ delete wrapped;
+ }
+};
+ /*}}}*/
+class APT_HIDDEN GzipFileFdPrivate: public FileFdPrivate { /*{{{*/
+#ifdef HAVE_ZLIB
+public:
+ gzFile gz;
+ virtual bool InternalOpen(int const iFd, unsigned int const Mode) override
+ {
+ if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite)
+ gz = gzdopen(iFd, "r+");
+ else if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly)
+ gz = gzdopen(iFd, "w");
+ else
+ gz = gzdopen(iFd, "r");
+ filefd->Flags |= FileFd::Compressed;
+ return gz != nullptr;
+ }
+ virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) override
+ {
+ return gzread(gz, To, Size);
+ }
+ virtual bool InternalReadError() override
+ {
+ int err;
+ char const * const errmsg = gzerror(gz, &err);
+ if (err != Z_ERRNO)
+ return filefd->FileFdError("gzread: %s (%d: %s)", _("Read error"), err, errmsg);
+ return FileFdPrivate::InternalReadError();
+ }
+ virtual char * InternalReadLine(char * To, unsigned long long Size) override
+ {
+ return gzgets(gz, To, Size);
+ }
+ virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override
+ {
+ return gzwrite(gz,From,Size);
+ }
+ virtual bool InternalWriteError() override
+ {
+ int err;
+ char const * const errmsg = gzerror(gz, &err);
+ if (err != Z_ERRNO)
+ return filefd->FileFdError("gzwrite: %s (%d: %s)", _("Write error"), err, errmsg);
+ return FileFdPrivate::InternalWriteError();
+ }
+ virtual bool InternalSeek(unsigned long long const To) override
+ {
+ off_t const res = gzseek(gz, To, SEEK_SET);
+ if (res != (off_t)To)
+ return filefd->FileFdError("Unable to seek to %llu", To);
+ seekpos = To;
+ buffer.reset();
+ return true;
+ }
+ virtual bool InternalSkip(unsigned long long Over) override
+ {
+ if (Over >= buffer.size())
+ {
+ Over -= buffer.size();
+ buffer.reset();
+ }
+ else
+ {
+ buffer.bufferstart += Over;
+ return true;
+ }
+ if (Over == 0)
+ return true;
+ off_t const res = gzseek(gz, Over, SEEK_CUR);
+ if (res < 0)
+ return filefd->FileFdError("Unable to seek ahead %llu",Over);
+ seekpos = res;
+ return true;
+ }
+ virtual unsigned long long InternalTell() override
+ {
+ return gztell(gz) - buffer.size();
+ }
+ virtual unsigned long long InternalSize() override
+ {
+ unsigned long long filesize = FileFdPrivate::InternalSize();
+ // only check gzsize if we are actually a gzip file, just checking for
+ // "gz" is not sufficient as uncompressed files could be opened with
+ // gzopen in "direct" mode as well
+ if (filesize == 0 || gzdirect(gz))
+ return filesize;
+
+ off_t const oldPos = lseek(filefd->iFd, 0, SEEK_CUR);
+ /* unfortunately zlib.h doesn't provide a gzsize(), so we have to do
+ * this ourselves; the original (uncompressed) file size is the last 32
+ * bits of the file */
+ // FIXME: Size for gz-files is limited by 32bit… no largefile support
+ if (lseek(filefd->iFd, -4, SEEK_END) < 0)
+ {
+ filefd->FileFdErrno("lseek","Unable to seek to end of gzipped file");
+ return 0;
+ }
+ uint32_t size = 0;
+ if (read(filefd->iFd, &size, 4) != 4)
+ {
+ filefd->FileFdErrno("read","Unable to read original size of gzipped file");
+ return 0;
+ }
+ size = le32toh(size);
+
+ if (lseek(filefd->iFd, oldPos, SEEK_SET) < 0)
+ {
+ filefd->FileFdErrno("lseek","Unable to seek in gzipped file");
+ return 0;
+ }
+ return size;
+ }
+ virtual bool InternalClose(std::string const &FileName) override
+ {
+ if (gz == nullptr)
+ return true;
+ int const e = gzclose(gz);
+ gz = nullptr;
+ // gzdclose() on empty files always fails with "buffer error" here, ignore that
+ if (e != 0 && e != Z_BUF_ERROR)
+ return _error->Errno("close",_("Problem closing the gzip file %s"), FileName.c_str());
+ return true;
+ }
+
+ explicit GzipFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), gz(nullptr) {}
+ virtual ~GzipFileFdPrivate() { InternalClose(""); }
+#endif
+};
+ /*}}}*/
+class APT_HIDDEN Bz2FileFdPrivate: public FileFdPrivate { /*{{{*/
+#ifdef HAVE_BZ2
+ BZFILE* bz2;
+public:
+ virtual bool InternalOpen(int const iFd, unsigned int const Mode) override
+ {
+ if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite)
+ bz2 = BZ2_bzdopen(iFd, "r+");
+ else if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly)
+ bz2 = BZ2_bzdopen(iFd, "w");
+ else
+ bz2 = BZ2_bzdopen(iFd, "r");
+ filefd->Flags |= FileFd::Compressed;
+ return bz2 != nullptr;
+ }
+ virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) override
+ {
+ return BZ2_bzread(bz2, To, Size);
+ }
+ virtual bool InternalReadError() override
+ {
+ int err;
+ char const * const errmsg = BZ2_bzerror(bz2, &err);
+ if (err != BZ_IO_ERROR)
+ return filefd->FileFdError("BZ2_bzread: %s %s (%d: %s)", filefd->FileName.c_str(), _("Read error"), err, errmsg);
+ return FileFdPrivate::InternalReadError();
+ }
+ virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override
+ {
+ return BZ2_bzwrite(bz2, (void*)From, Size);
+ }
+ virtual bool InternalWriteError() override
+ {
+ int err;
+ char const * const errmsg = BZ2_bzerror(bz2, &err);
+ if (err != BZ_IO_ERROR)
+ return filefd->FileFdError("BZ2_bzwrite: %s %s (%d: %s)", filefd->FileName.c_str(), _("Write error"), err, errmsg);
+ return FileFdPrivate::InternalWriteError();
+ }
+ virtual bool InternalStream() const override { return true; }
+ virtual bool InternalClose(std::string const &) override
+ {
+ if (bz2 == nullptr)
+ return true;
+ BZ2_bzclose(bz2);
+ bz2 = nullptr;
+ return true;
+ }
+
+ explicit Bz2FileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), bz2(nullptr) {}
+ virtual ~Bz2FileFdPrivate() { InternalClose(""); }
+#endif
+};
+ /*}}}*/
+class APT_HIDDEN Lz4FileFdPrivate: public FileFdPrivate { /*{{{*/
+ static constexpr unsigned long long LZ4_HEADER_SIZE = 19;
+ static constexpr unsigned long long LZ4_FOOTER_SIZE = 4;
+#ifdef HAVE_LZ4
+ LZ4F_decompressionContext_t dctx;
+ LZ4F_compressionContext_t cctx;
+ LZ4F_errorCode_t res;
+ FileFd backend;
+ simple_buffer lz4_buffer;
+ // Count of bytes that the decompressor expects to read next, or buffer size.
+ size_t next_to_load = APT_BUFFER_SIZE;
+public:
+ virtual bool InternalOpen(int const iFd, unsigned int const Mode) override
+ {
+ if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite)
+ return _error->Error("lz4 only supports write or read mode");
+
+ if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly) {
+ res = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION);
+ lz4_buffer.reset(LZ4F_compressBound(APT_BUFFER_SIZE, nullptr)
+ + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE);
+ } else {
+ res = LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION);
+ lz4_buffer.reset(APT_BUFFER_SIZE);
+ }
+
+ filefd->Flags |= FileFd::Compressed;
+
+ if (LZ4F_isError(res))
+ return false;
+
+ unsigned int flags = (Mode & (FileFd::WriteOnly|FileFd::ReadOnly));
+ if (backend.OpenDescriptor(iFd, flags) == false)
+ return false;
+
+ // Write the file header
+ if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly)
+ {
+ res = LZ4F_compressBegin(cctx, lz4_buffer.buffer, lz4_buffer.buffersize_max, nullptr);
+ if (LZ4F_isError(res) || backend.Write(lz4_buffer.buffer, res) == false)
+ return false;
+ }
+
+ return true;
+ }
+ virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) override
+ {
+ /* Keep reading as long as the compressor still wants to read */
+ while (next_to_load) {
+ // Fill compressed buffer;
+ if (lz4_buffer.empty()) {
+ unsigned long long read;
+ /* Reset - if LZ4 decompressor wants to read more, allocate more */
+ lz4_buffer.reset(next_to_load);
+ if (backend.Read(lz4_buffer.getend(), lz4_buffer.free(), &read) == false)
+ return -1;
+ lz4_buffer.bufferend += read;
+
+ /* Expected EOF */
+ if (read == 0) {
+ res = -1;
+ return filefd->FileFdError("LZ4F: %s %s",
+ filefd->FileName.c_str(),
+ _("Unexpected end of file")), -1;
+ }
+ }
+ // Drain compressed buffer as far as possible.
+ size_t in = lz4_buffer.size();
+ size_t out = Size;
+
+ res = LZ4F_decompress(dctx, To, &out, lz4_buffer.get(), &in, nullptr);
+ if (LZ4F_isError(res))
+ return -1;
+
+ next_to_load = res;
+ lz4_buffer.bufferstart += in;
+
+ if (out != 0)
+ return out;
+ }
+
+ return 0;
+ }
+ virtual bool InternalReadError() override
+ {
+ char const * const errmsg = LZ4F_getErrorName(res);
+
+ return filefd->FileFdError("LZ4F: %s %s (%zu: %s)", filefd->FileName.c_str(), _("Read error"), res, errmsg);
+ }
+ virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override
+ {
+ unsigned long long const towrite = std::min(APT_BUFFER_SIZE, Size);
+
+ res = LZ4F_compressUpdate(cctx,
+ lz4_buffer.buffer, lz4_buffer.buffersize_max,
+ From, towrite, nullptr);
+
+ if (LZ4F_isError(res) || backend.Write(lz4_buffer.buffer, res) == false)
+ return -1;
+
+ return towrite;
+ }
+ virtual bool InternalWriteError() override
+ {
+ char const * const errmsg = LZ4F_getErrorName(res);
+
+ return filefd->FileFdError("LZ4F: %s %s (%zu: %s)", filefd->FileName.c_str(), _("Write error"), res, errmsg);
+ }
+ virtual bool InternalStream() const override { return true; }
+
+ virtual bool InternalFlush() override
+ {
+ return backend.Flush();
+ }
+
+ virtual bool InternalClose(std::string const &) override
+ {
+ /* Reset variables */
+ res = 0;
+ next_to_load = APT_BUFFER_SIZE;
+
+ if (cctx != nullptr)
+ {
+ res = LZ4F_compressEnd(cctx, lz4_buffer.buffer, lz4_buffer.buffersize_max, nullptr);
+ if (LZ4F_isError(res) || backend.Write(lz4_buffer.buffer, res) == false)
+ return false;
+ if (!backend.Flush())
+ return false;
+ if (!backend.Close())
+ return false;
+
+ res = LZ4F_freeCompressionContext(cctx);
+ cctx = nullptr;
+ }
+
+ if (dctx != nullptr)
+ {
+ res = LZ4F_freeDecompressionContext(dctx);
+ dctx = nullptr;
+ }
+
+ return LZ4F_isError(res) == false;
+ }
+
+ explicit Lz4FileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), dctx(nullptr), cctx(nullptr) {}
+ virtual ~Lz4FileFdPrivate() {
+ InternalClose("");
+ }
+#endif
+};
+ /*}}}*/
+class APT_HIDDEN LzmaFileFdPrivate: public FileFdPrivate { /*{{{*/
+#ifdef HAVE_LZMA
+ struct LZMAFILE {
+ FILE* file;
+ uint8_t buffer[4096];
+ lzma_stream stream;
+ lzma_ret err;
+ bool eof;
+ bool compressing;
+
+ LZMAFILE() : file(nullptr), eof(false), compressing(false) { buffer[0] = '\0'; }
+ ~LZMAFILE()
+ {
+ if (compressing == true)
+ {
+ size_t constexpr buffersize = sizeof(buffer)/sizeof(buffer[0]);
+ while(true)
+ {
+ stream.avail_out = buffersize;
+ stream.next_out = buffer;
+ err = lzma_code(&stream, LZMA_FINISH);
+ if (err != LZMA_OK && err != LZMA_STREAM_END)
+ {
+ _error->Error("~LZMAFILE: Compress finalisation failed");
+ break;
+ }
+ size_t const n = buffersize - stream.avail_out;
+ if (n && fwrite(buffer, 1, n, file) != n)
+ {
+ _error->Errno("~LZMAFILE",_("Write error"));
+ break;
+ }
+ if (err == LZMA_STREAM_END)
+ break;
+ }
+ }
+ lzma_end(&stream);
+ fclose(file);
+ }
+ };
+ LZMAFILE* lzma;
+ static uint32_t findXZlevel(std::vector<std::string> const &Args)
+ {
+ for (auto a = Args.rbegin(); a != Args.rend(); ++a)
+ if (a->empty() == false && (*a)[0] == '-' && (*a)[1] != '-')
+ {
+ auto const number = a->find_last_of("0123456789");
+ if (number == std::string::npos)
+ continue;
+ auto const extreme = a->find("e", number);
+ uint32_t level = (extreme != std::string::npos) ? LZMA_PRESET_EXTREME : 0;
+ switch ((*a)[number])
+ {
+ case '0': return level | 0;
+ case '1': return level | 1;
+ case '2': return level | 2;
+ case '3': return level | 3;
+ case '4': return level | 4;
+ case '5': return level | 5;
+ case '6': return level | 6;
+ case '7': return level | 7;
+ case '8': return level | 8;
+ case '9': return level | 9;
+ }
+ }
+ return 6;
+ }
+public:
+ virtual bool InternalOpen(int const iFd, unsigned int const Mode) override
+ {
+ if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite)
+ return filefd->FileFdError("ReadWrite mode is not supported for lzma/xz files %s", filefd->FileName.c_str());
+
+ if (lzma == nullptr)
+ lzma = new LzmaFileFdPrivate::LZMAFILE;
+ if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly)
+ lzma->file = fdopen(iFd, "w");
+ else
+ lzma->file = fdopen(iFd, "r");
+ filefd->Flags |= FileFd::Compressed;
+ if (lzma->file == nullptr)
+ return false;
+
+ lzma_stream tmp_stream = LZMA_STREAM_INIT;
+ lzma->stream = tmp_stream;
+
+ if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly)
+ {
+ uint32_t const xzlevel = findXZlevel(compressor.CompressArgs);
+ if (compressor.Name == "xz")
+ {
+ if (lzma_easy_encoder(&lzma->stream, xzlevel, LZMA_CHECK_CRC64) != LZMA_OK)
+ return false;
+ }
+ else
+ {
+ lzma_options_lzma options;
+ lzma_lzma_preset(&options, xzlevel);
+ if (lzma_alone_encoder(&lzma->stream, &options) != LZMA_OK)
+ return false;
+ }
+ lzma->compressing = true;
+ }
+ else
+ {
+ uint64_t const memlimit = UINT64_MAX;
+ if (compressor.Name == "xz")
+ {
+ if (lzma_auto_decoder(&lzma->stream, memlimit, 0) != LZMA_OK)
+ return false;
+ }
+ else
+ {
+ if (lzma_alone_decoder(&lzma->stream, memlimit) != LZMA_OK)
+ return false;
+ }
+ lzma->compressing = false;
+ }
+ return true;
+ }
+ virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) override
+ {
+ ssize_t Res;
+ if (lzma->eof == true)
+ return 0;
+
+ lzma->stream.next_out = (uint8_t *) To;
+ lzma->stream.avail_out = Size;
+ if (lzma->stream.avail_in == 0)
+ {
+ lzma->stream.next_in = lzma->buffer;
+ lzma->stream.avail_in = fread(lzma->buffer, 1, sizeof(lzma->buffer)/sizeof(lzma->buffer[0]), lzma->file);
+ }
+ lzma->err = lzma_code(&lzma->stream, LZMA_RUN);
+ if (lzma->err == LZMA_STREAM_END)
+ {
+ lzma->eof = true;
+ Res = Size - lzma->stream.avail_out;
+ }
+ else if (lzma->err != LZMA_OK)
+ {
+ Res = -1;
+ errno = 0;
+ }
+ else
+ {
+ Res = Size - lzma->stream.avail_out;
+ if (Res == 0)
+ {
+ // lzma run was okay, but produced no output…
+ Res = -1;
+ errno = EINTR;
+ }
+ }
+ return Res;
+ }
+ virtual bool InternalReadError() override
+ {
+ return filefd->FileFdError("lzma_read: %s (%d)", _("Read error"), lzma->err);
+ }
+ virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override
+ {
+ ssize_t Res;
+ lzma->stream.next_in = (uint8_t *)From;
+ lzma->stream.avail_in = Size;
+ lzma->stream.next_out = lzma->buffer;
+ lzma->stream.avail_out = sizeof(lzma->buffer)/sizeof(lzma->buffer[0]);
+ lzma->err = lzma_code(&lzma->stream, LZMA_RUN);
+ if (lzma->err != LZMA_OK)
+ return -1;
+ size_t const n = sizeof(lzma->buffer)/sizeof(lzma->buffer[0]) - lzma->stream.avail_out;
+ size_t const m = (n == 0) ? 0 : fwrite(lzma->buffer, 1, n, lzma->file);
+ if (m != n)
+ {
+ Res = -1;
+ errno = 0;
+ }
+ else
+ {
+ Res = Size - lzma->stream.avail_in;
+ if (Res == 0)
+ {
+ // lzma run was okay, but produced no output…
+ Res = -1;
+ errno = EINTR;
+ }
+ }
+ return Res;
+ }
+ virtual bool InternalWriteError() override
+ {
+ return filefd->FileFdError("lzma_write: %s (%d)", _("Write error"), lzma->err);
+ }
+ virtual bool InternalStream() const override { return true; }
+ virtual bool InternalClose(std::string const &) override
+ {
+ delete lzma;
+ lzma = nullptr;
+ return true;
+ }
+
+ explicit LzmaFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), lzma(nullptr) {}
+ virtual ~LzmaFileFdPrivate() { InternalClose(""); }
+#endif
+};
+ /*}}}*/
+class APT_HIDDEN PipedFileFdPrivate: public FileFdPrivate /*{{{*/
+/* if we don't have a specific class dealing with library calls, we (un)compress
+ by executing a specified binary and pipe in/out what we need */
+{
+public:
+ virtual bool InternalOpen(int const, unsigned int const Mode) override
+ {
+ // collect zombies here in case we reopen
+ if (compressor_pid > 0)
+ ExecWait(compressor_pid, "FileFdCompressor", true);
+
+ if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite)
+ return filefd->FileFdError("ReadWrite mode is not supported for file %s", filefd->FileName.c_str());
+
+ bool const Comp = (Mode & FileFd::WriteOnly) == FileFd::WriteOnly;
+ if (Comp == false)
+ {
+ // Handle 'decompression' of empty files
+ struct stat Buf;
+ fstat(filefd->iFd, &Buf);
+ if (Buf.st_size == 0 && S_ISFIFO(Buf.st_mode) == false)
+ return true;
+
+ // We don't need the file open - instead let the compressor open it
+ // as he properly knows better how to efficiently read from 'his' file
+ if (filefd->FileName.empty() == false)
+ {
+ close(filefd->iFd);
+ filefd->iFd = -1;
+ }
+ }
+
+ // Create a data pipe
+ int Pipe[2] = {-1,-1};
+ if (pipe(Pipe) != 0)
+ return filefd->FileFdErrno("pipe",_("Failed to create subprocess IPC"));
+ for (int J = 0; J != 2; J++)
+ SetCloseExec(Pipe[J],true);
+
+ compressed_fd = filefd->iFd;
+ set_is_pipe(true);
+
+ if (Comp == true)
+ filefd->iFd = Pipe[1];
+ else
+ filefd->iFd = Pipe[0];
+
+ // The child..
+ compressor_pid = ExecFork();
+ if (compressor_pid == 0)
+ {
+ if (Comp == true)
+ {
+ dup2(compressed_fd,STDOUT_FILENO);
+ dup2(Pipe[0],STDIN_FILENO);
+ }
+ else
+ {
+ if (compressed_fd != -1)
+ dup2(compressed_fd,STDIN_FILENO);
+ dup2(Pipe[1],STDOUT_FILENO);
+ }
+ int const nullfd = open("/dev/null", O_WRONLY);
+ if (nullfd != -1)
+ {
+ dup2(nullfd,STDERR_FILENO);
+ close(nullfd);
+ }
+
+ SetCloseExec(STDOUT_FILENO,false);
+ SetCloseExec(STDIN_FILENO,false);
+
+ std::vector<char const*> Args;
+ Args.push_back(compressor.Binary.c_str());
+ std::vector<std::string> const * const addArgs =
+ (Comp == true) ? &(compressor.CompressArgs) : &(compressor.UncompressArgs);
+ for (std::vector<std::string>::const_iterator a = addArgs->begin();
+ a != addArgs->end(); ++a)
+ Args.push_back(a->c_str());
+ if (Comp == false && filefd->FileName.empty() == false)
+ {
+ // commands not needing arguments, do not need to be told about using standard output
+ // in reality, only testcases with tools like cat, rev, rot13, … are able to trigger this
+ if (compressor.CompressArgs.empty() == false && compressor.UncompressArgs.empty() == false)
+ Args.push_back("--stdout");
+ if (filefd->TemporaryFileName.empty() == false)
+ Args.push_back(filefd->TemporaryFileName.c_str());
+ else
+ Args.push_back(filefd->FileName.c_str());
+ }
+ Args.push_back(NULL);
+
+ execvp(Args[0],(char **)&Args[0]);
+ cerr << _("Failed to exec compressor ") << Args[0] << endl;
+ _exit(100);
+ }
+ if (Comp == true)
+ close(Pipe[0]);
+ else
+ close(Pipe[1]);
+
+ return true;
+ }
+ virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) override
+ {
+ return read(filefd->iFd, To, Size);
+ }
+ virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override
+ {
+ return write(filefd->iFd, From, Size);
+ }
+ virtual bool InternalClose(std::string const &) override
+ {
+ bool Ret = true;
+ if (compressor_pid > 0)
+ Ret &= ExecWait(compressor_pid, "FileFdCompressor", true);
+ compressor_pid = -1;
+ return Ret;
+ }
+ explicit PipedFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd) {}
+ virtual ~PipedFileFdPrivate() { InternalClose(""); }
+};
+ /*}}}*/
+class APT_HIDDEN DirectFileFdPrivate: public FileFdPrivate /*{{{*/
+{
+public:
+ virtual bool InternalOpen(int const, unsigned int const) override { return true; }
+ virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) override
+ {
+ return read(filefd->iFd, To, Size);
+ }
+ virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override
+ {
+ // files opened read+write are strange and only really "supported" for direct files
+ if (buffer.size() != 0)
+ {
+ lseek(filefd->iFd, -buffer.size(), SEEK_CUR);
+ buffer.reset();
+ }
+ return write(filefd->iFd, From, Size);
+ }
+ virtual bool InternalSeek(unsigned long long const To) override
+ {
+ off_t const res = lseek(filefd->iFd, To, SEEK_SET);
+ if (res != (off_t)To)
+ return filefd->FileFdError("Unable to seek to %llu", To);
+ seekpos = To;
+ buffer.reset();
+ return true;
+ }
+ virtual bool InternalSkip(unsigned long long Over) override
+ {
+ if (Over >= buffer.size())
+ {
+ Over -= buffer.size();
+ buffer.reset();
+ }
+ else
+ {
+ buffer.bufferstart += Over;
+ return true;
+ }
+ if (Over == 0)
+ return true;
+ off_t const res = lseek(filefd->iFd, Over, SEEK_CUR);
+ if (res < 0)
+ return filefd->FileFdError("Unable to seek ahead %llu",Over);
+ seekpos = res;
+ return true;
+ }
+ virtual bool InternalTruncate(unsigned long long const To) override
+ {
+ if (buffer.size() != 0)
+ {
+ unsigned long long const seekpos = lseek(filefd->iFd, 0, SEEK_CUR);
+ if ((seekpos - buffer.size()) >= To)
+ buffer.reset();
+ else if (seekpos >= To)
+ buffer.bufferend = (To - seekpos) + buffer.bufferstart;
+ else
+ buffer.reset();
+ }
+ if (ftruncate(filefd->iFd, To) != 0)
+ return filefd->FileFdError("Unable to truncate to %llu",To);
+ return true;
+ }
+ virtual unsigned long long InternalTell() override
+ {
+ return lseek(filefd->iFd,0,SEEK_CUR) - buffer.size();
+ }
+ virtual unsigned long long InternalSize() override
+ {
+ return filefd->FileSize();
+ }
+ virtual bool InternalClose(std::string const &) override { return true; }
+ virtual bool InternalAlwaysAutoClose() const override { return false; }
+
+ explicit DirectFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd) {}
+ virtual ~DirectFileFdPrivate() { InternalClose(""); }
+};
+ /*}}}*/
+// FileFd Constructors /*{{{*/
+FileFd::FileFd(std::string FileName,unsigned int const Mode,unsigned long AccessMode) : iFd(-1), Flags(0), d(NULL)
+{
+ Open(FileName,Mode, None, AccessMode);
+}
+FileFd::FileFd(std::string FileName,unsigned int const Mode, CompressMode Compress, unsigned long AccessMode) : iFd(-1), Flags(0), d(NULL)
+{
+ Open(FileName,Mode, Compress, AccessMode);
+}
+FileFd::FileFd() : iFd(-1), Flags(AutoClose), d(NULL) {}
+FileFd::FileFd(int const Fd, unsigned int const Mode, CompressMode Compress) : iFd(-1), Flags(0), d(NULL)
+{
+ OpenDescriptor(Fd, Mode, Compress);
+}
+FileFd::FileFd(int const Fd, bool const AutoClose) : iFd(-1), Flags(0), d(NULL)
+{
+ OpenDescriptor(Fd, ReadWrite, None, AutoClose);
+}
+ /*}}}*/
+// FileFd::Open - Open a file /*{{{*/
+// ---------------------------------------------------------------------
+/* The most commonly used open mode combinations are given with Mode */
+bool FileFd::Open(string FileName,unsigned int const Mode,CompressMode Compress, unsigned long const AccessMode)
+{
+ if (Mode == ReadOnlyGzip)
+ return Open(FileName, ReadOnly, Gzip, AccessMode);
+
+ if (Compress == Auto && (Mode & WriteOnly) == WriteOnly)
+ return FileFdError("Autodetection on %s only works in ReadOnly openmode!", FileName.c_str());
+
+ std::vector<APT::Configuration::Compressor> const compressors = APT::Configuration::getCompressors();
+ std::vector<APT::Configuration::Compressor>::const_iterator compressor = compressors.begin();
+ if (Compress == Auto)
+ {
+ for (; compressor != compressors.end(); ++compressor)
+ {
+ std::string file = FileName + compressor->Extension;
+ if (FileExists(file) == false)
+ continue;
+ FileName = file;
+ break;
+ }
+ }
+ else if (Compress == Extension)
+ {
+ std::string::size_type const found = FileName.find_last_of('.');
+ std::string ext;
+ if (found != std::string::npos)
+ {
+ ext = FileName.substr(found);
+ if (ext == ".new" || ext == ".bak")
+ {
+ std::string::size_type const found2 = FileName.find_last_of('.', found - 1);
+ if (found2 != std::string::npos)
+ ext = FileName.substr(found2, found - found2);
+ else
+ ext.clear();
+ }
+ }
+ for (; compressor != compressors.end(); ++compressor)
+ if (ext == compressor->Extension)
+ break;
+ // no matching extension - assume uncompressed (imagine files like 'example.org_Packages')
+ if (compressor == compressors.end())
+ for (compressor = compressors.begin(); compressor != compressors.end(); ++compressor)
+ if (compressor->Name == ".")
+ break;
+ }
+ else
+ {
+ std::string name;
+ switch (Compress)
+ {
+ case None: name = "."; break;
+ case Gzip: name = "gzip"; break;
+ case Bzip2: name = "bzip2"; break;
+ case Lzma: name = "lzma"; break;
+ case Xz: name = "xz"; break;
+ case Lz4: name = "lz4"; break;
+ case Auto:
+ case Extension:
+ // Unreachable
+ return FileFdError("Opening File %s in None, Auto or Extension should be already handled?!?", FileName.c_str());
+ }
+ for (; compressor != compressors.end(); ++compressor)
+ if (compressor->Name == name)
+ break;
+ if (compressor == compressors.end())
+ return FileFdError("Can't find a configured compressor %s for file %s", name.c_str(), FileName.c_str());
+ }
+
+ if (compressor == compressors.end())
+ return FileFdError("Can't find a match for specified compressor mode for file %s", FileName.c_str());
+ return Open(FileName, Mode, *compressor, AccessMode);
+}
+bool FileFd::Open(string FileName,unsigned int const Mode,APT::Configuration::Compressor const &compressor, unsigned long const AccessMode)
+{
+ Close();
+ Flags = AutoClose;
+
+ if ((Mode & WriteOnly) != WriteOnly && (Mode & (Atomic | Create | Empty | Exclusive)) != 0)
+ return FileFdError("ReadOnly mode for %s doesn't accept additional flags!", FileName.c_str());
+ if ((Mode & ReadWrite) == 0)
+ return FileFdError("No openmode provided in FileFd::Open for %s", FileName.c_str());
+
+ unsigned int OpenMode = Mode;
+ if (FileName == "/dev/null")
+ OpenMode = OpenMode & ~(Atomic | Exclusive | Create | Empty);
+
+ if ((OpenMode & Atomic) == Atomic)
+ {
+ Flags |= Replace;
+ }
+ else if ((OpenMode & (Exclusive | Create)) == (Exclusive | Create))
+ {
+ // for atomic, this will be done by rename in Close()
+ RemoveFile("FileFd::Open", FileName);
+ }
+ if ((OpenMode & Empty) == Empty)
+ {
+ struct stat Buf;
+ if (lstat(FileName.c_str(),&Buf) == 0 && S_ISLNK(Buf.st_mode))
+ RemoveFile("FileFd::Open", FileName);
+ }
+
+ int fileflags = 0;
+ #define if_FLAGGED_SET(FLAG, MODE) if ((OpenMode & FLAG) == FLAG) fileflags |= MODE
+ if_FLAGGED_SET(ReadWrite, O_RDWR);
+ else if_FLAGGED_SET(ReadOnly, O_RDONLY);
+ else if_FLAGGED_SET(WriteOnly, O_WRONLY);
+
+ if_FLAGGED_SET(Create, O_CREAT);
+ if_FLAGGED_SET(Empty, O_TRUNC);
+ if_FLAGGED_SET(Exclusive, O_EXCL);
+ #undef if_FLAGGED_SET
+
+ if ((OpenMode & Atomic) == Atomic)
+ {
+ char *name = strdup((FileName + ".XXXXXX").c_str());
+
+ if((iFd = mkstemp(name)) == -1)
+ {
+ free(name);
+ return FileFdErrno("mkstemp", "Could not create temporary file for %s", FileName.c_str());
+ }
+
+ TemporaryFileName = string(name);
+ free(name);
+
+ // umask() will always set the umask and return the previous value, so
+ // we first set the umask and then reset it to the old value
+ mode_t const CurrentUmask = umask(0);
+ umask(CurrentUmask);
+ // calculate the actual file permissions (just like open/creat)
+ mode_t const FilePermissions = (AccessMode & ~CurrentUmask);
+
+ if(fchmod(iFd, FilePermissions) == -1)
+ return FileFdErrno("fchmod", "Could not change permissions for temporary file %s", TemporaryFileName.c_str());
+ }
+ else
+ iFd = open(FileName.c_str(), fileflags, AccessMode);
+
+ this->FileName = FileName;
+ if (iFd == -1 || OpenInternDescriptor(OpenMode, compressor) == false)
+ {
+ if (iFd != -1)
+ {
+ close (iFd);
+ iFd = -1;
+ }
+ return FileFdErrno("open",_("Could not open file %s"), FileName.c_str());
+ }
+
+ SetCloseExec(iFd,true);
+ return true;
+}
+ /*}}}*/
+// FileFd::OpenDescriptor - Open a filedescriptor /*{{{*/
+bool FileFd::OpenDescriptor(int Fd, unsigned int const Mode, CompressMode Compress, bool AutoClose)
+{
+ std::vector<APT::Configuration::Compressor> const compressors = APT::Configuration::getCompressors();
+ std::vector<APT::Configuration::Compressor>::const_iterator compressor = compressors.begin();
+ std::string name;
+
+ // compat with the old API
+ if (Mode == ReadOnlyGzip && Compress == None)
+ Compress = Gzip;
+
+ switch (Compress)
+ {
+ case None: name = "."; break;
+ case Gzip: name = "gzip"; break;
+ case Bzip2: name = "bzip2"; break;
+ case Lzma: name = "lzma"; break;
+ case Xz: name = "xz"; break;
+ case Lz4: name = "lz4"; break;
+ case Auto:
+ case Extension:
+ if (AutoClose == true && Fd != -1)
+ close(Fd);
+ return FileFdError("Opening Fd %d in Auto or Extension compression mode is not supported", Fd);
+ }
+ for (; compressor != compressors.end(); ++compressor)
+ if (compressor->Name == name)
+ break;
+ if (compressor == compressors.end())
+ {
+ if (AutoClose == true && Fd != -1)
+ close(Fd);
+ return FileFdError("Can't find a configured compressor %s for file %s", name.c_str(), FileName.c_str());
}
return OpenDescriptor(Fd, Mode, *compressor, AutoClose);
}
bool FileFd::OpenInternDescriptor(unsigned int const Mode, APT::Configuration::Compressor const &compressor)
{
if (iFd == -1)
- return false;
- if (compressor.Name == "." || compressor.Binary.empty() == true)
- return true;
-
-#if defined HAVE_ZLIB || defined HAVE_BZ2 || defined HAVE_LZMA
- // the API to open files is similar, so setup to avoid code duplicates later
- // and while at it ensure that we close before opening (if its a reopen)
- void* (*compress_open)(int, const char *) = NULL;
- if (false)
- /* dummy so that the rest can be 'else if's */;
-#define APT_COMPRESS_INIT(NAME,OPEN) \
- else if (compressor.Name == NAME) \
- { \
- compress_open = (void*(*)(int, const char *)) OPEN; \
- if (d != NULL) d->InternalClose(FileName); \
- }
-#ifdef HAVE_ZLIB
- APT_COMPRESS_INIT("gzip", gzdopen)
-#endif
-#ifdef HAVE_BZ2
- APT_COMPRESS_INIT("bzip2", BZ2_bzdopen)
-#endif
-#ifdef HAVE_LZMA
- APT_COMPRESS_INIT("xz", fdopen)
- APT_COMPRESS_INIT("lzma", fdopen)
-#endif
-#undef APT_COMPRESS_INIT
-#endif
-
- if (d == NULL)
- {
- d = new FileFdPrivate();
- d->openmode = Mode;
- d->compressor = compressor;
-#if defined HAVE_ZLIB || defined HAVE_BZ2 || defined HAVE_LZMA
- if ((Flags & AutoClose) != AutoClose && compress_open != NULL)
- {
- // Need to duplicate fd here or gz/bz2 close for cleanup will close the fd as well
- int const internFd = dup(iFd);
- if (internFd == -1)
- return FileFdErrno("OpenInternDescriptor", _("Could not open file descriptor %d"), iFd);
- iFd = internFd;
- }
-#endif
- }
+ return false;
-#if defined HAVE_ZLIB || defined HAVE_BZ2 || defined HAVE_LZMA
- if (compress_open != NULL)
- {
- void* compress_struct = NULL;
- if ((Mode & ReadWrite) == ReadWrite)
- compress_struct = compress_open(iFd, "r+");
- else if ((Mode & WriteOnly) == WriteOnly)
- compress_struct = compress_open(iFd, "w");
- else
- compress_struct = compress_open(iFd, "r");
- if (compress_struct == NULL)
- return false;
+ if (d != nullptr)
+ d->InternalClose(FileName);
+ if (d == nullptr)
+ {
if (false)
/* dummy so that the rest can be 'else if's */;
+#define APT_COMPRESS_INIT(NAME, CONSTRUCTOR) \
+ else if (compressor.Name == NAME) \
+ d = new CONSTRUCTOR(this)
#ifdef HAVE_ZLIB
- else if (compressor.Name == "gzip")
- d->gz = (gzFile) compress_struct;
+ APT_COMPRESS_INIT("gzip", GzipFileFdPrivate);
#endif
#ifdef HAVE_BZ2
- else if (compressor.Name == "bzip2")
- d->bz2 = (BZFILE*) compress_struct;
+ APT_COMPRESS_INIT("bzip2", Bz2FileFdPrivate);
#endif
#ifdef HAVE_LZMA
- else if (compressor.Name == "xz" || compressor.Name == "lzma")
- {
- uint32_t const xzlevel = 6;
- uint64_t const memlimit = UINT64_MAX;
- if (d->lzma == NULL)
- d->lzma = new FileFdPrivate::LZMAFILE;
- d->lzma->file = (FILE*) compress_struct;
- lzma_stream tmp_stream = LZMA_STREAM_INIT;
- d->lzma->stream = tmp_stream;
-
- if ((Mode & ReadWrite) == ReadWrite)
- return FileFdError("ReadWrite mode is not supported for file %s", FileName.c_str());
-
- if ((Mode & WriteOnly) == WriteOnly)
- {
- if (compressor.Name == "xz")
- {
- if (lzma_easy_encoder(&d->lzma->stream, xzlevel, LZMA_CHECK_CRC32) != LZMA_OK)
- return false;
- }
- else
- {
- lzma_options_lzma options;
- lzma_lzma_preset(&options, xzlevel);
- if (lzma_alone_encoder(&d->lzma->stream, &options) != LZMA_OK)
- return false;
- }
- d->lzma->compressing = true;
- }
- else
- {
- if (compressor.Name == "xz")
- {
- if (lzma_auto_decoder(&d->lzma->stream, memlimit, 0) != LZMA_OK)
- return false;
- }
- else
- {
- if (lzma_alone_decoder(&d->lzma->stream, memlimit) != LZMA_OK)
- return false;
- }
- d->lzma->compressing = false;
- }
- }
+ APT_COMPRESS_INIT("xz", LzmaFileFdPrivate);
+ APT_COMPRESS_INIT("lzma", LzmaFileFdPrivate);
#endif
- Flags |= Compressed;
- return true;
- }
+#ifdef HAVE_LZ4
+ APT_COMPRESS_INIT("lz4", Lz4FileFdPrivate);
#endif
-
- // collect zombies here in case we reopen
- if (d->compressor_pid > 0)
- ExecWait(d->compressor_pid, "FileFdCompressor", true);
-
- if ((Mode & ReadWrite) == ReadWrite)
- return FileFdError("ReadWrite mode is not supported for file %s", FileName.c_str());
-
- bool const Comp = (Mode & WriteOnly) == WriteOnly;
- if (Comp == false)
- {
- // Handle 'decompression' of empty files
- struct stat Buf;
- fstat(iFd, &Buf);
- if (Buf.st_size == 0 && S_ISFIFO(Buf.st_mode) == false)
- return true;
-
- // We don't need the file open - instead let the compressor open it
- // as he properly knows better how to efficiently read from 'his' file
- if (FileName.empty() == false)
- {
- close(iFd);
- iFd = -1;
- }
- }
-
- // Create a data pipe
- int Pipe[2] = {-1,-1};
- if (pipe(Pipe) != 0)
- return FileFdErrno("pipe",_("Failed to create subprocess IPC"));
- for (int J = 0; J != 2; J++)
- SetCloseExec(Pipe[J],true);
-
- d->compressed_fd = iFd;
- d->pipe = true;
-
- if (Comp == true)
- iFd = Pipe[1];
- else
- iFd = Pipe[0];
-
- // The child..
- d->compressor_pid = ExecFork();
- if (d->compressor_pid == 0)
- {
- if (Comp == true)
- {
- dup2(d->compressed_fd,STDOUT_FILENO);
- dup2(Pipe[0],STDIN_FILENO);
- }
+#undef APT_COMPRESS_INIT
+ else if (compressor.Name == "." || compressor.Binary.empty() == true)
+ d = new DirectFileFdPrivate(this);
else
- {
- if (d->compressed_fd != -1)
- dup2(d->compressed_fd,STDIN_FILENO);
- dup2(Pipe[1],STDOUT_FILENO);
- }
- int const nullfd = open("/dev/null", O_WRONLY);
- if (nullfd != -1)
- {
- dup2(nullfd,STDERR_FILENO);
- close(nullfd);
- }
+ d = new PipedFileFdPrivate(this);
- SetCloseExec(STDOUT_FILENO,false);
- SetCloseExec(STDIN_FILENO,false);
-
- std::vector<char const*> Args;
- Args.push_back(compressor.Binary.c_str());
- std::vector<std::string> const * const addArgs =
- (Comp == true) ? &(compressor.CompressArgs) : &(compressor.UncompressArgs);
- for (std::vector<std::string>::const_iterator a = addArgs->begin();
- a != addArgs->end(); ++a)
- Args.push_back(a->c_str());
- if (Comp == false && FileName.empty() == false)
+ if (Mode & BufferedWrite)
+ d = new BufferedWriteFileFdPrivate(d);
+
+ d->set_openmode(Mode);
+ d->set_compressor(compressor);
+ if ((Flags & AutoClose) != AutoClose && d->InternalAlwaysAutoClose())
{
- // commands not needing arguments, do not need to be told about using standard output
- // in reality, only testcases with tools like cat, rev, rot13, … are able to trigger this
- if (compressor.CompressArgs.empty() == false && compressor.UncompressArgs.empty() == false)
- Args.push_back("--stdout");
- if (TemporaryFileName.empty() == false)
- Args.push_back(TemporaryFileName.c_str());
- else
- Args.push_back(FileName.c_str());
+ // Need to duplicate fd here or gz/bz2 close for cleanup will close the fd as well
+ int const internFd = dup(iFd);
+ if (internFd == -1)
+ return FileFdErrno("OpenInternDescriptor", _("Could not open file descriptor %d"), iFd);
+ iFd = internFd;
}
- Args.push_back(NULL);
-
- execvp(Args[0],(char **)&Args[0]);
- cerr << _("Failed to exec compressor ") << Args[0] << endl;
- _exit(100);
}
- if (Comp == true)
- close(Pipe[0]);
- else
- close(Pipe[1]);
-
- return true;
+ return d->InternalOpen(iFd, Mode);
}
/*}}}*/
// FileFd::~File - Closes the file /*{{{*/
{
Close();
if (d != NULL)
- d->CloseDown(FileName);
+ d->InternalClose(FileName);
delete d;
d = NULL;
}
gracefully. */
bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual)
{
- ssize_t Res;
+ if (d == nullptr)
+ return false;
+ ssize_t Res = 1;
errno = 0;
if (Actual != 0)
*Actual = 0;
*((char *)To) = '\0';
- do
+ while (Res > 0 && Size > 0)
{
- if (false)
- /* dummy so that the rest can be 'else if's */;
-#ifdef HAVE_ZLIB
- else if (d != NULL && d->gz != NULL)
- Res = gzread(d->gz,To,Size);
-#endif
-#ifdef HAVE_BZ2
- else if (d != NULL && d->bz2 != NULL)
- Res = BZ2_bzread(d->bz2,To,Size);
-#endif
-#ifdef HAVE_LZMA
- else if (d != NULL && d->lzma != NULL)
- {
- if (d->lzma->eof == true)
- break;
-
- d->lzma->stream.next_out = (uint8_t *) To;
- d->lzma->stream.avail_out = Size;
- if (d->lzma->stream.avail_in == 0)
- {
- d->lzma->stream.next_in = d->lzma->buffer;
- d->lzma->stream.avail_in = fread(d->lzma->buffer, 1, sizeof(d->lzma->buffer)/sizeof(d->lzma->buffer[0]), d->lzma->file);
- }
- d->lzma->err = lzma_code(&d->lzma->stream, LZMA_RUN);
- if (d->lzma->err == LZMA_STREAM_END)
- {
- d->lzma->eof = true;
- Res = Size - d->lzma->stream.avail_out;
- }
- else if (d->lzma->err != LZMA_OK)
- {
- Res = -1;
- errno = 0;
- }
- else
- {
- Res = Size - d->lzma->stream.avail_out;
- if (Res == 0)
- {
- // lzma run was okay, but produced no output…
- Res = -1;
- errno = EINTR;
- }
- }
- }
-#endif
- else
- Res = read(iFd,To,Size);
+ Res = d->InternalRead(To, Size);
if (Res < 0)
{
errno = 0;
continue;
}
- if (false)
- /* dummy so that the rest can be 'else if's */;
-#ifdef HAVE_ZLIB
- else if (d != NULL && d->gz != NULL)
- {
- int err;
- char const * const errmsg = gzerror(d->gz, &err);
- if (err != Z_ERRNO)
- return FileFdError("gzread: %s (%d: %s)", _("Read error"), err, errmsg);
- }
-#endif
-#ifdef HAVE_BZ2
- else if (d != NULL && d->bz2 != NULL)
- {
- int err;
- char const * const errmsg = BZ2_bzerror(d->bz2, &err);
- if (err != BZ_IO_ERROR)
- return FileFdError("BZ2_bzread: %s %s (%d: %s)", FileName.c_str(), _("Read error"), err, errmsg);
- }
-#endif
-#ifdef HAVE_LZMA
- else if (d != NULL && d->lzma != NULL)
- return FileFdError("lzma_read: %s (%d)", _("Read error"), d->lzma->err);
-#endif
- return FileFdErrno("read",_("Read error"));
+ return d->InternalReadError();
}
To = (char *)To + Res;
Size -= Res;
if (d != NULL)
- d->seekpos += Res;
+ d->set_seekpos(d->get_seekpos() + Res);
if (Actual != 0)
*Actual += Res;
}
- while (Res > 0 && Size > 0);
if (Size == 0)
return true;
/*}}}*/
// FileFd::ReadLine - Read a complete line from the file /*{{{*/
// ---------------------------------------------------------------------
-/* Beware: This method can be quiet slow for big buffers on UNcompressed
+/* Beware: This method can be quite slow for big buffers on UNcompressed
files because of the naive implementation! */
char* FileFd::ReadLine(char *To, unsigned long long const Size)
{
*To = '\0';
-#ifdef HAVE_ZLIB
- if (d != NULL && d->gz != NULL)
- return gzgets(d->gz, To, Size);
-#endif
+ if (d == nullptr)
+ return nullptr;
+ return d->InternalReadLine(To, Size);
+}
+ /*}}}*/
+// FileFd::Flush - Flush the file /*{{{*/
+bool FileFd::Flush()
+{
+ if (d == nullptr)
+ return true;
- unsigned long long read = 0;
- while ((Size - 1) != read)
- {
- unsigned long long done = 0;
- if (Read(To + read, 1, &done) == false)
- return NULL;
- if (done == 0)
- break;
- if (To[read++] == '\n')
- break;
- }
- if (read == 0)
- return NULL;
- To[read] = '\0';
- return To;
+ return d->InternalFlush();
}
/*}}}*/
// FileFd::Write - Write to the file /*{{{*/
-// ---------------------------------------------------------------------
-/* */
bool FileFd::Write(const void *From,unsigned long long Size)
{
- ssize_t Res;
+ if (d == nullptr)
+ return false;
+ ssize_t Res = 1;
errno = 0;
- do
+ while (Res > 0 && Size > 0)
{
- if (false)
- /* dummy so that the rest can be 'else if's */;
-#ifdef HAVE_ZLIB
- else if (d != NULL && d->gz != NULL)
- Res = gzwrite(d->gz,From,Size);
-#endif
-#ifdef HAVE_BZ2
- else if (d != NULL && d->bz2 != NULL)
- Res = BZ2_bzwrite(d->bz2,(void*)From,Size);
-#endif
-#ifdef HAVE_LZMA
- else if (d != NULL && d->lzma != NULL)
- {
- d->lzma->stream.next_in = (uint8_t *)From;
- d->lzma->stream.avail_in = Size;
- d->lzma->stream.next_out = d->lzma->buffer;
- d->lzma->stream.avail_out = sizeof(d->lzma->buffer)/sizeof(d->lzma->buffer[0]);
- d->lzma->err = lzma_code(&d->lzma->stream, LZMA_RUN);
- if (d->lzma->err != LZMA_OK)
- return false;
- size_t const n = sizeof(d->lzma->buffer)/sizeof(d->lzma->buffer[0]) - d->lzma->stream.avail_out;
- size_t const m = (n == 0) ? 0 : fwrite(d->lzma->buffer, 1, n, d->lzma->file);
- if (m != n)
- Res = -1;
- else
- Res = Size - d->lzma->stream.avail_in;
- }
-#endif
- else
- Res = write(iFd,From,Size);
+ Res = d->InternalWrite(From, Size);
- if (Res < 0 && errno == EINTR)
- continue;
if (Res < 0)
{
- if (false)
- /* dummy so that the rest can be 'else if's */;
-#ifdef HAVE_ZLIB
- else if (d != NULL && d->gz != NULL)
- {
- int err;
- char const * const errmsg = gzerror(d->gz, &err);
- if (err != Z_ERRNO)
- return FileFdError("gzwrite: %s (%d: %s)", _("Write error"), err, errmsg);
- }
-#endif
-#ifdef HAVE_BZ2
- else if (d != NULL && d->bz2 != NULL)
+ if (errno == EINTR)
{
- int err;
- char const * const errmsg = BZ2_bzerror(d->bz2, &err);
- if (err != BZ_IO_ERROR)
- return FileFdError("BZ2_bzwrite: %s (%d: %s)", _("Write error"), err, errmsg);
+ // trick the while-loop into running again
+ Res = 1;
+ errno = 0;
+ continue;
}
-#endif
-#ifdef HAVE_LZMA
- else if (d != NULL && d->lzma != NULL)
- return FileFdErrno("lzma_fwrite", _("Write error"));
-#endif
- return FileFdErrno("write",_("Write error"));
+ return d->InternalWriteError();
}
-
+
From = (char const *)From + Res;
Size -= Res;
if (d != NULL)
- d->seekpos += Res;
+ d->set_seekpos(d->get_seekpos() + Res);
}
- while (Res > 0 && Size > 0);
-
+
if (Size == 0)
return true;
}
bool FileFd::Write(int Fd, const void *From, unsigned long long Size)
{
- ssize_t Res;
+ ssize_t Res = 1;
errno = 0;
- do
+ while (Res > 0 && Size > 0)
{
Res = write(Fd,From,Size);
if (Res < 0 && errno == EINTR)
From = (char const *)From + Res;
Size -= Res;
}
- while (Res > 0 && Size > 0);
if (Size == 0)
return true;
}
/*}}}*/
// FileFd::Seek - Seek in the file /*{{{*/
-// ---------------------------------------------------------------------
-/* */
bool FileFd::Seek(unsigned long long To)
{
+ if (d == nullptr)
+ return false;
Flags &= ~HitEof;
-
- if (d != NULL && (d->pipe == true || d->InternalStream() == true))
- {
- // Our poor man seeking in pipes is costly, so try to avoid it
- unsigned long long seekpos = Tell();
- if (seekpos == To)
- return true;
- else if (seekpos < To)
- return Skip(To - seekpos);
-
- if ((d->openmode & ReadOnly) != ReadOnly)
- return FileFdError("Reopen is only implemented for read-only files!");
- d->InternalClose(FileName);
- if (iFd != -1)
- close(iFd);
- iFd = -1;
- if (TemporaryFileName.empty() == false)
- iFd = open(TemporaryFileName.c_str(), O_RDONLY);
- else if (FileName.empty() == false)
- iFd = open(FileName.c_str(), O_RDONLY);
- else
- {
- if (d->compressed_fd > 0)
- if (lseek(d->compressed_fd, 0, SEEK_SET) != 0)
- iFd = d->compressed_fd;
- if (iFd < 0)
- return FileFdError("Reopen is not implemented for pipes opened with FileFd::OpenDescriptor()!");
- }
-
- if (OpenInternDescriptor(d->openmode, d->compressor) == false)
- return FileFdError("Seek on file %s because it couldn't be reopened", FileName.c_str());
-
- if (To != 0)
- return Skip(To);
-
- d->seekpos = To;
- return true;
- }
- off_t res;
-#ifdef HAVE_ZLIB
- if (d != NULL && d->gz)
- res = gzseek(d->gz,To,SEEK_SET);
- else
-#endif
- res = lseek(iFd,To,SEEK_SET);
- if (res != (off_t)To)
- return FileFdError("Unable to seek to %llu", To);
-
- if (d != NULL)
- d->seekpos = To;
- return true;
+ return d->InternalSeek(To);
}
/*}}}*/
-// FileFd::Skip - Seek in the file /*{{{*/
-// ---------------------------------------------------------------------
-/* */
+// FileFd::Skip - Skip over data in the file /*{{{*/
bool FileFd::Skip(unsigned long long Over)
{
- if (d != NULL && (d->pipe == true || d->InternalStream() == true))
- {
- char buffer[1024];
- while (Over != 0)
- {
- unsigned long long toread = std::min((unsigned long long) sizeof(buffer), Over);
- if (Read(buffer, toread) == false)
- return FileFdError("Unable to seek ahead %llu",Over);
- Over -= toread;
- }
- return true;
- }
-
- off_t res;
-#ifdef HAVE_ZLIB
- if (d != NULL && d->gz != NULL)
- res = gzseek(d->gz,Over,SEEK_CUR);
- else
-#endif
- res = lseek(iFd,Over,SEEK_CUR);
- if (res < 0)
- return FileFdError("Unable to seek ahead %llu",Over);
- if (d != NULL)
- d->seekpos = res;
-
- return true;
+ if (d == nullptr)
+ return false;
+ return d->InternalSkip(Over);
}
/*}}}*/
-// FileFd::Truncate - Truncate the file /*{{{*/
-// ---------------------------------------------------------------------
-/* */
+// FileFd::Truncate - Truncate the file /*{{{*/
bool FileFd::Truncate(unsigned long long To)
{
+ if (d == nullptr)
+ return false;
// truncating /dev/null is always successful - as we get an error otherwise
if (To == 0 && FileName == "/dev/null")
return true;
-#if defined HAVE_ZLIB || defined HAVE_BZ2 || defined HAVE_LZMA
- if (d != NULL && (d->InternalStream() == true
-#ifdef HAVE_ZLIB
- || d->gz != NULL
-#endif
- ))
- return FileFdError("Truncating compressed files is not implemented (%s)", FileName.c_str());
-#endif
- if (ftruncate(iFd,To) != 0)
- return FileFdError("Unable to truncate to %llu",To);
-
- return true;
+ return d->InternalTruncate(To);
}
/*}}}*/
// FileFd::Tell - Current seek position /*{{{*/
/* */
unsigned long long FileFd::Tell()
{
- // In theory, we could just return seekpos here always instead of
- // seeking around, but not all users of FileFd use always Seek() and co
- // so d->seekpos isn't always true and we can just use it as a hint if
- // we have nothing else, but not always as an authority…
- if (d != NULL && (d->pipe == true || d->InternalStream() == true))
- return d->seekpos;
-
- off_t Res;
-#ifdef HAVE_ZLIB
- if (d != NULL && d->gz != NULL)
- Res = gztell(d->gz);
- else
-#endif
- Res = lseek(iFd,0,SEEK_CUR);
+ if (d == nullptr)
+ return false;
+ off_t const Res = d->InternalTell();
if (Res == (off_t)-1)
FileFdErrno("lseek","Failed to determine the current file position");
- if (d != NULL)
- d->seekpos = Res;
+ d->set_seekpos(Res);
return Res;
}
/*}}}*/
static bool StatFileFd(char const * const msg, int const iFd, std::string const &FileName, struct stat &Buf, FileFdPrivate * const d) /*{{{*/
{
- bool ispipe = (d != NULL && d->pipe == true);
+ bool ispipe = (d != NULL && d->get_is_pipe() == true);
if (ispipe == false)
{
if (fstat(iFd,&Buf) != 0)
// we set it here, too, as we get the info here for free
// in theory the Open-methods should take care of it already
if (d != NULL)
- d->pipe = true;
+ d->set_is_pipe(true);
if (stat(FileName.c_str(), &Buf) != 0)
return _error->Errno("fstat", "Unable to determine %s for file %s", msg, FileName.c_str());
}
}
/*}}}*/
// FileFd::Size - Return the size of the content in the file /*{{{*/
-// ---------------------------------------------------------------------
-/* */
unsigned long long FileFd::Size()
{
- unsigned long long size = FileSize();
-
- // for compressor pipes st_size is undefined and at 'best' zero,
- // so we 'read' the content and 'seek' back - see there
- if (d != NULL && (d->pipe == true || (d->InternalStream() == true && size > 0)))
- {
- unsigned long long const oldSeek = Tell();
- char ignore[1000];
- unsigned long long read = 0;
- do {
- if (Read(ignore, sizeof(ignore), &read) == false)
- {
- Seek(oldSeek);
- return 0;
- }
- } while(read != 0);
- size = Tell();
- Seek(oldSeek);
- }
-#ifdef HAVE_ZLIB
- // only check gzsize if we are actually a gzip file, just checking for
- // "gz" is not sufficient as uncompressed files could be opened with
- // gzopen in "direct" mode as well
- else if (d != NULL && d->gz && !gzdirect(d->gz) && size > 0)
- {
- off_t const oldPos = lseek(iFd,0,SEEK_CUR);
- /* unfortunately zlib.h doesn't provide a gzsize(), so we have to do
- * this ourselves; the original (uncompressed) file size is the last 32
- * bits of the file */
- // FIXME: Size for gz-files is limited by 32bit… no largefile support
- if (lseek(iFd, -4, SEEK_END) < 0)
- {
- FileFdErrno("lseek","Unable to seek to end of gzipped file");
- return 0;
- }
- uint32_t size = 0;
- if (read(iFd, &size, 4) != 4)
- {
- FileFdErrno("read","Unable to read original size of gzipped file");
- return 0;
- }
- size = le32toh(size);
-
- if (lseek(iFd, oldPos, SEEK_SET) < 0)
- {
- FileFdErrno("lseek","Unable to seek in gzipped file");
- return 0;
- }
-
- return size;
- }
-#endif
-
- return size;
+ if (d == nullptr)
+ return false;
+ return d->InternalSize();
}
/*}}}*/
// FileFd::Close - Close the file if the close flag is set /*{{{*/
/* */
bool FileFd::Close()
{
+ if (Flush() == false)
+ return false;
if (iFd == -1)
return true;
{
if ((Flags & Compressed) != Compressed && iFd > 0 && close(iFd) != 0)
Res &= _error->Errno("close",_("Problem closing the file %s"), FileName.c_str());
- if (d != NULL)
- {
- Res &= d->CloseDown(FileName);
- delete d;
- d = NULL;
- }
+ }
+
+ if (d != NULL)
+ {
+ Res &= d->InternalClose(FileName);
+ delete d;
+ d = NULL;
}
if ((Flags & Replace) == Replace) {
if ((Flags & Fail) == Fail && (Flags & DelOnFail) == DelOnFail &&
FileName.empty() == false)
- if (unlink(FileName.c_str()) != 0)
- Res &= _error->WarningE("unlnk",_("Problem unlinking the file %s"), FileName.c_str());
+ Res &= RemoveFile("FileFd::Close", FileName);
if (Res == false)
Flags |= Fail;
return false;
}
/*}}}*/
-
-APT_DEPRECATED gzFile FileFd::gzFd() {
+gzFile FileFd::gzFd() { /*{{{*/
#ifdef HAVE_ZLIB
- return d->gz;
+ GzipFileFdPrivate * const gzipd = dynamic_cast<GzipFileFdPrivate*>(d);
+ if (gzipd == nullptr)
+ return nullptr;
+ else
+ return gzipd->gz;
#else
- return NULL;
+ return nullptr;
#endif
}
+ /*}}}*/
// Glob - wrapper around "glob()" /*{{{*/
std::vector<std::string> Glob(std::string const &pattern, int flags)
struct stat st;
if (!tmpdir || strlen(tmpdir) == 0 || // tmpdir is set
- stat(tmpdir, &st) != 0 || (st.st_mode & S_IFDIR) == 0 || // exists and is directory
- access(tmpdir, R_OK | W_OK | X_OK) != 0 // current user has rwx access to directory
- )
+ stat(tmpdir, &st) != 0 || (st.st_mode & S_IFDIR) == 0) // exists and is directory
+ tmpdir = "/tmp";
+ else if (geteuid() != 0 && // root can do everything anyway
+ faccessat(-1, tmpdir, R_OK | W_OK | X_OK, AT_EACCESS | AT_SYMLINK_NOFOLLOW) != 0) // current user has rwx access to directory
tmpdir = "/tmp";
return string(tmpdir);
+}
+std::string GetTempDir(std::string const &User)
+{
+ // no need/possibility to drop privs
+ if(getuid() != 0 || User.empty() || User == "root")
+ return GetTempDir();
+
+ struct passwd const * const pw = getpwnam(User.c_str());
+ if (pw == NULL)
+ return GetTempDir();
+
+ gid_t const old_euid = geteuid();
+ gid_t const old_egid = getegid();
+ if (setegid(pw->pw_gid) != 0)
+ _error->Errno("setegid", "setegid %u failed", pw->pw_gid);
+ if (seteuid(pw->pw_uid) != 0)
+ _error->Errno("seteuid", "seteuid %u failed", pw->pw_uid);
+
+ std::string const tmp = GetTempDir();
+
+ if (seteuid(old_euid) != 0)
+ _error->Errno("seteuid", "seteuid %u failed", old_euid);
+ if (setegid(old_egid) != 0)
+ _error->Errno("setegid", "setegid %u failed", old_egid);
+
+ return tmp;
}
/*}}}*/
FileFd* GetTempFile(std::string const &Prefix, bool ImmediateUnlink, FileFd * const TmpFd) /*{{{*/
{
close(Pipe[1]);
fd = Pipe[0];
- } else if(Mode == FileFd::WriteOnly)
+ }
+ else if(Mode == FileFd::WriteOnly)
{
close(Pipe[0]);
fd = Pipe[1];
}
+ else
+ return _error->Error("Popen supports ReadOnly (x)or WriteOnly mode only");
Fd.OpenDescriptor(fd, Mode, FileFd::None, true);
return true;
// empty setting disables privilege dropping - this also ensures
// backward compatibility, see bug #764506
const std::string toUser = _config->Find("APT::Sandbox::User");
- if (toUser.empty())
+ if (toUser.empty() || toUser == "root")
return true;
+ // a lot can go wrong trying to drop privileges completely,
+ // so ideally we would like to verify that we have done it –
+ // but the verify asks for too much in case of fakeroot (and alike)
+ // [Specific checks can be overridden with dedicated options]
+ bool const VerifySandboxing = _config->FindB("APT::Sandbox::Verify", false);
+
// uid will be 0 in the end, but gid might be different anyway
uid_t const old_uid = getuid();
gid_t const old_gid = getgid();
return _error->Errno("seteuid", "Failed to seteuid");
#endif
- // Verify that the user has only a single group, and the correct one
- gid_t groups[1];
- if (getgroups(1, groups) != 1)
- return _error->Errno("getgroups", "Could not get new groups");
- if (groups[0] != pw->pw_gid)
- return _error->Error("Could not switch group");
-
- // Verify that gid, egid, uid, and euid changed
- if (getgid() != pw->pw_gid)
- return _error->Error("Could not switch group");
- if (getegid() != pw->pw_gid)
- return _error->Error("Could not switch effective group");
- if (getuid() != pw->pw_uid)
- return _error->Error("Could not switch user");
- if (geteuid() != pw->pw_uid)
- return _error->Error("Could not switch effective user");
+ // disabled by default as fakeroot doesn't implement getgroups currently (#806521)
+ if (VerifySandboxing == true || _config->FindB("APT::Sandbox::Verify::Groups", false) == true)
+ {
+ // Verify that the user isn't still in any supplementary groups
+ long const ngroups_max = sysconf(_SC_NGROUPS_MAX);
+ std::unique_ptr<gid_t[]> gidlist(new gid_t[ngroups_max]);
+ if (unlikely(gidlist == NULL))
+ return _error->Error("Allocation of a list of size %lu for getgroups failed", ngroups_max);
+ ssize_t gidlist_nr;
+ if ((gidlist_nr = getgroups(ngroups_max, gidlist.get())) < 0)
+ return _error->Errno("getgroups", "Could not get new groups (%lu)", ngroups_max);
+ for (ssize_t i = 0; i < gidlist_nr; ++i)
+ if (gidlist[i] != pw->pw_gid)
+ return _error->Error("Could not switch group, user %s is still in group %d", toUser.c_str(), gidlist[i]);
+ }
+
+ // enabled by default as all fakeroot-lookalikes should fake that accordingly
+ if (VerifySandboxing == true || _config->FindB("APT::Sandbox::Verify::IDs", true) == true)
+ {
+ // Verify that gid, egid, uid, and euid changed
+ if (getgid() != pw->pw_gid)
+ return _error->Error("Could not switch group");
+ if (getegid() != pw->pw_gid)
+ return _error->Error("Could not switch effective group");
+ if (getuid() != pw->pw_uid)
+ return _error->Error("Could not switch user");
+ if (geteuid() != pw->pw_uid)
+ return _error->Error("Could not switch effective user");
#ifdef HAVE_GETRESUID
- // verify that the saved set-user-id was changed as well
- uid_t ruid = 0;
- uid_t euid = 0;
- uid_t suid = 0;
- if (getresuid(&ruid, &euid, &suid))
- return _error->Errno("getresuid", "Could not get saved set-user-ID");
- if (suid != pw->pw_uid)
- return _error->Error("Could not switch saved set-user-ID");
+ // verify that the saved set-user-id was changed as well
+ uid_t ruid = 0;
+ uid_t euid = 0;
+ uid_t suid = 0;
+ if (getresuid(&ruid, &euid, &suid))
+ return _error->Errno("getresuid", "Could not get saved set-user-ID");
+ if (suid != pw->pw_uid)
+ return _error->Error("Could not switch saved set-user-ID");
#endif
#ifdef HAVE_GETRESGID
- // verify that the saved set-group-id was changed as well
- gid_t rgid = 0;
- gid_t egid = 0;
- gid_t sgid = 0;
- if (getresgid(&rgid, &egid, &sgid))
- return _error->Errno("getresuid", "Could not get saved set-group-ID");
- if (sgid != pw->pw_gid)
- return _error->Error("Could not switch saved set-group-ID");
+ // verify that the saved set-group-id was changed as well
+ gid_t rgid = 0;
+ gid_t egid = 0;
+ gid_t sgid = 0;
+ if (getresgid(&rgid, &egid, &sgid))
+ return _error->Errno("getresuid", "Could not get saved set-group-ID");
+ if (sgid != pw->pw_gid)
+ return _error->Error("Could not switch saved set-group-ID");
#endif
+ }
- // Check that uid and gid changes do not work anymore
- if (pw->pw_gid != old_gid && (setgid(old_gid) != -1 || setegid(old_gid) != -1))
- return _error->Error("Could restore a gid to root, privilege dropping did not work");
+ // disabled as fakeroot doesn't forbid (by design) (re)gaining root from unprivileged
+ if (VerifySandboxing == true || _config->FindB("APT::Sandbox::Verify::Regain", false) == true)
+ {
+ // Check that uid and gid changes do not work anymore
+ if (pw->pw_gid != old_gid && (setgid(old_gid) != -1 || setegid(old_gid) != -1))
+ return _error->Error("Could restore a gid to root, privilege dropping did not work");
- if (pw->pw_uid != old_uid && (setuid(old_uid) != -1 || seteuid(old_uid) != -1))
- return _error->Error("Could restore a uid to root, privilege dropping did not work");
+ if (pw->pw_uid != old_uid && (setuid(old_uid) != -1 || seteuid(old_uid) != -1))
+ return _error->Error("Could restore a uid to root, privilege dropping did not work");
+ }
return true;
}
projects / apt.git / blobdiff