X-Git-Url: https://git.saurik.com/apt.git/blobdiff_plain/b3db9d81325769fb9d9c6518f9d917f90328bef1..9fa247dc9ba2aa28ae564e96cba5b2b23bcac91b:/apt-pkg/contrib/fileutl.cc diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index 9b3618082..dd36ffa79 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -63,6 +63,9 @@ #ifdef HAVE_LZMA #include #endif +#ifdef HAVE_LZ4 + #include +#endif #include #include @@ -75,6 +78,9 @@ using namespace std; +/* Should be a multiple of the common page size (4096) */ +static constexpr unsigned long long APT_BUFFER_SIZE = 64 * 1024; + // RunScripts - Run a set of scripts from a configuration subtree /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -160,7 +166,7 @@ bool CopyFile(FileFd &From,FileFd &To) return false; // Buffered copy between fds - constexpr size_t BufSize = 64000; + constexpr size_t BufSize = APT_BUFFER_SIZE; std::unique_ptr Buf(new unsigned char[BufSize]); unsigned long long ToRead = 0; do { @@ -688,6 +694,25 @@ string flAbsPath(string File) return AbsPath; } /*}}}*/ +std::string flNormalize(std::string file) /*{{{*/ +{ + if (file.empty()) + return file; + // do some normalisation by removing // and /./ from the path + size_t found = string::npos; + while ((found = file.find("/./")) != string::npos) + file.replace(found, 3, "/"); + while ((found = file.find("//")) != string::npos) + file.replace(found, 2, "/"); + + if (APT::String::Startswith(file, "/dev/null")) + { + file.erase(strlen("/dev/null")); + return file; + } + return file; +} + /*}}}*/ // SetCloseExec - Set the close on exec flag /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -910,62 +935,130 @@ bool ChangeOwnerAndPermissionOfFile(char const * const requester, char const * c // ensure the file is owned by root and has good permissions struct passwd const * const pw = getpwnam(user); struct group const * const gr = getgrnam(group); - if (pw != NULL && gr != NULL && chown(file, pw->pw_uid, gr->gr_gid) != 0) + if (pw != NULL && gr != NULL && lchown(file, pw->pw_uid, gr->gr_gid) != 0) Res &= _error->WarningE(requester, "chown to %s:%s of file %s failed", user, group, file); } + struct stat Buf; + if (lstat(file, &Buf) != 0 || S_ISLNK(Buf.st_mode)) + return Res; if (chmod(file, mode) != 0) Res &= _error->WarningE(requester, "chmod 0%o of file %s failed", mode, file); return Res; } /*}}}*/ -class FileFdPrivate { /*{{{*/ +struct APT_HIDDEN simple_buffer { /*{{{*/ + size_t buffersize_max = 0; + unsigned long long bufferstart = 0; + unsigned long long bufferend = 0; + char *buffer = nullptr; + + simple_buffer() { + reset(4096); + } + ~simple_buffer() { + delete[] buffer; + } + + const char *get() const { return buffer + bufferstart; } + char *get() { return buffer + bufferstart; } + const char *getend() const { return buffer + bufferend; } + char *getend() { return buffer + bufferend; } + bool empty() const { return bufferend <= bufferstart; } + bool full() const { return bufferend == buffersize_max; } + unsigned long long free() const { return buffersize_max - bufferend; } + unsigned long long size() const { return bufferend-bufferstart; } + void reset(size_t size) + { + if (size > buffersize_max) { + delete[] buffer; + buffersize_max = size; + buffer = new char[size]; + } + reset(); + } + void reset() { bufferend = bufferstart = 0; } + ssize_t read(void *to, unsigned long long requested_size) APT_MUSTCHECK + { + if (size() < requested_size) + requested_size = size(); + memcpy(to, buffer + bufferstart, requested_size); + bufferstart += requested_size; + if (bufferstart == bufferend) + bufferstart = bufferend = 0; + return requested_size; + } + ssize_t write(const void *from, unsigned long long requested_size) APT_MUSTCHECK + { + if (free() < requested_size) + requested_size = free(); + memcpy(getend(), from, requested_size); + bufferend += requested_size; + if (bufferstart == bufferend) + bufferstart = bufferend = 0; + return requested_size; + } +}; + /*}}}*/ + +class APT_HIDDEN FileFdPrivate { /*{{{*/ + friend class BufferedWriteFileFdPrivate; protected: FileFd * const filefd; - size_t buffersize_max = 0; - std::unique_ptr buffer; - unsigned long long buffersize = 0; -public: + simple_buffer buffer; int compressed_fd; pid_t compressor_pid; bool is_pipe; APT::Configuration::Compressor compressor; unsigned int openmode; unsigned long long seekpos; - FileFdPrivate(FileFd * const pfilefd) : filefd(pfilefd), buffer(nullptr), +public: + + explicit FileFdPrivate(FileFd * const pfilefd) : filefd(pfilefd), compressed_fd(-1), compressor_pid(-1), is_pipe(false), openmode(0), seekpos(0) {}; + virtual APT::Configuration::Compressor get_compressor() const + { + return compressor; + } + virtual void set_compressor(APT::Configuration::Compressor const &compressor) + { + this->compressor = compressor; + } + virtual unsigned int get_openmode() const + { + return openmode; + } + virtual void set_openmode(unsigned int openmode) + { + this->openmode = openmode; + } + virtual bool get_is_pipe() const + { + return is_pipe; + } + virtual void set_is_pipe(bool is_pipe) + { + this->is_pipe = is_pipe; + } + virtual unsigned long long get_seekpos() const + { + return seekpos; + } + virtual void set_seekpos(unsigned long long seekpos) + { + this->seekpos = seekpos; + } virtual bool InternalOpen(int const iFd, unsigned int const Mode) = 0; ssize_t InternalRead(void * To, unsigned long long Size) { - unsigned long long tmp = 0; - if (buffersize != 0) + // Drain the buffer if needed. + if (buffer.empty() == false) { - if (buffersize >= Size) - { - memcpy(To, buffer.get(), Size); - if (buffersize == Size) - { - buffersize = 0; - } - else - { - buffersize -= Size; - memmove(buffer.get(), buffer.get() + Size, buffersize); - } - return Size; - } - else - { - memcpy(To, buffer.get(), buffersize); - Size -= buffersize; - To = static_cast(To) + buffersize; - tmp = buffersize; - buffersize = 0; - } + return buffer.read(To, Size); } - return InternalUnbufferedRead(To, Size) + tmp; + return InternalUnbufferedRead(To, Size); } virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) = 0; virtual bool InternalReadError() { return filefd->FileFdErrno("read",_("Read error")); } @@ -973,28 +1066,21 @@ public: { if (unlikely(Size == 0)) return nullptr; + // Read one byte less than buffer size to have space for trailing 0. --Size; - To[0] = '\0'; - if (unlikely(Size == 0)) - return To; + char * const InitialTo = To; - do { - if (buffersize == 0) + while (Size > 0) { + if (buffer.empty() == true) { - if (buffer.get() == nullptr) - { - buffer.reset(new char[Size]); - buffersize_max = Size; - } + buffer.reset(); unsigned long long actualread = 0; - if (filefd->Read(buffer.get(), buffersize_max, &actualread) == false) + if (filefd->Read(buffer.getend(), buffer.free(), &actualread) == false) return nullptr; - buffersize = actualread; - if (buffersize == 0) + buffer.bufferend = actualread; + if (buffer.size() == 0) { - buffer.reset(nullptr); - buffersize_max = 0; if (To == InitialTo) return nullptr; break; @@ -1002,35 +1088,25 @@ public: filefd->Flags &= ~FileFd::HitEof; } - unsigned long long const OutputSize = std::min(Size, buffersize); + unsigned long long const OutputSize = std::min(Size, buffer.size()); char const * const newline = static_cast(memchr(buffer.get(), '\n', OutputSize)); + // Read until end of line or up to Size bytes from the buffer. + unsigned long long actualread = buffer.read(To, + (newline != nullptr) + ? (newline - buffer.get()) + 1 + : OutputSize); + To += actualread; + Size -= actualread; if (newline != nullptr) - { - size_t length = (newline - buffer.get()); - ++length; - memcpy(To, buffer.get(), length); - To += length; - if (length < buffersize) - { - buffersize -= length; - memmove(buffer.get(), buffer.get() + length, buffersize); - } - else - buffersize = 0; break; - } - else - { - memcpy(To, buffer.get(), OutputSize); - To += OutputSize; - Size -= OutputSize; - buffersize -= OutputSize; - memmove(buffer.get(), buffer.get() + OutputSize, buffersize); - } - } while (Size > 0); + } *To = '\0'; return InitialTo; } + virtual bool InternalFlush() + { + return true; + } virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) = 0; virtual bool InternalWriteError() { return filefd->FileFdErrno("write",_("Write error")); } virtual bool InternalSeek(unsigned long long const To) @@ -1064,7 +1140,8 @@ public: if (filefd->OpenInternDescriptor(openmode, compressor) == false) return filefd->FileFdError("Seek on file %s because it couldn't be reopened", filefd->FileName.c_str()); - buffersize = 0; + buffer.reset(); + set_seekpos(0); if (To != 0) return filefd->Skip(To); @@ -1094,7 +1171,7 @@ public: // seeking around, but not all users of FileFd use always Seek() and co // so d->seekpos isn't always true and we can just use it as a hint if // we have nothing else, but not always as an authority… - return seekpos - buffersize; + return seekpos - buffer.size(); } virtual unsigned long long InternalSize() { @@ -1121,11 +1198,156 @@ public: virtual ~FileFdPrivate() {} }; /*}}}*/ -class GzipFileFdPrivate: public FileFdPrivate { /*{{{*/ +class APT_HIDDEN BufferedWriteFileFdPrivate : public FileFdPrivate { /*{{{*/ +protected: + FileFdPrivate *wrapped; + simple_buffer writebuffer; + +public: + + explicit BufferedWriteFileFdPrivate(FileFdPrivate *Priv) : + FileFdPrivate(Priv->filefd), wrapped(Priv) {}; + + virtual APT::Configuration::Compressor get_compressor() const APT_OVERRIDE + { + return wrapped->get_compressor(); + } + virtual void set_compressor(APT::Configuration::Compressor const &compressor) APT_OVERRIDE + { + return wrapped->set_compressor(compressor); + } + virtual unsigned int get_openmode() const APT_OVERRIDE + { + return wrapped->get_openmode(); + } + virtual void set_openmode(unsigned int openmode) APT_OVERRIDE + { + return wrapped->set_openmode(openmode); + } + virtual bool get_is_pipe() const APT_OVERRIDE + { + return wrapped->get_is_pipe(); + } + virtual void set_is_pipe(bool is_pipe) APT_OVERRIDE + { + FileFdPrivate::set_is_pipe(is_pipe); + wrapped->set_is_pipe(is_pipe); + } + virtual unsigned long long get_seekpos() const APT_OVERRIDE + { + return wrapped->get_seekpos(); + } + virtual void set_seekpos(unsigned long long seekpos) APT_OVERRIDE + { + return wrapped->set_seekpos(seekpos); + } + virtual bool InternalOpen(int const iFd, unsigned int const Mode) APT_OVERRIDE + { + if (InternalFlush() == false) + return false; + return wrapped->InternalOpen(iFd, Mode); + } + virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) APT_OVERRIDE + { + if (InternalFlush() == false) + return -1; + return wrapped->InternalUnbufferedRead(To, Size); + + } + virtual bool InternalReadError() APT_OVERRIDE + { + return wrapped->InternalReadError(); + } + virtual char * InternalReadLine(char * To, unsigned long long Size) APT_OVERRIDE + { + if (InternalFlush() == false) + return nullptr; + return wrapped->InternalReadLine(To, Size); + } + virtual bool InternalFlush() APT_OVERRIDE + { + while (writebuffer.empty() == false) { + auto written = wrapped->InternalWrite(writebuffer.get(), + writebuffer.size()); + // Ignore interrupted syscalls + if (written < 0 && errno == EINTR) + continue; + if (written < 0) + return wrapped->InternalWriteError(); + + writebuffer.bufferstart += written; + } + writebuffer.reset(); + return wrapped->InternalFlush(); + } + virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) APT_OVERRIDE + { + // Optimisation: If the buffer is empty and we have more to write than + // would fit in the buffer (or equal number of bytes), write directly. + if (writebuffer.empty() == true && Size >= writebuffer.free()) + return wrapped->InternalWrite(From, Size); + + // Write as much into the buffer as possible and then flush if needed + auto written = writebuffer.write(From, Size); + + if (writebuffer.full() && InternalFlush() == false) + return -1; + + return written; + } + virtual bool InternalWriteError() APT_OVERRIDE + { + return wrapped->InternalWriteError(); + } + virtual bool InternalSeek(unsigned long long const To) APT_OVERRIDE + { + if (InternalFlush() == false) + return false; + return wrapped->InternalSeek(To); + } + virtual bool InternalSkip(unsigned long long Over) APT_OVERRIDE + { + if (InternalFlush() == false) + return false; + return wrapped->InternalSkip(Over); + } + virtual bool InternalTruncate(unsigned long long const Size) APT_OVERRIDE + { + if (InternalFlush() == false) + return false; + return wrapped->InternalTruncate(Size); + } + virtual unsigned long long InternalTell() APT_OVERRIDE + { + if (InternalFlush() == false) + return -1; + return wrapped->InternalTell(); + } + virtual unsigned long long InternalSize() APT_OVERRIDE + { + if (InternalFlush() == false) + return -1; + return wrapped->InternalSize(); + } + virtual bool InternalClose(std::string const &FileName) APT_OVERRIDE + { + return wrapped->InternalClose(FileName); + } + virtual bool InternalAlwaysAutoClose() const APT_OVERRIDE + { + return wrapped->InternalAlwaysAutoClose(); + } + virtual ~BufferedWriteFileFdPrivate() + { + delete wrapped; + } +}; + /*}}}*/ +class APT_HIDDEN GzipFileFdPrivate: public FileFdPrivate { /*{{{*/ #ifdef HAVE_ZLIB public: gzFile gz; - virtual bool InternalOpen(int const iFd, unsigned int const Mode) override + virtual bool InternalOpen(int const iFd, unsigned int const Mode) APT_OVERRIDE { if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite) gz = gzdopen(iFd, "r+"); @@ -1136,11 +1358,11 @@ public: filefd->Flags |= FileFd::Compressed; return gz != nullptr; } - virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) override + virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) APT_OVERRIDE { return gzread(gz, To, Size); } - virtual bool InternalReadError() override + virtual bool InternalReadError() APT_OVERRIDE { int err; char const * const errmsg = gzerror(gz, &err); @@ -1148,15 +1370,15 @@ public: return filefd->FileFdError("gzread: %s (%d: %s)", _("Read error"), err, errmsg); return FileFdPrivate::InternalReadError(); } - virtual char * InternalReadLine(char * To, unsigned long long Size) override + virtual char * InternalReadLine(char * To, unsigned long long Size) APT_OVERRIDE { return gzgets(gz, To, Size); } - virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override + virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) APT_OVERRIDE { return gzwrite(gz,From,Size); } - virtual bool InternalWriteError() override + virtual bool InternalWriteError() APT_OVERRIDE { int err; char const * const errmsg = gzerror(gz, &err); @@ -1164,26 +1386,25 @@ public: return filefd->FileFdError("gzwrite: %s (%d: %s)", _("Write error"), err, errmsg); return FileFdPrivate::InternalWriteError(); } - virtual bool InternalSeek(unsigned long long const To) override + virtual bool InternalSeek(unsigned long long const To) APT_OVERRIDE { off_t const res = gzseek(gz, To, SEEK_SET); if (res != (off_t)To) return filefd->FileFdError("Unable to seek to %llu", To); seekpos = To; - buffersize = 0; + buffer.reset(); return true; } - virtual bool InternalSkip(unsigned long long Over) override + virtual bool InternalSkip(unsigned long long Over) APT_OVERRIDE { - if (Over >= buffersize) + if (Over >= buffer.size()) { - Over -= buffersize; - buffersize = 0; + Over -= buffer.size(); + buffer.reset(); } else { - buffersize -= Over; - memmove(buffer.get(), buffer.get() + Over, buffersize); + buffer.bufferstart += Over; return true; } if (Over == 0) @@ -1194,11 +1415,11 @@ public: seekpos = res; return true; } - virtual unsigned long long InternalTell() override + virtual unsigned long long InternalTell() APT_OVERRIDE { - return gztell(gz) - buffersize; + return gztell(gz) - buffer.size(); } - virtual unsigned long long InternalSize() override + virtual unsigned long long InternalSize() APT_OVERRIDE { unsigned long long filesize = FileFdPrivate::InternalSize(); // only check gzsize if we are actually a gzip file, just checking for @@ -1232,7 +1453,7 @@ public: } return size; } - virtual bool InternalClose(std::string const &FileName) override + virtual bool InternalClose(std::string const &FileName) APT_OVERRIDE { if (gz == nullptr) return true; @@ -1244,16 +1465,16 @@ public: return true; } - GzipFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), gz(nullptr) {} + explicit GzipFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), gz(nullptr) {} virtual ~GzipFileFdPrivate() { InternalClose(""); } #endif }; /*}}}*/ -class Bz2FileFdPrivate: public FileFdPrivate { /*{{{*/ +class APT_HIDDEN Bz2FileFdPrivate: public FileFdPrivate { /*{{{*/ #ifdef HAVE_BZ2 BZFILE* bz2; public: - virtual bool InternalOpen(int const iFd, unsigned int const Mode) override + virtual bool InternalOpen(int const iFd, unsigned int const Mode) APT_OVERRIDE { if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite) bz2 = BZ2_bzdopen(iFd, "r+"); @@ -1264,11 +1485,11 @@ public: filefd->Flags |= FileFd::Compressed; return bz2 != nullptr; } - virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) override + virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) APT_OVERRIDE { return BZ2_bzread(bz2, To, Size); } - virtual bool InternalReadError() override + virtual bool InternalReadError() APT_OVERRIDE { int err; char const * const errmsg = BZ2_bzerror(bz2, &err); @@ -1276,11 +1497,11 @@ public: return filefd->FileFdError("BZ2_bzread: %s %s (%d: %s)", filefd->FileName.c_str(), _("Read error"), err, errmsg); return FileFdPrivate::InternalReadError(); } - virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override + virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) APT_OVERRIDE { return BZ2_bzwrite(bz2, (void*)From, Size); } - virtual bool InternalWriteError() override + virtual bool InternalWriteError() APT_OVERRIDE { int err; char const * const errmsg = BZ2_bzerror(bz2, &err); @@ -1288,8 +1509,8 @@ public: return filefd->FileFdError("BZ2_bzwrite: %s %s (%d: %s)", filefd->FileName.c_str(), _("Write error"), err, errmsg); return FileFdPrivate::InternalWriteError(); } - virtual bool InternalStream() const override { return true; } - virtual bool InternalClose(std::string const &) override + virtual bool InternalStream() const APT_OVERRIDE { return true; } + virtual bool InternalClose(std::string const &) APT_OVERRIDE { if (bz2 == nullptr) return true; @@ -1298,25 +1519,185 @@ public: return true; } - Bz2FileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), bz2(nullptr) {} + explicit Bz2FileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), bz2(nullptr) {} virtual ~Bz2FileFdPrivate() { InternalClose(""); } #endif }; /*}}}*/ -class LzmaFileFdPrivate: public FileFdPrivate { /*{{{*/ +class APT_HIDDEN Lz4FileFdPrivate: public FileFdPrivate { /*{{{*/ + static constexpr unsigned long long LZ4_HEADER_SIZE = 19; + static constexpr unsigned long long LZ4_FOOTER_SIZE = 4; +#ifdef HAVE_LZ4 + LZ4F_decompressionContext_t dctx; + LZ4F_compressionContext_t cctx; + LZ4F_errorCode_t res; + FileFd backend; + simple_buffer lz4_buffer; + // Count of bytes that the decompressor expects to read next, or buffer size. + size_t next_to_load = APT_BUFFER_SIZE; +public: + virtual bool InternalOpen(int const iFd, unsigned int const Mode) APT_OVERRIDE + { + if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite) + return _error->Error("lz4 only supports write or read mode"); + + if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly) { + res = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION); + lz4_buffer.reset(LZ4F_compressBound(APT_BUFFER_SIZE, nullptr) + + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE); + } else { + res = LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION); + lz4_buffer.reset(APT_BUFFER_SIZE); + } + + filefd->Flags |= FileFd::Compressed; + + if (LZ4F_isError(res)) + return false; + + unsigned int flags = (Mode & (FileFd::WriteOnly|FileFd::ReadOnly)); + if (backend.OpenDescriptor(iFd, flags, FileFd::None, true) == false) + return false; + + // Write the file header + if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly) + { + res = LZ4F_compressBegin(cctx, lz4_buffer.buffer, lz4_buffer.buffersize_max, nullptr); + if (LZ4F_isError(res) || backend.Write(lz4_buffer.buffer, res) == false) + return false; + } + + return true; + } + virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) APT_OVERRIDE + { + /* Keep reading as long as the compressor still wants to read */ + while (next_to_load) { + // Fill compressed buffer; + if (lz4_buffer.empty()) { + unsigned long long read; + /* Reset - if LZ4 decompressor wants to read more, allocate more */ + lz4_buffer.reset(next_to_load); + if (backend.Read(lz4_buffer.getend(), lz4_buffer.free(), &read) == false) + return -1; + lz4_buffer.bufferend += read; + + /* Expected EOF */ + if (read == 0) { + res = -1; + return filefd->FileFdError("LZ4F: %s %s", + filefd->FileName.c_str(), + _("Unexpected end of file")), -1; + } + } + // Drain compressed buffer as far as possible. + size_t in = lz4_buffer.size(); + size_t out = Size; + + res = LZ4F_decompress(dctx, To, &out, lz4_buffer.get(), &in, nullptr); + if (LZ4F_isError(res)) + return -1; + + next_to_load = res; + lz4_buffer.bufferstart += in; + + if (out != 0) + return out; + } + + return 0; + } + virtual bool InternalReadError() APT_OVERRIDE + { + char const * const errmsg = LZ4F_getErrorName(res); + + return filefd->FileFdError("LZ4F: %s %s (%zu: %s)", filefd->FileName.c_str(), _("Read error"), res, errmsg); + } + virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) APT_OVERRIDE + { + unsigned long long const towrite = std::min(APT_BUFFER_SIZE, Size); + + res = LZ4F_compressUpdate(cctx, + lz4_buffer.buffer, lz4_buffer.buffersize_max, + From, towrite, nullptr); + + if (LZ4F_isError(res) || backend.Write(lz4_buffer.buffer, res) == false) + return -1; + + return towrite; + } + virtual bool InternalWriteError() APT_OVERRIDE + { + char const * const errmsg = LZ4F_getErrorName(res); + + return filefd->FileFdError("LZ4F: %s %s (%zu: %s)", filefd->FileName.c_str(), _("Write error"), res, errmsg); + } + virtual bool InternalStream() const APT_OVERRIDE { return true; } + + virtual bool InternalFlush() APT_OVERRIDE + { + return backend.Flush(); + } + + virtual bool InternalClose(std::string const &) APT_OVERRIDE + { + /* Reset variables */ + res = 0; + next_to_load = APT_BUFFER_SIZE; + + if (cctx != nullptr) + { + if (filefd->Failed() == false) + { + res = LZ4F_compressEnd(cctx, lz4_buffer.buffer, lz4_buffer.buffersize_max, nullptr); + if (LZ4F_isError(res) || backend.Write(lz4_buffer.buffer, res) == false) + return false; + if (!backend.Flush()) + return false; + } + if (!backend.Close()) + return false; + + res = LZ4F_freeCompressionContext(cctx); + cctx = nullptr; + } + + if (dctx != nullptr) + { + res = LZ4F_freeDecompressionContext(dctx); + dctx = nullptr; + } + if (backend.IsOpen()) + { + backend.Close(); + filefd->iFd = -1; + } + + return LZ4F_isError(res) == false; + } + + explicit Lz4FileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), dctx(nullptr), cctx(nullptr) {} + virtual ~Lz4FileFdPrivate() { + InternalClose(""); + } +#endif +}; + /*}}}*/ +class APT_HIDDEN LzmaFileFdPrivate: public FileFdPrivate { /*{{{*/ #ifdef HAVE_LZMA struct LZMAFILE { FILE* file; + FileFd * const filefd; uint8_t buffer[4096]; lzma_stream stream; lzma_ret err; bool eof; bool compressing; - LZMAFILE() : file(nullptr), eof(false), compressing(false) { buffer[0] = '\0'; } + LZMAFILE(FileFd * const fd) : file(nullptr), filefd(fd), eof(false), compressing(false) { buffer[0] = '\0'; } ~LZMAFILE() { - if (compressing == true) + if (compressing == true && filefd->Failed() == false) { size_t constexpr buffersize = sizeof(buffer)/sizeof(buffer[0]); while(true) @@ -1371,13 +1752,13 @@ class LzmaFileFdPrivate: public FileFdPrivate { /*{{{*/ return 6; } public: - virtual bool InternalOpen(int const iFd, unsigned int const Mode) override + virtual bool InternalOpen(int const iFd, unsigned int const Mode) APT_OVERRIDE { if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite) return filefd->FileFdError("ReadWrite mode is not supported for lzma/xz files %s", filefd->FileName.c_str()); if (lzma == nullptr) - lzma = new LzmaFileFdPrivate::LZMAFILE; + lzma = new LzmaFileFdPrivate::LZMAFILE(filefd); if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly) lzma->file = fdopen(iFd, "w"); else @@ -1423,7 +1804,7 @@ public: } return true; } - virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) override + virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) APT_OVERRIDE { ssize_t Res; if (lzma->eof == true) @@ -1459,12 +1840,13 @@ public: } return Res; } - virtual bool InternalReadError() override + virtual bool InternalReadError() APT_OVERRIDE { return filefd->FileFdError("lzma_read: %s (%d)", _("Read error"), lzma->err); } - virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override + virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) APT_OVERRIDE { + ssize_t Res; lzma->stream.next_in = (uint8_t *)From; lzma->stream.avail_in = Size; lzma->stream.next_out = lzma->buffer; @@ -1475,33 +1857,45 @@ public: size_t const n = sizeof(lzma->buffer)/sizeof(lzma->buffer[0]) - lzma->stream.avail_out; size_t const m = (n == 0) ? 0 : fwrite(lzma->buffer, 1, n, lzma->file); if (m != n) - return -1; + { + Res = -1; + errno = 0; + } else - return Size - lzma->stream.avail_in; + { + Res = Size - lzma->stream.avail_in; + if (Res == 0) + { + // lzma run was okay, but produced no output… + Res = -1; + errno = EINTR; + } + } + return Res; } - virtual bool InternalWriteError() override + virtual bool InternalWriteError() APT_OVERRIDE { return filefd->FileFdError("lzma_write: %s (%d)", _("Write error"), lzma->err); } - virtual bool InternalStream() const override { return true; } - virtual bool InternalClose(std::string const &) override + virtual bool InternalStream() const APT_OVERRIDE { return true; } + virtual bool InternalClose(std::string const &) APT_OVERRIDE { delete lzma; lzma = nullptr; return true; } - LzmaFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), lzma(nullptr) {} + explicit LzmaFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), lzma(nullptr) {} virtual ~LzmaFileFdPrivate() { InternalClose(""); } #endif }; /*}}}*/ -class PipedFileFdPrivate: public FileFdPrivate /*{{{*/ +class APT_HIDDEN PipedFileFdPrivate: public FileFdPrivate /*{{{*/ /* if we don't have a specific class dealing with library calls, we (un)compress by executing a specified binary and pipe in/out what we need */ { public: - virtual bool InternalOpen(int const, unsigned int const Mode) override + virtual bool InternalOpen(int const, unsigned int const Mode) APT_OVERRIDE { // collect zombies here in case we reopen if (compressor_pid > 0) @@ -1509,13 +1903,17 @@ public: if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite) return filefd->FileFdError("ReadWrite mode is not supported for file %s", filefd->FileName.c_str()); + if (compressor.Binary == "false") + return filefd->FileFdError("libapt has inbuilt support for the %s compression," + " but was forced to ignore it in favor of an external binary – which isn't installed.", compressor.Name.c_str()); bool const Comp = (Mode & FileFd::WriteOnly) == FileFd::WriteOnly; - if (Comp == false) + if (Comp == false && filefd->iFd != -1) { // Handle 'decompression' of empty files struct stat Buf; - fstat(filefd->iFd, &Buf); + if (fstat(filefd->iFd, &Buf) != 0) + return filefd->FileFdErrno("fstat", "Could not stat fd %d for file %s", filefd->iFd, filefd->FileName.c_str()); if (Buf.st_size == 0 && S_ISFIFO(Buf.st_mode) == false) return true; @@ -1536,7 +1934,7 @@ public: SetCloseExec(Pipe[J],true); compressed_fd = filefd->iFd; - is_pipe = true; + set_is_pipe(true); if (Comp == true) filefd->iFd = Pipe[1]; @@ -1558,12 +1956,6 @@ public: dup2(compressed_fd,STDIN_FILENO); dup2(Pipe[1],STDOUT_FILENO); } - int const nullfd = open("/dev/null", O_WRONLY); - if (nullfd != -1) - { - dup2(nullfd,STDERR_FILENO); - close(nullfd); - } SetCloseExec(STDOUT_FILENO,false); SetCloseExec(STDIN_FILENO,false); @@ -1599,64 +1991,68 @@ public: return true; } - virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) override + virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) APT_OVERRIDE { return read(filefd->iFd, To, Size); } - virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override + virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) APT_OVERRIDE { return write(filefd->iFd, From, Size); } - virtual bool InternalClose(std::string const &) override + virtual bool InternalClose(std::string const &) APT_OVERRIDE { bool Ret = true; + if (filefd->iFd != -1) + { + close(filefd->iFd); + filefd->iFd = -1; + } if (compressor_pid > 0) Ret &= ExecWait(compressor_pid, "FileFdCompressor", true); compressor_pid = -1; return Ret; } - PipedFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd) {} + explicit PipedFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd) {} virtual ~PipedFileFdPrivate() { InternalClose(""); } }; /*}}}*/ -class DirectFileFdPrivate: public FileFdPrivate /*{{{*/ +class APT_HIDDEN DirectFileFdPrivate: public FileFdPrivate /*{{{*/ { public: - virtual bool InternalOpen(int const, unsigned int const) override { return true; } - virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) override + virtual bool InternalOpen(int const, unsigned int const) APT_OVERRIDE { return true; } + virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) APT_OVERRIDE { return read(filefd->iFd, To, Size); } - virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override + virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) APT_OVERRIDE { // files opened read+write are strange and only really "supported" for direct files - if (buffersize != 0) + if (buffer.size() != 0) { - lseek(filefd->iFd, -buffersize, SEEK_CUR); - buffersize = 0; + lseek(filefd->iFd, -buffer.size(), SEEK_CUR); + buffer.reset(); } return write(filefd->iFd, From, Size); } - virtual bool InternalSeek(unsigned long long const To) override + virtual bool InternalSeek(unsigned long long const To) APT_OVERRIDE { off_t const res = lseek(filefd->iFd, To, SEEK_SET); if (res != (off_t)To) return filefd->FileFdError("Unable to seek to %llu", To); seekpos = To; - buffersize = 0; + buffer.reset(); return true; } - virtual bool InternalSkip(unsigned long long Over) override + virtual bool InternalSkip(unsigned long long Over) APT_OVERRIDE { - if (Over >= buffersize) + if (Over >= buffer.size()) { - Over -= buffersize; - buffersize = 0; + Over -= buffer.size(); + buffer.reset(); } else { - buffersize -= Over; - memmove(buffer.get(), buffer.get() + Over, buffersize); + buffer.bufferstart += Over; return true; } if (Over == 0) @@ -1667,34 +2063,34 @@ public: seekpos = res; return true; } - virtual bool InternalTruncate(unsigned long long const To) override + virtual bool InternalTruncate(unsigned long long const To) APT_OVERRIDE { - if (buffersize != 0) + if (buffer.size() != 0) { unsigned long long const seekpos = lseek(filefd->iFd, 0, SEEK_CUR); - if ((seekpos - buffersize) >= To) - buffersize = 0; + if ((seekpos - buffer.size()) >= To) + buffer.reset(); else if (seekpos >= To) - buffersize = (To - seekpos); + buffer.bufferend = (To - seekpos) + buffer.bufferstart; else - buffersize = 0; + buffer.reset(); } if (ftruncate(filefd->iFd, To) != 0) return filefd->FileFdError("Unable to truncate to %llu",To); return true; } - virtual unsigned long long InternalTell() override + virtual unsigned long long InternalTell() APT_OVERRIDE { - return lseek(filefd->iFd,0,SEEK_CUR) - buffersize; + return lseek(filefd->iFd,0,SEEK_CUR) - buffer.size(); } - virtual unsigned long long InternalSize() override + virtual unsigned long long InternalSize() APT_OVERRIDE { return filefd->FileSize(); } - virtual bool InternalClose(std::string const &) override { return true; } - virtual bool InternalAlwaysAutoClose() const override { return false; } + virtual bool InternalClose(std::string const &) APT_OVERRIDE { return true; } + virtual bool InternalAlwaysAutoClose() const APT_OVERRIDE { return false; } - DirectFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd) {} + explicit DirectFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd) {} virtual ~DirectFileFdPrivate() { InternalClose(""); } }; /*}}}*/ @@ -1776,6 +2172,7 @@ bool FileFd::Open(string FileName,unsigned int const Mode,CompressMode Compress, case Bzip2: name = "bzip2"; break; case Lzma: name = "lzma"; break; case Xz: name = "xz"; break; + case Lz4: name = "lz4"; break; case Auto: case Extension: // Unreachable @@ -1892,6 +2289,7 @@ bool FileFd::OpenDescriptor(int Fd, unsigned int const Mode, CompressMode Compre case Bzip2: name = "bzip2"; break; case Lzma: name = "lzma"; break; case Xz: name = "xz"; break; + case Lz4: name = "lz4"; break; case Auto: case Extension: if (AutoClose == true && Fd != -1) @@ -1953,14 +2351,20 @@ bool FileFd::OpenInternDescriptor(unsigned int const Mode, APT::Configuration::C APT_COMPRESS_INIT("xz", LzmaFileFdPrivate); APT_COMPRESS_INIT("lzma", LzmaFileFdPrivate); #endif +#ifdef HAVE_LZ4 + APT_COMPRESS_INIT("lz4", Lz4FileFdPrivate); +#endif #undef APT_COMPRESS_INIT else if (compressor.Name == "." || compressor.Binary.empty() == true) d = new DirectFileFdPrivate(this); else d = new PipedFileFdPrivate(this); - d->openmode = Mode; - d->compressor = compressor; + if (Mode & BufferedWrite) + d = new BufferedWriteFileFdPrivate(d); + + d->set_openmode(Mode); + d->set_compressor(compressor); if ((Flags & AutoClose) != AutoClose && d->InternalAlwaysAutoClose()) { // Need to duplicate fd here or gz/bz2 close for cleanup will close the fd as well @@ -1992,7 +2396,7 @@ FileFd::~FileFd() gracefully. */ bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual) { - if (d == nullptr) + if (d == nullptr || Failed()) return false; ssize_t Res = 1; errno = 0; @@ -2018,7 +2422,7 @@ bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual) To = (char *)To + Res; Size -= Res; if (d != NULL) - d->seekpos += Res; + d->set_seekpos(d->get_seekpos() + Res); if (Actual != 0) *Actual += Res; } @@ -2034,6 +2438,37 @@ bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual) } return FileFdError(_("read, still have %llu to read but none left"), Size); +} +bool FileFd::Read(int const Fd, void *To, unsigned long long Size, unsigned long long * const Actual) +{ + ssize_t Res = 1; + errno = 0; + if (Actual != nullptr) + *Actual = 0; + *static_cast(To) = '\0'; + while (Res > 0 && Size > 0) + { + Res = read(Fd, To, Size); + if (Res < 0) + { + if (errno == EINTR) + { + Res = 1; + errno = 0; + continue; + } + return _error->Errno("read", _("Read error")); + } + To = static_cast(To) + Res; + Size -= Res; + if (Actual != 0) + *Actual += Res; + } + if (Size == 0) + return true; + if (Actual != nullptr) + return true; + return _error->Error(_("read, still have %llu to read but none left"), Size); } /*}}}*/ // FileFd::ReadLine - Read a complete line from the file /*{{{*/ @@ -2043,30 +2478,49 @@ bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual) char* FileFd::ReadLine(char *To, unsigned long long const Size) { *To = '\0'; - if (d == nullptr) + if (d == nullptr || Failed()) return nullptr; return d->InternalReadLine(To, Size); } /*}}}*/ +// FileFd::Flush - Flush the file /*{{{*/ +bool FileFd::Flush() +{ + if (Failed()) + return false; + if (d == nullptr) + return true; + + return d->InternalFlush(); +} + /*}}}*/ // FileFd::Write - Write to the file /*{{{*/ bool FileFd::Write(const void *From,unsigned long long Size) { - if (d == nullptr) + if (d == nullptr || Failed()) return false; ssize_t Res = 1; errno = 0; while (Res > 0 && Size > 0) { Res = d->InternalWrite(From, Size); - if (Res < 0 && errno == EINTR) - continue; + if (Res < 0) + { + if (errno == EINTR) + { + // trick the while-loop into running again + Res = 1; + errno = 0; + continue; + } return d->InternalWriteError(); + } From = (char const *)From + Res; Size -= Res; if (d != NULL) - d->seekpos += Res; + d->set_seekpos(d->get_seekpos() + Res); } if (Size == 0) @@ -2099,7 +2553,7 @@ bool FileFd::Write(int Fd, const void *From, unsigned long long Size) // FileFd::Seek - Seek in the file /*{{{*/ bool FileFd::Seek(unsigned long long To) { - if (d == nullptr) + if (d == nullptr || Failed()) return false; Flags &= ~HitEof; return d->InternalSeek(To); @@ -2108,7 +2562,7 @@ bool FileFd::Seek(unsigned long long To) // FileFd::Skip - Skip over data in the file /*{{{*/ bool FileFd::Skip(unsigned long long Over) { - if (d == nullptr) + if (d == nullptr || Failed()) return false; return d->InternalSkip(Over); } @@ -2116,7 +2570,7 @@ bool FileFd::Skip(unsigned long long Over) // FileFd::Truncate - Truncate the file /*{{{*/ bool FileFd::Truncate(unsigned long long To) { - if (d == nullptr) + if (d == nullptr || Failed()) return false; // truncating /dev/null is always successful - as we get an error otherwise if (To == 0 && FileName == "/dev/null") @@ -2129,18 +2583,18 @@ bool FileFd::Truncate(unsigned long long To) /* */ unsigned long long FileFd::Tell() { - if (d == nullptr) + if (d == nullptr || Failed()) return false; off_t const Res = d->InternalTell(); if (Res == (off_t)-1) FileFdErrno("lseek","Failed to determine the current file position"); - d->seekpos = Res; + d->set_seekpos(Res); return Res; } /*}}}*/ static bool StatFileFd(char const * const msg, int const iFd, std::string const &FileName, struct stat &Buf, FileFdPrivate * const d) /*{{{*/ { - bool ispipe = (d != NULL && d->is_pipe == true); + bool ispipe = (d != NULL && d->get_is_pipe() == true); if (ispipe == false) { if (fstat(iFd,&Buf) != 0) @@ -2157,7 +2611,7 @@ static bool StatFileFd(char const * const msg, int const iFd, std::string const // we set it here, too, as we get the info here for free // in theory the Open-methods should take care of it already if (d != NULL) - d->is_pipe = true; + d->set_is_pipe(true); if (stat(FileName.c_str(), &Buf) != 0) return _error->Errno("fstat", "Unable to determine %s for file %s", msg, FileName.c_str()); } @@ -2192,7 +2646,7 @@ time_t FileFd::ModificationTime() unsigned long long FileFd::Size() { if (d == nullptr) - return false; + return 0; return d->InternalSize(); } /*}}}*/ @@ -2201,6 +2655,8 @@ unsigned long long FileFd::Size() /* */ bool FileFd::Close() { + if (Failed() == false && Flush() == false) + return false; if (iFd == -1) return true; @@ -2219,7 +2675,7 @@ bool FileFd::Close() } if ((Flags & Replace) == Replace) { - if (rename(TemporaryFileName.c_str(), FileName.c_str()) != 0) + if (Failed() == false && rename(TemporaryFileName.c_str(), FileName.c_str()) != 0) Res &= _error->Errno("rename",_("Problem renaming the file %s to %s"), TemporaryFileName.c_str(), FileName.c_str()); FileName = TemporaryFileName; // for the unlink() below. @@ -2254,13 +2710,12 @@ bool FileFd::FileFdErrno(const char *Function, const char *Description,...) va_list args; size_t msgSize = 400; int const errsv = errno; - while (true) - { + bool retry; + do { va_start(args,Description); - if (_error->InsertErrno(GlobalError::ERROR, Function, Description, args, errsv, msgSize) == false) - break; + retry = _error->InsertErrno(GlobalError::ERROR, Function, Description, args, errsv, msgSize); va_end(args); - } + } while (retry); return false; } /*}}}*/ @@ -2269,13 +2724,12 @@ bool FileFd::FileFdError(const char *Description,...) { Flags |= Fail; va_list args; size_t msgSize = 400; - while (true) - { + bool retry; + do { va_start(args,Description); - if (_error->Insert(GlobalError::ERROR, Description, args, msgSize) == false) - break; + retry = _error->Insert(GlobalError::ERROR, Description, args, msgSize); va_end(args); - } + } while (retry); return false; } /*}}}*/ @@ -2318,9 +2772,9 @@ std::vector Glob(std::string const &pattern, int flags) return result; } /*}}}*/ -std::string GetTempDir() /*{{{*/ +static std::string APT_NONNULL(1) GetTempDirEnv(char const * const env) /*{{{*/ { - const char *tmpdir = getenv("TMPDIR"); + const char *tmpdir = getenv(env); #ifdef P_tmpdir if (!tmpdir) @@ -2336,6 +2790,11 @@ std::string GetTempDir() /*{{{*/ tmpdir = "/tmp"; return string(tmpdir); +} + /*}}}*/ +std::string GetTempDir() /*{{{*/ +{ + return GetTempDirEnv("TMPDIR"); } std::string GetTempDir(std::string const &User) { @@ -2400,6 +2859,11 @@ bool Rename(std::string From, std::string To) /*{{{*/ } /*}}}*/ bool Popen(const char* Args[], FileFd &Fd, pid_t &Child, FileFd::OpenMode Mode)/*{{{*/ +{ + return Popen(Args, Fd, Child, Mode, true); +} + /*}}}*/ +bool Popen(const char* Args[], FileFd &Fd, pid_t &Child, FileFd::OpenMode Mode, bool CaptureStderr)/*{{{*/ { int fd; if (Mode != FileFd::ReadOnly && Mode != FileFd::WriteOnly) @@ -2431,7 +2895,8 @@ bool Popen(const char* Args[], FileFd &Fd, pid_t &Child, FileFd::OpenMode Mode)/ if(Mode == FileFd::ReadOnly) { dup2(fd, 1); - dup2(fd, 2); + if (CaptureStderr == true) + dup2(fd, 2); } else if(Mode == FileFd::WriteOnly) dup2(fd, 0); @@ -2585,6 +3050,32 @@ bool DropPrivileges() /*{{{*/ return _error->Error("Could restore a uid to root, privilege dropping did not work"); } + if (_config->FindB("APT::Sandbox::ResetEnvironment", true)) + { + setenv("HOME", pw->pw_dir, 1); + setenv("USER", pw->pw_name, 1); + setenv("USERNAME", pw->pw_name, 1); + setenv("LOGNAME", pw->pw_name, 1); + auto const shell = flNotDir(pw->pw_shell); + if (shell == "false" || shell == "nologin") + setenv("SHELL", "/bin/sh", 1); + else + setenv("SHELL", pw->pw_shell, 1); + auto const apt_setenv_tmp = [](char const * const env) { + auto const tmpdir = getenv(env); + if (tmpdir != nullptr) + { + auto const ourtmpdir = GetTempDirEnv(env); + if (ourtmpdir != tmpdir) + setenv(env, ourtmpdir.c_str(), 1); + } + }; + apt_setenv_tmp("TMPDIR"); + apt_setenv_tmp("TEMPDIR"); + apt_setenv_tmp("TMP"); + apt_setenv_tmp("TEMP"); + } + return true; } /*}}}*/