#ifdef HAVE_LZMA
#include <lzma.h>
#endif
+#ifdef HAVE_LZ4
+ #include <lz4frame.h>
+#endif
#include <endian.h>
#include <stdint.h>
using namespace std;
+/* Should be a multiple of the common page size (4096) */
+static constexpr unsigned long long APT_BUFFER_SIZE = 64 * 1024;
+
// RunScripts - Run a set of scripts from a configuration subtree /*{{{*/
// ---------------------------------------------------------------------
/* */
return false;
// Buffered copy between fds
- constexpr size_t BufSize = 64000;
+ constexpr size_t BufSize = APT_BUFFER_SIZE;
std::unique_ptr<unsigned char[]> Buf(new unsigned char[BufSize]);
unsigned long long ToRead = 0;
do {
return AbsPath;
}
/*}}}*/
+std::string flNormalize(std::string file) /*{{{*/
+{
+ if (file.empty())
+ return file;
+ // do some normalisation by removing // and /./ from the path
+ size_t found = string::npos;
+ while ((found = file.find("/./")) != string::npos)
+ file.replace(found, 3, "/");
+ while ((found = file.find("//")) != string::npos)
+ file.replace(found, 2, "/");
+
+ if (APT::String::Startswith(file, "/dev/null"))
+ {
+ file.erase(strlen("/dev/null"));
+ return file;
+ }
+ return file;
+}
+ /*}}}*/
// SetCloseExec - Set the close on exec flag /*{{{*/
// ---------------------------------------------------------------------
/* */
// ensure the file is owned by root and has good permissions
struct passwd const * const pw = getpwnam(user);
struct group const * const gr = getgrnam(group);
- if (pw != NULL && gr != NULL && chown(file, pw->pw_uid, gr->gr_gid) != 0)
+ if (pw != NULL && gr != NULL && lchown(file, pw->pw_uid, gr->gr_gid) != 0)
Res &= _error->WarningE(requester, "chown to %s:%s of file %s failed", user, group, file);
}
+ struct stat Buf;
+ if (lstat(file, &Buf) != 0 || S_ISLNK(Buf.st_mode))
+ return Res;
if (chmod(file, mode) != 0)
Res &= _error->WarningE(requester, "chmod 0%o of file %s failed", mode, file);
return Res;
}
/*}}}*/
-class FileFdPrivate { /*{{{*/
+struct APT_HIDDEN simple_buffer { /*{{{*/
+ size_t buffersize_max = 0;
+ unsigned long long bufferstart = 0;
+ unsigned long long bufferend = 0;
+ char *buffer = nullptr;
+
+ simple_buffer() {
+ reset(4096);
+ }
+ ~simple_buffer() {
+ delete[] buffer;
+ }
+
+ const char *get() const { return buffer + bufferstart; }
+ char *get() { return buffer + bufferstart; }
+ const char *getend() const { return buffer + bufferend; }
+ char *getend() { return buffer + bufferend; }
+ bool empty() const { return bufferend <= bufferstart; }
+ bool full() const { return bufferend == buffersize_max; }
+ unsigned long long free() const { return buffersize_max - bufferend; }
+ unsigned long long size() const { return bufferend-bufferstart; }
+ void reset(size_t size)
+ {
+ if (size > buffersize_max) {
+ delete[] buffer;
+ buffersize_max = size;
+ buffer = new char[size];
+ }
+ reset();
+ }
+ void reset() { bufferend = bufferstart = 0; }
+ ssize_t read(void *to, unsigned long long requested_size) APT_MUSTCHECK
+ {
+ if (size() < requested_size)
+ requested_size = size();
+ memcpy(to, buffer + bufferstart, requested_size);
+ bufferstart += requested_size;
+ if (bufferstart == bufferend)
+ bufferstart = bufferend = 0;
+ return requested_size;
+ }
+ ssize_t write(const void *from, unsigned long long requested_size) APT_MUSTCHECK
+ {
+ if (free() < requested_size)
+ requested_size = free();
+ memcpy(getend(), from, requested_size);
+ bufferend += requested_size;
+ if (bufferstart == bufferend)
+ bufferstart = bufferend = 0;
+ return requested_size;
+ }
+};
+ /*}}}*/
+
+class APT_HIDDEN FileFdPrivate { /*{{{*/
+ friend class BufferedWriteFileFdPrivate;
protected:
FileFd * const filefd;
-public:
+ simple_buffer buffer;
int compressed_fd;
pid_t compressor_pid;
bool is_pipe;
APT::Configuration::Compressor compressor;
unsigned int openmode;
unsigned long long seekpos;
- FileFdPrivate(FileFd * const pfilefd) : filefd(pfilefd),
+public:
+
+ explicit FileFdPrivate(FileFd * const pfilefd) : filefd(pfilefd),
compressed_fd(-1), compressor_pid(-1), is_pipe(false),
openmode(0), seekpos(0) {};
+ virtual APT::Configuration::Compressor get_compressor() const
+ {
+ return compressor;
+ }
+ virtual void set_compressor(APT::Configuration::Compressor const &compressor)
+ {
+ this->compressor = compressor;
+ }
+ virtual unsigned int get_openmode() const
+ {
+ return openmode;
+ }
+ virtual void set_openmode(unsigned int openmode)
+ {
+ this->openmode = openmode;
+ }
+ virtual bool get_is_pipe() const
+ {
+ return is_pipe;
+ }
+ virtual void set_is_pipe(bool is_pipe)
+ {
+ this->is_pipe = is_pipe;
+ }
+ virtual unsigned long long get_seekpos() const
+ {
+ return seekpos;
+ }
+ virtual void set_seekpos(unsigned long long seekpos)
+ {
+ this->seekpos = seekpos;
+ }
virtual bool InternalOpen(int const iFd, unsigned int const Mode) = 0;
- virtual ssize_t InternalRead(void * const To, unsigned long long const Size) = 0;
- virtual bool InternalReadError() { return filefd->FileFdErrno("read",_("Read error")); }
- virtual char * InternalReadLine(char * const To, unsigned long long const Size)
+ ssize_t InternalRead(void * To, unsigned long long Size)
{
- unsigned long long read = 0;
- while ((Size - 1) != read)
+ // Drain the buffer if needed.
+ if (buffer.empty() == false)
{
- unsigned long long done = 0;
- if (filefd->Read(To + read, 1, &done) == false)
- return NULL;
- if (done == 0)
- break;
- if (To[read++] == '\n')
+ return buffer.read(To, Size);
+ }
+ return InternalUnbufferedRead(To, Size);
+ }
+ virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) = 0;
+ virtual bool InternalReadError() { return filefd->FileFdErrno("read",_("Read error")); }
+ virtual char * InternalReadLine(char * To, unsigned long long Size)
+ {
+ if (unlikely(Size == 0))
+ return nullptr;
+ // Read one byte less than buffer size to have space for trailing 0.
+ --Size;
+
+ char * const InitialTo = To;
+
+ while (Size > 0) {
+ if (buffer.empty() == true)
+ {
+ buffer.reset();
+ unsigned long long actualread = 0;
+ if (filefd->Read(buffer.getend(), buffer.free(), &actualread) == false)
+ return nullptr;
+ buffer.bufferend = actualread;
+ if (buffer.size() == 0)
+ {
+ if (To == InitialTo)
+ return nullptr;
+ break;
+ }
+ filefd->Flags &= ~FileFd::HitEof;
+ }
+
+ unsigned long long const OutputSize = std::min(Size, buffer.size());
+ char const * const newline = static_cast<char const * const>(memchr(buffer.get(), '\n', OutputSize));
+ // Read until end of line or up to Size bytes from the buffer.
+ unsigned long long actualread = buffer.read(To,
+ (newline != nullptr)
+ ? (newline - buffer.get()) + 1
+ : OutputSize);
+ To += actualread;
+ Size -= actualread;
+ if (newline != nullptr)
break;
}
- if (read == 0)
- return NULL;
- To[read] = '\0';
- return To;
+ *To = '\0';
+ return InitialTo;
+ }
+ virtual bool InternalFlush()
+ {
+ return true;
}
virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) = 0;
virtual bool InternalWriteError() { return filefd->FileFdErrno("write",_("Write error")); }
if (filefd->OpenInternDescriptor(openmode, compressor) == false)
return filefd->FileFdError("Seek on file %s because it couldn't be reopened", filefd->FileName.c_str());
+ buffer.reset();
+ set_seekpos(0);
if (To != 0)
return filefd->Skip(To);
// seeking around, but not all users of FileFd use always Seek() and co
// so d->seekpos isn't always true and we can just use it as a hint if
// we have nothing else, but not always as an authority…
- return seekpos;
+ return seekpos - buffer.size();
}
virtual unsigned long long InternalSize()
{
virtual ~FileFdPrivate() {}
};
/*}}}*/
-class GzipFileFdPrivate: public FileFdPrivate { /*{{{*/
+class APT_HIDDEN BufferedWriteFileFdPrivate : public FileFdPrivate { /*{{{*/
+protected:
+ FileFdPrivate *wrapped;
+ simple_buffer writebuffer;
+
+public:
+
+ explicit BufferedWriteFileFdPrivate(FileFdPrivate *Priv) :
+ FileFdPrivate(Priv->filefd), wrapped(Priv) {};
+
+ virtual APT::Configuration::Compressor get_compressor() const APT_OVERRIDE
+ {
+ return wrapped->get_compressor();
+ }
+ virtual void set_compressor(APT::Configuration::Compressor const &compressor) APT_OVERRIDE
+ {
+ return wrapped->set_compressor(compressor);
+ }
+ virtual unsigned int get_openmode() const APT_OVERRIDE
+ {
+ return wrapped->get_openmode();
+ }
+ virtual void set_openmode(unsigned int openmode) APT_OVERRIDE
+ {
+ return wrapped->set_openmode(openmode);
+ }
+ virtual bool get_is_pipe() const APT_OVERRIDE
+ {
+ return wrapped->get_is_pipe();
+ }
+ virtual void set_is_pipe(bool is_pipe) APT_OVERRIDE
+ {
+ FileFdPrivate::set_is_pipe(is_pipe);
+ wrapped->set_is_pipe(is_pipe);
+ }
+ virtual unsigned long long get_seekpos() const APT_OVERRIDE
+ {
+ return wrapped->get_seekpos();
+ }
+ virtual void set_seekpos(unsigned long long seekpos) APT_OVERRIDE
+ {
+ return wrapped->set_seekpos(seekpos);
+ }
+ virtual bool InternalOpen(int const iFd, unsigned int const Mode) APT_OVERRIDE
+ {
+ if (InternalFlush() == false)
+ return false;
+ return wrapped->InternalOpen(iFd, Mode);
+ }
+ virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) APT_OVERRIDE
+ {
+ if (InternalFlush() == false)
+ return -1;
+ return wrapped->InternalUnbufferedRead(To, Size);
+
+ }
+ virtual bool InternalReadError() APT_OVERRIDE
+ {
+ return wrapped->InternalReadError();
+ }
+ virtual char * InternalReadLine(char * To, unsigned long long Size) APT_OVERRIDE
+ {
+ if (InternalFlush() == false)
+ return nullptr;
+ return wrapped->InternalReadLine(To, Size);
+ }
+ virtual bool InternalFlush() APT_OVERRIDE
+ {
+ while (writebuffer.empty() == false) {
+ auto written = wrapped->InternalWrite(writebuffer.get(),
+ writebuffer.size());
+ // Ignore interrupted syscalls
+ if (written < 0 && errno == EINTR)
+ continue;
+ if (written < 0)
+ return wrapped->InternalWriteError();
+
+ writebuffer.bufferstart += written;
+ }
+ writebuffer.reset();
+ return wrapped->InternalFlush();
+ }
+ virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) APT_OVERRIDE
+ {
+ // Optimisation: If the buffer is empty and we have more to write than
+ // would fit in the buffer (or equal number of bytes), write directly.
+ if (writebuffer.empty() == true && Size >= writebuffer.free())
+ return wrapped->InternalWrite(From, Size);
+
+ // Write as much into the buffer as possible and then flush if needed
+ auto written = writebuffer.write(From, Size);
+
+ if (writebuffer.full() && InternalFlush() == false)
+ return -1;
+
+ return written;
+ }
+ virtual bool InternalWriteError() APT_OVERRIDE
+ {
+ return wrapped->InternalWriteError();
+ }
+ virtual bool InternalSeek(unsigned long long const To) APT_OVERRIDE
+ {
+ if (InternalFlush() == false)
+ return false;
+ return wrapped->InternalSeek(To);
+ }
+ virtual bool InternalSkip(unsigned long long Over) APT_OVERRIDE
+ {
+ if (InternalFlush() == false)
+ return false;
+ return wrapped->InternalSkip(Over);
+ }
+ virtual bool InternalTruncate(unsigned long long const Size) APT_OVERRIDE
+ {
+ if (InternalFlush() == false)
+ return false;
+ return wrapped->InternalTruncate(Size);
+ }
+ virtual unsigned long long InternalTell() APT_OVERRIDE
+ {
+ if (InternalFlush() == false)
+ return -1;
+ return wrapped->InternalTell();
+ }
+ virtual unsigned long long InternalSize() APT_OVERRIDE
+ {
+ if (InternalFlush() == false)
+ return -1;
+ return wrapped->InternalSize();
+ }
+ virtual bool InternalClose(std::string const &FileName) APT_OVERRIDE
+ {
+ return wrapped->InternalClose(FileName);
+ }
+ virtual bool InternalAlwaysAutoClose() const APT_OVERRIDE
+ {
+ return wrapped->InternalAlwaysAutoClose();
+ }
+ virtual ~BufferedWriteFileFdPrivate()
+ {
+ delete wrapped;
+ }
+};
+ /*}}}*/
+class APT_HIDDEN GzipFileFdPrivate: public FileFdPrivate { /*{{{*/
#ifdef HAVE_ZLIB
public:
gzFile gz;
- virtual bool InternalOpen(int const iFd, unsigned int const Mode) override
+ virtual bool InternalOpen(int const iFd, unsigned int const Mode) APT_OVERRIDE
{
if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite)
gz = gzdopen(iFd, "r+");
filefd->Flags |= FileFd::Compressed;
return gz != nullptr;
}
- virtual ssize_t InternalRead(void * const To, unsigned long long const Size) override
+ virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) APT_OVERRIDE
{
return gzread(gz, To, Size);
}
- virtual bool InternalReadError() override
+ virtual bool InternalReadError() APT_OVERRIDE
{
int err;
char const * const errmsg = gzerror(gz, &err);
return filefd->FileFdError("gzread: %s (%d: %s)", _("Read error"), err, errmsg);
return FileFdPrivate::InternalReadError();
}
- virtual char * InternalReadLine(char * const To, unsigned long long const Size) override
+ virtual char * InternalReadLine(char * To, unsigned long long Size) APT_OVERRIDE
{
return gzgets(gz, To, Size);
}
- virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override
+ virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) APT_OVERRIDE
{
return gzwrite(gz,From,Size);
}
- virtual bool InternalWriteError() override
+ virtual bool InternalWriteError() APT_OVERRIDE
{
int err;
char const * const errmsg = gzerror(gz, &err);
return filefd->FileFdError("gzwrite: %s (%d: %s)", _("Write error"), err, errmsg);
return FileFdPrivate::InternalWriteError();
}
- virtual bool InternalSeek(unsigned long long const To) override
+ virtual bool InternalSeek(unsigned long long const To) APT_OVERRIDE
{
off_t const res = gzseek(gz, To, SEEK_SET);
if (res != (off_t)To)
return filefd->FileFdError("Unable to seek to %llu", To);
-
seekpos = To;
+ buffer.reset();
return true;
}
- virtual bool InternalSkip(unsigned long long Over) override
+ virtual bool InternalSkip(unsigned long long Over) APT_OVERRIDE
{
+ if (Over >= buffer.size())
+ {
+ Over -= buffer.size();
+ buffer.reset();
+ }
+ else
+ {
+ buffer.bufferstart += Over;
+ return true;
+ }
+ if (Over == 0)
+ return true;
off_t const res = gzseek(gz, Over, SEEK_CUR);
if (res < 0)
return filefd->FileFdError("Unable to seek ahead %llu",Over);
seekpos = res;
return true;
}
- virtual unsigned long long InternalTell() override
+ virtual unsigned long long InternalTell() APT_OVERRIDE
{
- return gztell(gz);
+ return gztell(gz) - buffer.size();
}
- virtual unsigned long long InternalSize() override
+ virtual unsigned long long InternalSize() APT_OVERRIDE
{
unsigned long long filesize = FileFdPrivate::InternalSize();
// only check gzsize if we are actually a gzip file, just checking for
}
return size;
}
- virtual bool InternalClose(std::string const &FileName) override
+ virtual bool InternalClose(std::string const &FileName) APT_OVERRIDE
{
if (gz == nullptr)
return true;
return true;
}
- GzipFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), gz(nullptr) {}
+ explicit GzipFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), gz(nullptr) {}
virtual ~GzipFileFdPrivate() { InternalClose(""); }
#endif
};
/*}}}*/
-class Bz2FileFdPrivate: public FileFdPrivate { /*{{{*/
+class APT_HIDDEN Bz2FileFdPrivate: public FileFdPrivate { /*{{{*/
#ifdef HAVE_BZ2
BZFILE* bz2;
public:
- virtual bool InternalOpen(int const iFd, unsigned int const Mode) override
+ virtual bool InternalOpen(int const iFd, unsigned int const Mode) APT_OVERRIDE
{
if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite)
bz2 = BZ2_bzdopen(iFd, "r+");
filefd->Flags |= FileFd::Compressed;
return bz2 != nullptr;
}
- virtual ssize_t InternalRead(void * const To, unsigned long long const Size) override
+ virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) APT_OVERRIDE
{
return BZ2_bzread(bz2, To, Size);
}
- virtual bool InternalReadError() override
+ virtual bool InternalReadError() APT_OVERRIDE
{
int err;
char const * const errmsg = BZ2_bzerror(bz2, &err);
return filefd->FileFdError("BZ2_bzread: %s %s (%d: %s)", filefd->FileName.c_str(), _("Read error"), err, errmsg);
return FileFdPrivate::InternalReadError();
}
- virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override
+ virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) APT_OVERRIDE
{
return BZ2_bzwrite(bz2, (void*)From, Size);
}
- virtual bool InternalWriteError() override
+ virtual bool InternalWriteError() APT_OVERRIDE
{
int err;
char const * const errmsg = BZ2_bzerror(bz2, &err);
return filefd->FileFdError("BZ2_bzwrite: %s %s (%d: %s)", filefd->FileName.c_str(), _("Write error"), err, errmsg);
return FileFdPrivate::InternalWriteError();
}
- virtual bool InternalStream() const override { return true; }
- virtual bool InternalClose(std::string const &) override
+ virtual bool InternalStream() const APT_OVERRIDE { return true; }
+ virtual bool InternalClose(std::string const &) APT_OVERRIDE
{
if (bz2 == nullptr)
return true;
return true;
}
- Bz2FileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), bz2(nullptr) {}
+ explicit Bz2FileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), bz2(nullptr) {}
virtual ~Bz2FileFdPrivate() { InternalClose(""); }
#endif
};
/*}}}*/
-class LzmaFileFdPrivate: public FileFdPrivate { /*{{{*/
+class APT_HIDDEN Lz4FileFdPrivate: public FileFdPrivate { /*{{{*/
+ static constexpr unsigned long long LZ4_HEADER_SIZE = 19;
+ static constexpr unsigned long long LZ4_FOOTER_SIZE = 4;
+#ifdef HAVE_LZ4
+ LZ4F_decompressionContext_t dctx;
+ LZ4F_compressionContext_t cctx;
+ LZ4F_errorCode_t res;
+ FileFd backend;
+ simple_buffer lz4_buffer;
+ // Count of bytes that the decompressor expects to read next, or buffer size.
+ size_t next_to_load = APT_BUFFER_SIZE;
+public:
+ virtual bool InternalOpen(int const iFd, unsigned int const Mode) APT_OVERRIDE
+ {
+ if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite)
+ return _error->Error("lz4 only supports write or read mode");
+
+ if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly) {
+ res = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION);
+ lz4_buffer.reset(LZ4F_compressBound(APT_BUFFER_SIZE, nullptr)
+ + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE);
+ } else {
+ res = LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION);
+ lz4_buffer.reset(APT_BUFFER_SIZE);
+ }
+
+ filefd->Flags |= FileFd::Compressed;
+
+ if (LZ4F_isError(res))
+ return false;
+
+ unsigned int flags = (Mode & (FileFd::WriteOnly|FileFd::ReadOnly));
+ if (backend.OpenDescriptor(iFd, flags, FileFd::None, true) == false)
+ return false;
+
+ // Write the file header
+ if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly)
+ {
+ res = LZ4F_compressBegin(cctx, lz4_buffer.buffer, lz4_buffer.buffersize_max, nullptr);
+ if (LZ4F_isError(res) || backend.Write(lz4_buffer.buffer, res) == false)
+ return false;
+ }
+
+ return true;
+ }
+ virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) APT_OVERRIDE
+ {
+ /* Keep reading as long as the compressor still wants to read */
+ while (next_to_load) {
+ // Fill compressed buffer;
+ if (lz4_buffer.empty()) {
+ unsigned long long read;
+ /* Reset - if LZ4 decompressor wants to read more, allocate more */
+ lz4_buffer.reset(next_to_load);
+ if (backend.Read(lz4_buffer.getend(), lz4_buffer.free(), &read) == false)
+ return -1;
+ lz4_buffer.bufferend += read;
+
+ /* Expected EOF */
+ if (read == 0) {
+ res = -1;
+ return filefd->FileFdError("LZ4F: %s %s",
+ filefd->FileName.c_str(),
+ _("Unexpected end of file")), -1;
+ }
+ }
+ // Drain compressed buffer as far as possible.
+ size_t in = lz4_buffer.size();
+ size_t out = Size;
+
+ res = LZ4F_decompress(dctx, To, &out, lz4_buffer.get(), &in, nullptr);
+ if (LZ4F_isError(res))
+ return -1;
+
+ next_to_load = res;
+ lz4_buffer.bufferstart += in;
+
+ if (out != 0)
+ return out;
+ }
+
+ return 0;
+ }
+ virtual bool InternalReadError() APT_OVERRIDE
+ {
+ char const * const errmsg = LZ4F_getErrorName(res);
+
+ return filefd->FileFdError("LZ4F: %s %s (%zu: %s)", filefd->FileName.c_str(), _("Read error"), res, errmsg);
+ }
+ virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) APT_OVERRIDE
+ {
+ unsigned long long const towrite = std::min(APT_BUFFER_SIZE, Size);
+
+ res = LZ4F_compressUpdate(cctx,
+ lz4_buffer.buffer, lz4_buffer.buffersize_max,
+ From, towrite, nullptr);
+
+ if (LZ4F_isError(res) || backend.Write(lz4_buffer.buffer, res) == false)
+ return -1;
+
+ return towrite;
+ }
+ virtual bool InternalWriteError() APT_OVERRIDE
+ {
+ char const * const errmsg = LZ4F_getErrorName(res);
+
+ return filefd->FileFdError("LZ4F: %s %s (%zu: %s)", filefd->FileName.c_str(), _("Write error"), res, errmsg);
+ }
+ virtual bool InternalStream() const APT_OVERRIDE { return true; }
+
+ virtual bool InternalFlush() APT_OVERRIDE
+ {
+ return backend.Flush();
+ }
+
+ virtual bool InternalClose(std::string const &) APT_OVERRIDE
+ {
+ /* Reset variables */
+ res = 0;
+ next_to_load = APT_BUFFER_SIZE;
+
+ if (cctx != nullptr)
+ {
+ if (filefd->Failed() == false)
+ {
+ res = LZ4F_compressEnd(cctx, lz4_buffer.buffer, lz4_buffer.buffersize_max, nullptr);
+ if (LZ4F_isError(res) || backend.Write(lz4_buffer.buffer, res) == false)
+ return false;
+ if (!backend.Flush())
+ return false;
+ }
+ if (!backend.Close())
+ return false;
+
+ res = LZ4F_freeCompressionContext(cctx);
+ cctx = nullptr;
+ }
+
+ if (dctx != nullptr)
+ {
+ res = LZ4F_freeDecompressionContext(dctx);
+ dctx = nullptr;
+ }
+ if (backend.IsOpen())
+ {
+ backend.Close();
+ filefd->iFd = -1;
+ }
+
+ return LZ4F_isError(res) == false;
+ }
+
+ explicit Lz4FileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), dctx(nullptr), cctx(nullptr) {}
+ virtual ~Lz4FileFdPrivate() {
+ InternalClose("");
+ }
+#endif
+};
+ /*}}}*/
+class APT_HIDDEN LzmaFileFdPrivate: public FileFdPrivate { /*{{{*/
#ifdef HAVE_LZMA
struct LZMAFILE {
FILE* file;
+ FileFd * const filefd;
uint8_t buffer[4096];
lzma_stream stream;
lzma_ret err;
bool eof;
bool compressing;
- LZMAFILE() : file(nullptr), eof(false), compressing(false) { buffer[0] = '\0'; }
+ LZMAFILE(FileFd * const fd) : file(nullptr), filefd(fd), eof(false), compressing(false) { buffer[0] = '\0'; }
~LZMAFILE()
{
- if (compressing == true)
+ if (compressing == true && filefd->Failed() == false)
{
size_t constexpr buffersize = sizeof(buffer)/sizeof(buffer[0]);
while(true)
}
};
LZMAFILE* lzma;
+ static uint32_t findXZlevel(std::vector<std::string> const &Args)
+ {
+ for (auto a = Args.rbegin(); a != Args.rend(); ++a)
+ if (a->empty() == false && (*a)[0] == '-' && (*a)[1] != '-')
+ {
+ auto const number = a->find_last_of("0123456789");
+ if (number == std::string::npos)
+ continue;
+ auto const extreme = a->find("e", number);
+ uint32_t level = (extreme != std::string::npos) ? LZMA_PRESET_EXTREME : 0;
+ switch ((*a)[number])
+ {
+ case '0': return level | 0;
+ case '1': return level | 1;
+ case '2': return level | 2;
+ case '3': return level | 3;
+ case '4': return level | 4;
+ case '5': return level | 5;
+ case '6': return level | 6;
+ case '7': return level | 7;
+ case '8': return level | 8;
+ case '9': return level | 9;
+ }
+ }
+ return 6;
+ }
public:
- virtual bool InternalOpen(int const iFd, unsigned int const Mode) override
+ virtual bool InternalOpen(int const iFd, unsigned int const Mode) APT_OVERRIDE
{
if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite)
return filefd->FileFdError("ReadWrite mode is not supported for lzma/xz files %s", filefd->FileName.c_str());
if (lzma == nullptr)
- lzma = new LzmaFileFdPrivate::LZMAFILE;
+ lzma = new LzmaFileFdPrivate::LZMAFILE(filefd);
if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly)
lzma->file = fdopen(iFd, "w");
else
if (lzma->file == nullptr)
return false;
- uint32_t const xzlevel = 6;
- uint64_t const memlimit = UINT64_MAX;
lzma_stream tmp_stream = LZMA_STREAM_INIT;
lzma->stream = tmp_stream;
if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly)
{
+ uint32_t const xzlevel = findXZlevel(compressor.CompressArgs);
if (compressor.Name == "xz")
{
if (lzma_easy_encoder(&lzma->stream, xzlevel, LZMA_CHECK_CRC64) != LZMA_OK)
}
else
{
+ uint64_t const memlimit = UINT64_MAX;
if (compressor.Name == "xz")
{
if (lzma_auto_decoder(&lzma->stream, memlimit, 0) != LZMA_OK)
}
return true;
}
- virtual ssize_t InternalRead(void * const To, unsigned long long const Size) override
+ virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) APT_OVERRIDE
{
ssize_t Res;
if (lzma->eof == true)
}
return Res;
}
- virtual bool InternalReadError() override
+ virtual bool InternalReadError() APT_OVERRIDE
{
return filefd->FileFdError("lzma_read: %s (%d)", _("Read error"), lzma->err);
}
- virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override
+ virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) APT_OVERRIDE
{
+ ssize_t Res;
lzma->stream.next_in = (uint8_t *)From;
lzma->stream.avail_in = Size;
lzma->stream.next_out = lzma->buffer;
size_t const n = sizeof(lzma->buffer)/sizeof(lzma->buffer[0]) - lzma->stream.avail_out;
size_t const m = (n == 0) ? 0 : fwrite(lzma->buffer, 1, n, lzma->file);
if (m != n)
- return -1;
+ {
+ Res = -1;
+ errno = 0;
+ }
else
- return Size - lzma->stream.avail_in;
+ {
+ Res = Size - lzma->stream.avail_in;
+ if (Res == 0)
+ {
+ // lzma run was okay, but produced no output…
+ Res = -1;
+ errno = EINTR;
+ }
+ }
+ return Res;
}
- virtual bool InternalWriteError() override
+ virtual bool InternalWriteError() APT_OVERRIDE
{
return filefd->FileFdError("lzma_write: %s (%d)", _("Write error"), lzma->err);
}
- virtual bool InternalStream() const override { return true; }
- virtual bool InternalClose(std::string const &) override
+ virtual bool InternalStream() const APT_OVERRIDE { return true; }
+ virtual bool InternalClose(std::string const &) APT_OVERRIDE
{
delete lzma;
lzma = nullptr;
return true;
}
- LzmaFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), lzma(nullptr) {}
+ explicit LzmaFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), lzma(nullptr) {}
virtual ~LzmaFileFdPrivate() { InternalClose(""); }
#endif
};
/*}}}*/
-class PipedFileFdPrivate: public FileFdPrivate /*{{{*/
+class APT_HIDDEN PipedFileFdPrivate: public FileFdPrivate /*{{{*/
/* if we don't have a specific class dealing with library calls, we (un)compress
by executing a specified binary and pipe in/out what we need */
{
public:
- virtual bool InternalOpen(int const, unsigned int const Mode) override
+ virtual bool InternalOpen(int const, unsigned int const Mode) APT_OVERRIDE
{
// collect zombies here in case we reopen
if (compressor_pid > 0)
if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite)
return filefd->FileFdError("ReadWrite mode is not supported for file %s", filefd->FileName.c_str());
+ if (compressor.Binary == "false")
+ return filefd->FileFdError("libapt has inbuilt support for the %s compression,"
+ " but was forced to ignore it in favor of an external binary – which isn't installed.", compressor.Name.c_str());
bool const Comp = (Mode & FileFd::WriteOnly) == FileFd::WriteOnly;
- if (Comp == false)
+ if (Comp == false && filefd->iFd != -1)
{
// Handle 'decompression' of empty files
struct stat Buf;
- fstat(filefd->iFd, &Buf);
+ if (fstat(filefd->iFd, &Buf) != 0)
+ return filefd->FileFdErrno("fstat", "Could not stat fd %d for file %s", filefd->iFd, filefd->FileName.c_str());
if (Buf.st_size == 0 && S_ISFIFO(Buf.st_mode) == false)
return true;
SetCloseExec(Pipe[J],true);
compressed_fd = filefd->iFd;
- is_pipe = true;
+ set_is_pipe(true);
if (Comp == true)
filefd->iFd = Pipe[1];
dup2(compressed_fd,STDIN_FILENO);
dup2(Pipe[1],STDOUT_FILENO);
}
- int const nullfd = open("/dev/null", O_WRONLY);
- if (nullfd != -1)
- {
- dup2(nullfd,STDERR_FILENO);
- close(nullfd);
- }
SetCloseExec(STDOUT_FILENO,false);
SetCloseExec(STDIN_FILENO,false);
return true;
}
- virtual ssize_t InternalRead(void * const To, unsigned long long const Size) override
+ virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) APT_OVERRIDE
{
return read(filefd->iFd, To, Size);
}
- virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override
+ virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) APT_OVERRIDE
{
return write(filefd->iFd, From, Size);
}
- virtual bool InternalClose(std::string const &) override
+ virtual bool InternalClose(std::string const &) APT_OVERRIDE
{
bool Ret = true;
+ if (filefd->iFd != -1)
+ {
+ close(filefd->iFd);
+ filefd->iFd = -1;
+ }
if (compressor_pid > 0)
Ret &= ExecWait(compressor_pid, "FileFdCompressor", true);
compressor_pid = -1;
return Ret;
}
- PipedFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd) {}
+ explicit PipedFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd) {}
virtual ~PipedFileFdPrivate() { InternalClose(""); }
};
/*}}}*/
-class DirectFileFdPrivate: public FileFdPrivate /*{{{*/
+class APT_HIDDEN DirectFileFdPrivate: public FileFdPrivate /*{{{*/
{
public:
- virtual bool InternalOpen(int const, unsigned int const) override { return true; }
- virtual ssize_t InternalRead(void * const To, unsigned long long const Size) override
+ virtual bool InternalOpen(int const, unsigned int const) APT_OVERRIDE { return true; }
+ virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) APT_OVERRIDE
{
return read(filefd->iFd, To, Size);
}
-
- virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override
+ virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) APT_OVERRIDE
{
+ // files opened read+write are strange and only really "supported" for direct files
+ if (buffer.size() != 0)
+ {
+ lseek(filefd->iFd, -buffer.size(), SEEK_CUR);
+ buffer.reset();
+ }
return write(filefd->iFd, From, Size);
}
- virtual bool InternalSeek(unsigned long long const To) override
+ virtual bool InternalSeek(unsigned long long const To) APT_OVERRIDE
{
off_t const res = lseek(filefd->iFd, To, SEEK_SET);
if (res != (off_t)To)
return filefd->FileFdError("Unable to seek to %llu", To);
seekpos = To;
+ buffer.reset();
return true;
}
- virtual bool InternalSkip(unsigned long long Over) override
+ virtual bool InternalSkip(unsigned long long Over) APT_OVERRIDE
{
+ if (Over >= buffer.size())
+ {
+ Over -= buffer.size();
+ buffer.reset();
+ }
+ else
+ {
+ buffer.bufferstart += Over;
+ return true;
+ }
+ if (Over == 0)
+ return true;
off_t const res = lseek(filefd->iFd, Over, SEEK_CUR);
if (res < 0)
return filefd->FileFdError("Unable to seek ahead %llu",Over);
seekpos = res;
return true;
}
- virtual bool InternalTruncate(unsigned long long const To) override
+ virtual bool InternalTruncate(unsigned long long const To) APT_OVERRIDE
{
+ if (buffer.size() != 0)
+ {
+ unsigned long long const seekpos = lseek(filefd->iFd, 0, SEEK_CUR);
+ if ((seekpos - buffer.size()) >= To)
+ buffer.reset();
+ else if (seekpos >= To)
+ buffer.bufferend = (To - seekpos) + buffer.bufferstart;
+ else
+ buffer.reset();
+ }
if (ftruncate(filefd->iFd, To) != 0)
return filefd->FileFdError("Unable to truncate to %llu",To);
return true;
}
- virtual unsigned long long InternalTell() override
+ virtual unsigned long long InternalTell() APT_OVERRIDE
{
- return lseek(filefd->iFd,0,SEEK_CUR);
+ return lseek(filefd->iFd,0,SEEK_CUR) - buffer.size();
}
- virtual unsigned long long InternalSize() override
+ virtual unsigned long long InternalSize() APT_OVERRIDE
{
return filefd->FileSize();
}
- virtual bool InternalClose(std::string const &) override { return true; }
- virtual bool InternalAlwaysAutoClose() const override { return false; }
+ virtual bool InternalClose(std::string const &) APT_OVERRIDE { return true; }
+ virtual bool InternalAlwaysAutoClose() const APT_OVERRIDE { return false; }
- DirectFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd) {}
+ explicit DirectFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd) {}
virtual ~DirectFileFdPrivate() { InternalClose(""); }
};
/*}}}*/
case Bzip2: name = "bzip2"; break;
case Lzma: name = "lzma"; break;
case Xz: name = "xz"; break;
+ case Lz4: name = "lz4"; break;
case Auto:
case Extension:
// Unreachable
case Bzip2: name = "bzip2"; break;
case Lzma: name = "lzma"; break;
case Xz: name = "xz"; break;
+ case Lz4: name = "lz4"; break;
case Auto:
case Extension:
if (AutoClose == true && Fd != -1)
APT_COMPRESS_INIT("xz", LzmaFileFdPrivate);
APT_COMPRESS_INIT("lzma", LzmaFileFdPrivate);
#endif
+#ifdef HAVE_LZ4
+ APT_COMPRESS_INIT("lz4", Lz4FileFdPrivate);
+#endif
#undef APT_COMPRESS_INIT
else if (compressor.Name == "." || compressor.Binary.empty() == true)
d = new DirectFileFdPrivate(this);
else
d = new PipedFileFdPrivate(this);
- d->openmode = Mode;
- d->compressor = compressor;
+ if (Mode & BufferedWrite)
+ d = new BufferedWriteFileFdPrivate(d);
+
+ d->set_openmode(Mode);
+ d->set_compressor(compressor);
if ((Flags & AutoClose) != AutoClose && d->InternalAlwaysAutoClose())
{
// Need to duplicate fd here or gz/bz2 close for cleanup will close the fd as well
gracefully. */
bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual)
{
- if (d == nullptr)
+ if (d == nullptr || Failed())
return false;
ssize_t Res = 1;
errno = 0;
To = (char *)To + Res;
Size -= Res;
if (d != NULL)
- d->seekpos += Res;
+ d->set_seekpos(d->get_seekpos() + Res);
if (Actual != 0)
*Actual += Res;
}
}
return FileFdError(_("read, still have %llu to read but none left"), Size);
+}
+bool FileFd::Read(int const Fd, void *To, unsigned long long Size, unsigned long long * const Actual)
+{
+ ssize_t Res = 1;
+ errno = 0;
+ if (Actual != nullptr)
+ *Actual = 0;
+ *static_cast<char *>(To) = '\0';
+ while (Res > 0 && Size > 0)
+ {
+ Res = read(Fd, To, Size);
+ if (Res < 0)
+ {
+ if (errno == EINTR)
+ {
+ Res = 1;
+ errno = 0;
+ continue;
+ }
+ return _error->Errno("read", _("Read error"));
+ }
+ To = static_cast<char *>(To) + Res;
+ Size -= Res;
+ if (Actual != 0)
+ *Actual += Res;
+ }
+ if (Size == 0)
+ return true;
+ if (Actual != nullptr)
+ return true;
+ return _error->Error(_("read, still have %llu to read but none left"), Size);
}
/*}}}*/
// FileFd::ReadLine - Read a complete line from the file /*{{{*/
char* FileFd::ReadLine(char *To, unsigned long long const Size)
{
*To = '\0';
- if (d == nullptr)
+ if (d == nullptr || Failed())
return nullptr;
return d->InternalReadLine(To, Size);
}
/*}}}*/
+// FileFd::Flush - Flush the file /*{{{*/
+bool FileFd::Flush()
+{
+ if (Failed())
+ return false;
+ if (d == nullptr)
+ return true;
+
+ return d->InternalFlush();
+}
+ /*}}}*/
// FileFd::Write - Write to the file /*{{{*/
bool FileFd::Write(const void *From,unsigned long long Size)
{
- if (d == nullptr)
+ if (d == nullptr || Failed())
return false;
ssize_t Res = 1;
errno = 0;
while (Res > 0 && Size > 0)
{
Res = d->InternalWrite(From, Size);
- if (Res < 0 && errno == EINTR)
- continue;
+
if (Res < 0)
+ {
+ if (errno == EINTR)
+ {
+ // trick the while-loop into running again
+ Res = 1;
+ errno = 0;
+ continue;
+ }
return d->InternalWriteError();
+ }
From = (char const *)From + Res;
Size -= Res;
if (d != NULL)
- d->seekpos += Res;
+ d->set_seekpos(d->get_seekpos() + Res);
}
if (Size == 0)
// FileFd::Seek - Seek in the file /*{{{*/
bool FileFd::Seek(unsigned long long To)
{
- if (d == nullptr)
+ if (d == nullptr || Failed())
return false;
Flags &= ~HitEof;
return d->InternalSeek(To);
// FileFd::Skip - Skip over data in the file /*{{{*/
bool FileFd::Skip(unsigned long long Over)
{
- if (d == nullptr)
+ if (d == nullptr || Failed())
return false;
return d->InternalSkip(Over);
}
// FileFd::Truncate - Truncate the file /*{{{*/
bool FileFd::Truncate(unsigned long long To)
{
- if (d == nullptr)
+ if (d == nullptr || Failed())
return false;
// truncating /dev/null is always successful - as we get an error otherwise
if (To == 0 && FileName == "/dev/null")
/* */
unsigned long long FileFd::Tell()
{
- if (d == nullptr)
+ if (d == nullptr || Failed())
return false;
off_t const Res = d->InternalTell();
if (Res == (off_t)-1)
FileFdErrno("lseek","Failed to determine the current file position");
- d->seekpos = Res;
+ d->set_seekpos(Res);
return Res;
}
/*}}}*/
static bool StatFileFd(char const * const msg, int const iFd, std::string const &FileName, struct stat &Buf, FileFdPrivate * const d) /*{{{*/
{
- bool ispipe = (d != NULL && d->is_pipe == true);
+ bool ispipe = (d != NULL && d->get_is_pipe() == true);
if (ispipe == false)
{
if (fstat(iFd,&Buf) != 0)
// we set it here, too, as we get the info here for free
// in theory the Open-methods should take care of it already
if (d != NULL)
- d->is_pipe = true;
+ d->set_is_pipe(true);
if (stat(FileName.c_str(), &Buf) != 0)
return _error->Errno("fstat", "Unable to determine %s for file %s", msg, FileName.c_str());
}
unsigned long long FileFd::Size()
{
if (d == nullptr)
- return false;
+ return 0;
return d->InternalSize();
}
/*}}}*/
/* */
bool FileFd::Close()
{
+ if (Failed() == false && Flush() == false)
+ return false;
if (iFd == -1)
return true;
}
if ((Flags & Replace) == Replace) {
- if (rename(TemporaryFileName.c_str(), FileName.c_str()) != 0)
+ if (Failed() == false && rename(TemporaryFileName.c_str(), FileName.c_str()) != 0)
Res &= _error->Errno("rename",_("Problem renaming the file %s to %s"), TemporaryFileName.c_str(), FileName.c_str());
FileName = TemporaryFileName; // for the unlink() below.
va_list args;
size_t msgSize = 400;
int const errsv = errno;
- while (true)
- {
+ bool retry;
+ do {
va_start(args,Description);
- if (_error->InsertErrno(GlobalError::ERROR, Function, Description, args, errsv, msgSize) == false)
- break;
+ retry = _error->InsertErrno(GlobalError::ERROR, Function, Description, args, errsv, msgSize);
va_end(args);
- }
+ } while (retry);
return false;
}
/*}}}*/
Flags |= Fail;
va_list args;
size_t msgSize = 400;
- while (true)
- {
+ bool retry;
+ do {
va_start(args,Description);
- if (_error->Insert(GlobalError::ERROR, Description, args, msgSize) == false)
- break;
+ retry = _error->Insert(GlobalError::ERROR, Description, args, msgSize);
va_end(args);
- }
+ } while (retry);
return false;
}
/*}}}*/
return result;
}
/*}}}*/
-std::string GetTempDir() /*{{{*/
+static std::string APT_NONNULL(1) GetTempDirEnv(char const * const env) /*{{{*/
{
- const char *tmpdir = getenv("TMPDIR");
+ const char *tmpdir = getenv(env);
#ifdef P_tmpdir
if (!tmpdir)
tmpdir = "/tmp";
return string(tmpdir);
+}
+ /*}}}*/
+std::string GetTempDir() /*{{{*/
+{
+ return GetTempDirEnv("TMPDIR");
}
std::string GetTempDir(std::string const &User)
{
}
/*}}}*/
bool Popen(const char* Args[], FileFd &Fd, pid_t &Child, FileFd::OpenMode Mode)/*{{{*/
+{
+ return Popen(Args, Fd, Child, Mode, true);
+}
+ /*}}}*/
+bool Popen(const char* Args[], FileFd &Fd, pid_t &Child, FileFd::OpenMode Mode, bool CaptureStderr)/*{{{*/
{
int fd;
if (Mode != FileFd::ReadOnly && Mode != FileFd::WriteOnly)
if(Mode == FileFd::ReadOnly)
{
dup2(fd, 1);
- dup2(fd, 2);
+ if (CaptureStderr == true)
+ dup2(fd, 2);
} else if(Mode == FileFd::WriteOnly)
dup2(fd, 0);
return _error->Error("Could restore a uid to root, privilege dropping did not work");
}
+ if (_config->FindB("APT::Sandbox::ResetEnvironment", true))
+ {
+ setenv("HOME", pw->pw_dir, 1);
+ setenv("USER", pw->pw_name, 1);
+ setenv("USERNAME", pw->pw_name, 1);
+ setenv("LOGNAME", pw->pw_name, 1);
+ auto const shell = flNotDir(pw->pw_shell);
+ if (shell == "false" || shell == "nologin")
+ setenv("SHELL", "/bin/sh", 1);
+ else
+ setenv("SHELL", pw->pw_shell, 1);
+ auto const apt_setenv_tmp = [](char const * const env) {
+ auto const tmpdir = getenv(env);
+ if (tmpdir != nullptr)
+ {
+ auto const ourtmpdir = GetTempDirEnv(env);
+ if (ourtmpdir != tmpdir)
+ setenv(env, ourtmpdir.c_str(), 1);
+ }
+ };
+ apt_setenv_tmp("TMPDIR");
+ apt_setenv_tmp("TEMPDIR");
+ apt_setenv_tmp("TMP");
+ apt_setenv_tmp("TEMP");
+ }
+
return true;
}
/*}}}*/
projects / apt.git / blobdiff