X-Git-Url: https://git.saurik.com/apt.git/blobdiff_plain/171c45bcc2a0a59db27f6bfc142efdd466cdd8bc..4e1c86a6c0cf9a6e51ec4f9b9ad9271547fa036d:/apt-pkg/contrib/fileutl.cc diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index 00afc8050..c8e685a5a 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: fileutl.cc,v 1.41 2002/04/24 05:02:40 jgg Exp $ +// $Id: fileutl.cc,v 1.42 2002/09/14 05:29:22 jgg Exp $ /* ###################################################################### File Utilities @@ -8,20 +8,28 @@ CopyFile - Buffered copy of a single file GetLock - dpkg compatible lock file manipulation (fcntl) - This source is placed in the Public Domain, do with it what you will - It was originally written by Jason Gunthorpe. + Most of this source is placed in the Public Domain, do with it what + you will + It was originally written by Jason Gunthorpe . + FileFd gzip support added by Martin Pitt + The exception is RunScripts() it is under the GPLv2 + ##################################################################### */ /*}}}*/ // Include Files /*{{{*/ -#ifdef __GNUG__ -#pragma implementation "apt-pkg/fileutl.h" -#endif +#include + #include +#include #include #include +#include +#include -#include +#include +#include +#include #include #include @@ -30,12 +38,122 @@ #include #include #include +#include #include #include +#include +#include + +#ifdef HAVE_ZLIB + #include +#endif +#ifdef HAVE_BZ2 + #include +#endif + +#ifdef WORDS_BIGENDIAN +#include +#endif + +#include /*}}}*/ using namespace std; +class FileFdPrivate { + public: +#ifdef HAVE_ZLIB + gzFile gz; +#else + void* gz; +#endif +#ifdef HAVE_BZ2 + BZFILE* bz2; +#else + void* bz2; +#endif + int compressed_fd; + pid_t compressor_pid; + bool pipe; + APT::Configuration::Compressor compressor; + unsigned int openmode; + unsigned long long seekpos; + FileFdPrivate() : gz(NULL), bz2(NULL), + compressed_fd(-1), compressor_pid(-1), pipe(false), + openmode(0), seekpos(0) {}; +}; + +// RunScripts - Run a set of scripts from a configuration subtree /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool RunScripts(const char *Cnf) +{ + Configuration::Item const *Opts = _config->Tree(Cnf); + if (Opts == 0 || Opts->Child == 0) + return true; + Opts = Opts->Child; + + // Fork for running the system calls + pid_t Child = ExecFork(); + + // This is the child + if (Child == 0) + { + if (_config->FindDir("DPkg::Chroot-Directory","/") != "/") + { + std::cerr << "Chrooting into " + << _config->FindDir("DPkg::Chroot-Directory") + << std::endl; + if (chroot(_config->FindDir("DPkg::Chroot-Directory","/").c_str()) != 0) + _exit(100); + } + + if (chdir("/tmp/") != 0) + _exit(100); + + unsigned int Count = 1; + for (; Opts != 0; Opts = Opts->Next, Count++) + { + if (Opts->Value.empty() == true) + continue; + + if (system(Opts->Value.c_str()) != 0) + _exit(100+Count); + } + _exit(0); + } + + // Wait for the child + int Status = 0; + while (waitpid(Child,&Status,0) != Child) + { + if (errno == EINTR) + continue; + return _error->Errno("waitpid","Couldn't wait for subprocess"); + } + + // Restore sig int/quit + signal(SIGQUIT,SIG_DFL); + signal(SIGINT,SIG_DFL); + + // Check for an error code. + if (WIFEXITED(Status) == 0 || WEXITSTATUS(Status) != 0) + { + unsigned int Count = WEXITSTATUS(Status); + if (Count > 100) + { + Count -= 100; + for (; Opts != 0 && Count != 1; Opts = Opts->Next, Count--); + _error->Error("Problem executing scripts %s '%s'",Cnf,Opts->Value.c_str()); + } + + return _error->Error("Sub-process returned an error code"); + } + + return true; +} + /*}}}*/ + // CopyFile - Buffered copy of a file /*{{{*/ // --------------------------------------------------------------------- /* The caller is expected to set things so that failure causes erasure */ @@ -46,10 +164,10 @@ bool CopyFile(FileFd &From,FileFd &To) // Buffered copy between fds SPtrArray Buf = new unsigned char[64000]; - unsigned long Size = From.Size(); + unsigned long long Size = From.Size(); while (Size != 0) { - unsigned long ToRead = Size; + unsigned long long ToRead = Size; if (Size > 64000) ToRead = 64000; @@ -71,7 +189,9 @@ bool CopyFile(FileFd &From,FileFd &To) close at some time. */ int GetLock(string File,bool Errors) { - int FD = open(File.c_str(),O_RDWR | O_CREAT | O_TRUNC,0640); + // GetLock() is used in aptitude on directories with public-write access + // Use O_NOFOLLOW here to prevent symlink traversal attacks + int FD = open(File.c_str(),O_RDWR | O_CREAT | O_NOFOLLOW,0640); if (FD < 0) { // Read only .. cant have locking problems there. @@ -117,7 +237,7 @@ int GetLock(string File,bool Errors) /*}}}*/ // FileExists - Check if a file exists /*{{{*/ // --------------------------------------------------------------------- -/* */ +/* Beware: Directories are also files! */ bool FileExists(string File) { struct stat Buf; @@ -126,6 +246,303 @@ bool FileExists(string File) return true; } /*}}}*/ +// RealFileExists - Check if a file exists and if it is really a file /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool RealFileExists(string File) +{ + struct stat Buf; + if (stat(File.c_str(),&Buf) != 0) + return false; + return ((Buf.st_mode & S_IFREG) != 0); +} + /*}}}*/ +// DirectoryExists - Check if a directory exists and is really one /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool DirectoryExists(string const &Path) +{ + struct stat Buf; + if (stat(Path.c_str(),&Buf) != 0) + return false; + return ((Buf.st_mode & S_IFDIR) != 0); +} + /*}}}*/ +// CreateDirectory - poor man's mkdir -p guarded by a parent directory /*{{{*/ +// --------------------------------------------------------------------- +/* This method will create all directories needed for path in good old + mkdir -p style but refuses to do this if Parent is not a prefix of + this Path. Example: /var/cache/ and /var/cache/apt/archives are given, + so it will create apt/archives if /var/cache exists - on the other + hand if the parent is /var/lib the creation will fail as this path + is not a parent of the path to be generated. */ +bool CreateDirectory(string const &Parent, string const &Path) +{ + if (Parent.empty() == true || Path.empty() == true) + return false; + + if (DirectoryExists(Path) == true) + return true; + + if (DirectoryExists(Parent) == false) + return false; + + // we are not going to create directories "into the blue" + if (Path.find(Parent, 0) != 0) + return false; + + vector const dirs = VectorizeString(Path.substr(Parent.size()), '/'); + string progress = Parent; + for (vector::const_iterator d = dirs.begin(); d != dirs.end(); ++d) + { + if (d->empty() == true) + continue; + + progress.append("/").append(*d); + if (DirectoryExists(progress) == true) + continue; + + if (mkdir(progress.c_str(), 0755) != 0) + return false; + } + return true; +} + /*}}}*/ +// CreateAPTDirectoryIfNeeded - ensure that the given directory exists /*{{{*/ +// --------------------------------------------------------------------- +/* a small wrapper around CreateDirectory to check if it exists and to + remove the trailing "/apt/" from the parent directory if needed */ +bool CreateAPTDirectoryIfNeeded(string const &Parent, string const &Path) +{ + if (DirectoryExists(Path) == true) + return true; + + size_t const len = Parent.size(); + if (len > 5 && Parent.find("/apt/", len - 6, 5) == len - 5) + { + if (CreateDirectory(Parent.substr(0,len-5), Path) == true) + return true; + } + else if (CreateDirectory(Parent, Path) == true) + return true; + + return false; +} + /*}}}*/ +// GetListOfFilesInDir - returns a vector of files in the given dir /*{{{*/ +// --------------------------------------------------------------------- +/* If an extension is given only files with this extension are included + in the returned vector, otherwise every "normal" file is included. */ +std::vector GetListOfFilesInDir(string const &Dir, string const &Ext, + bool const &SortList, bool const &AllowNoExt) +{ + std::vector ext; + ext.reserve(2); + if (Ext.empty() == false) + ext.push_back(Ext); + if (AllowNoExt == true && ext.empty() == false) + ext.push_back(""); + return GetListOfFilesInDir(Dir, ext, SortList); +} +std::vector GetListOfFilesInDir(string const &Dir, std::vector const &Ext, + bool const &SortList) +{ + // Attention debuggers: need to be set with the environment config file! + bool const Debug = _config->FindB("Debug::GetListOfFilesInDir", false); + if (Debug == true) + { + std::clog << "Accept in " << Dir << " only files with the following " << Ext.size() << " extensions:" << std::endl; + if (Ext.empty() == true) + std::clog << "\tNO extension" << std::endl; + else + for (std::vector::const_iterator e = Ext.begin(); + e != Ext.end(); ++e) + std::clog << '\t' << (e->empty() == true ? "NO" : *e) << " extension" << std::endl; + } + + std::vector List; + + if (DirectoryExists(Dir.c_str()) == false) + { + _error->Error(_("List of files can't be created as '%s' is not a directory"), Dir.c_str()); + return List; + } + + Configuration::MatchAgainstConfig SilentIgnore("Dir::Ignore-Files-Silently"); + DIR *D = opendir(Dir.c_str()); + if (D == 0) + { + _error->Errno("opendir",_("Unable to read %s"),Dir.c_str()); + return List; + } + + for (struct dirent *Ent = readdir(D); Ent != 0; Ent = readdir(D)) + { + // skip "hidden" files + if (Ent->d_name[0] == '.') + continue; + + // Make sure it is a file and not something else + string const File = flCombine(Dir,Ent->d_name); +#ifdef _DIRENT_HAVE_D_TYPE + if (Ent->d_type != DT_REG) +#endif + { + if (RealFileExists(File.c_str()) == false) + { + // do not show ignoration warnings for directories + if ( +#ifdef _DIRENT_HAVE_D_TYPE + Ent->d_type == DT_DIR || +#endif + DirectoryExists(File.c_str()) == true) + continue; + if (SilentIgnore.Match(Ent->d_name) == false) + _error->Notice(_("Ignoring '%s' in directory '%s' as it is not a regular file"), Ent->d_name, Dir.c_str()); + continue; + } + } + + // check for accepted extension: + // no extension given -> periods are bad as hell! + // extensions given -> "" extension allows no extension + if (Ext.empty() == false) + { + string d_ext = flExtension(Ent->d_name); + if (d_ext == Ent->d_name) // no extension + { + if (std::find(Ext.begin(), Ext.end(), "") == Ext.end()) + { + if (Debug == true) + std::clog << "Bad file: " << Ent->d_name << " → no extension" << std::endl; + if (SilentIgnore.Match(Ent->d_name) == false) + _error->Notice(_("Ignoring file '%s' in directory '%s' as it has no filename extension"), Ent->d_name, Dir.c_str()); + continue; + } + } + else if (std::find(Ext.begin(), Ext.end(), d_ext) == Ext.end()) + { + if (Debug == true) + std::clog << "Bad file: " << Ent->d_name << " → bad extension »" << flExtension(Ent->d_name) << "«" << std::endl; + if (SilentIgnore.Match(Ent->d_name) == false) + _error->Notice(_("Ignoring file '%s' in directory '%s' as it has an invalid filename extension"), Ent->d_name, Dir.c_str()); + continue; + } + } + + // Skip bad filenames ala run-parts + const char *C = Ent->d_name; + for (; *C != 0; ++C) + if (isalpha(*C) == 0 && isdigit(*C) == 0 + && *C != '_' && *C != '-') { + // no required extension -> dot is a bad character + if (*C == '.' && Ext.empty() == false) + continue; + break; + } + + // we don't reach the end of the name -> bad character included + if (*C != 0) + { + if (Debug == true) + std::clog << "Bad file: " << Ent->d_name << " → bad character »" + << *C << "« in filename (period allowed: " << (Ext.empty() ? "no" : "yes") << ")" << std::endl; + continue; + } + + // skip filenames which end with a period. These are never valid + if (*(C - 1) == '.') + { + if (Debug == true) + std::clog << "Bad file: " << Ent->d_name << " → Period as last character" << std::endl; + continue; + } + + if (Debug == true) + std::clog << "Accept file: " << Ent->d_name << " in " << Dir << std::endl; + List.push_back(File); + } + closedir(D); + + if (SortList == true) + std::sort(List.begin(),List.end()); + return List; +} +std::vector GetListOfFilesInDir(string const &Dir, bool SortList) +{ + bool const Debug = _config->FindB("Debug::GetListOfFilesInDir", false); + if (Debug == true) + std::clog << "Accept in " << Dir << " all regular files" << std::endl; + + std::vector List; + + if (DirectoryExists(Dir.c_str()) == false) + { + _error->Error(_("List of files can't be created as '%s' is not a directory"), Dir.c_str()); + return List; + } + + DIR *D = opendir(Dir.c_str()); + if (D == 0) + { + _error->Errno("opendir",_("Unable to read %s"),Dir.c_str()); + return List; + } + + for (struct dirent *Ent = readdir(D); Ent != 0; Ent = readdir(D)) + { + // skip "hidden" files + if (Ent->d_name[0] == '.') + continue; + + // Make sure it is a file and not something else + string const File = flCombine(Dir,Ent->d_name); +#ifdef _DIRENT_HAVE_D_TYPE + if (Ent->d_type != DT_REG) +#endif + { + if (RealFileExists(File.c_str()) == false) + { + if (Debug == true) + std::clog << "Bad file: " << Ent->d_name << " → it is not a real file" << std::endl; + continue; + } + } + + // Skip bad filenames ala run-parts + const char *C = Ent->d_name; + for (; *C != 0; ++C) + if (isalpha(*C) == 0 && isdigit(*C) == 0 + && *C != '_' && *C != '-' && *C != '.') + break; + + // we don't reach the end of the name -> bad character included + if (*C != 0) + { + if (Debug == true) + std::clog << "Bad file: " << Ent->d_name << " → bad character »" << *C << "« in filename" << std::endl; + continue; + } + + // skip filenames which end with a period. These are never valid + if (*(C - 1) == '.') + { + if (Debug == true) + std::clog << "Bad file: " << Ent->d_name << " → Period as last character" << std::endl; + continue; + } + + if (Debug == true) + std::clog << "Accept file: " << Ent->d_name << " in " << Dir << std::endl; + List.push_back(File); + } + closedir(D); + + if (SortList == true) + std::sort(List.begin(),List.end()); + return List; +} + /*}}}*/ // SafeGetCWD - This is a safer getcwd that returns a dynamic string /*{{{*/ // --------------------------------------------------------------------- /* We return / on failure. */ @@ -142,6 +559,17 @@ string SafeGetCWD() return S; } /*}}}*/ +// GetModificationTime - Get the mtime of the given file or -1 on error /*{{{*/ +// --------------------------------------------------------------------- +/* We return / on failure. */ +time_t GetModificationTime(string const &Path) +{ + struct stat St; + if (stat(Path.c_str(), &St) < 0) + return -1; + return St.st_mtime; +} + /*}}}*/ // flNotDir - Strip the directory from the filename /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -306,7 +734,7 @@ bool WaitFd(int Fd,bool write,unsigned long timeout) /* This is used if you want to cleanse the environment for the forked child, it fixes up the important signals and nukes all of the fds, otherwise acts like normal fork. */ -int ExecFork() +pid_t ExecFork() { // Fork off the process pid_t Process = fork(); @@ -326,10 +754,27 @@ int ExecFork() signal(SIGWINCH,SIG_DFL); signal(SIGCONT,SIG_DFL); signal(SIGTSTP,SIG_DFL); - + + set KeepFDs; + Configuration::Item const *Opts = _config->Tree("APT::Keep-Fds"); + if (Opts != 0 && Opts->Child != 0) + { + Opts = Opts->Child; + for (; Opts != 0; Opts = Opts->Next) + { + if (Opts->Value.empty() == true) + continue; + int fd = atoi(Opts->Value.c_str()); + KeepFDs.insert(fd); + } + } + // Close all of our FDs - just in case for (int K = 3; K != 40; K++) - fcntl(K,F_SETFD,FD_CLOEXEC); + { + if(KeepFDs.find(K) == KeepFDs.end()) + fcntl(K,F_SETFD,FD_CLOEXEC); + } } return Process; @@ -340,7 +785,7 @@ int ExecFork() /* Waits for the given sub process. If Reap is set then no errors are generated. Otherwise a failed subprocess will generate a proper descriptive message */ -bool ExecWait(int Pid,const char *Name,bool Reap) +bool ExecWait(pid_t Pid,const char *Name,bool Reap) { if (Pid <= 1) return true; @@ -355,7 +800,7 @@ bool ExecWait(int Pid,const char *Name,bool Reap) if (Reap == true) return false; - return _error->Error(_("Waited, for %s but it wasn't there"),Name); + return _error->Error(_("Waited for %s but it wasn't there"),Name); } @@ -364,8 +809,13 @@ bool ExecWait(int Pid,const char *Name,bool Reap) { if (Reap == true) return false; - if (WIFSIGNALED(Status) != 0 && WTERMSIG(Status) == SIGSEGV) - return _error->Error(_("Sub-process %s received a segmentation fault."),Name); + if (WIFSIGNALED(Status) != 0) + { + if( WTERMSIG(Status) == SIGSEGV) + return _error->Error(_("Sub-process %s received a segmentation fault."),Name); + else + return _error->Error(_("Sub-process %s received signal %u."),Name, WTERMSIG(Status)); + } if (WIFEXITED(Status) != 0) return _error->Error(_("Sub-process %s returned an error code (%u)"),Name,WEXITSTATUS(Status)); @@ -380,47 +830,340 @@ bool ExecWait(int Pid,const char *Name,bool Reap) // FileFd::Open - Open a file /*{{{*/ // --------------------------------------------------------------------- /* The most commonly used open mode combinations are given with Mode */ -bool FileFd::Open(string FileName,OpenMode Mode, unsigned long Perms) +bool FileFd::Open(string FileName,unsigned int const Mode,CompressMode Compress, unsigned long const Perms) { - Close(); - Flags = AutoClose; - switch (Mode) + if (Mode == ReadOnlyGzip) + return Open(FileName, ReadOnly, Gzip, Perms); + + if (Compress == Auto && (Mode & WriteOnly) == WriteOnly) + return _error->Error("Autodetection on %s only works in ReadOnly openmode!", FileName.c_str()); + + std::vector const compressors = APT::Configuration::getCompressors(); + std::vector::const_iterator compressor = compressors.begin(); + if (Compress == Auto) { - case ReadOnly: - iFd = open(FileName.c_str(),O_RDONLY); - break; - - case WriteEmpty: + for (; compressor != compressors.end(); ++compressor) { - struct stat Buf; - if (lstat(FileName.c_str(),&Buf) == 0 && S_ISLNK(Buf.st_mode)) - unlink(FileName.c_str()); - iFd = open(FileName.c_str(),O_RDWR | O_CREAT | O_TRUNC,Perms); + std::string file = std::string(FileName).append(compressor->Extension); + if (FileExists(file) == false) + continue; + FileName = file; break; } - - case WriteExists: - iFd = open(FileName.c_str(),O_RDWR); - break; + } + else if (Compress == Extension) + { + std::string::size_type const found = FileName.find_last_of('.'); + std::string ext; + if (found != std::string::npos) + { + ext = FileName.substr(found); + if (ext == ".new" || ext == ".bak") + { + std::string::size_type const found2 = FileName.find_last_of('.', found - 1); + if (found2 != std::string::npos) + ext = FileName.substr(found2, found - found2); + else + ext.clear(); + } + } + for (; compressor != compressors.end(); ++compressor) + if (ext == compressor->Extension) + break; + // no matching extension - assume uncompressed (imagine files like 'example.org_Packages') + if (compressor == compressors.end()) + for (compressor = compressors.begin(); compressor != compressors.end(); ++compressor) + if (compressor->Name == ".") + break; + } + else + { + std::string name; + switch (Compress) + { + case None: name = "."; break; + case Gzip: name = "gzip"; break; + case Bzip2: name = "bzip2"; break; + case Lzma: name = "lzma"; break; + case Xz: name = "xz"; break; + case Auto: + case Extension: + // Unreachable + return _error->Error("Opening File %s in None, Auto or Extension should be already handled?!?", FileName.c_str()); + } + for (; compressor != compressors.end(); ++compressor) + if (compressor->Name == name) + break; + if (compressor == compressors.end()) + return _error->Error("Can't find a configured compressor %s for file %s", name.c_str(), FileName.c_str()); + } + + if (compressor == compressors.end()) + return _error->Error("Can't find a match for specified compressor mode for file %s", FileName.c_str()); + return Open(FileName, Mode, *compressor, Perms); +} +bool FileFd::Open(string FileName,unsigned int const Mode,APT::Configuration::Compressor const &compressor, unsigned long const Perms) +{ + Close(); + Flags = AutoClose; - case WriteAny: - iFd = open(FileName.c_str(),O_RDWR | O_CREAT,Perms); - break; + if ((Mode & WriteOnly) != WriteOnly && (Mode & (Atomic | Create | Empty | Exclusive)) != 0) + return _error->Error("ReadOnly mode for %s doesn't accept additional flags!", FileName.c_str()); + if ((Mode & ReadWrite) == 0) + return _error->Error("No openmode provided in FileFd::Open for %s", FileName.c_str()); - case WriteTemp: + if ((Mode & Atomic) == Atomic) + { + Flags |= Replace; + char *name = strdup((FileName + ".XXXXXX").c_str()); + TemporaryFileName = string(mktemp(name)); + free(name); + } + else if ((Mode & (Exclusive | Create)) == (Exclusive | Create)) + { + // for atomic, this will be done by rename in Close() unlink(FileName.c_str()); - iFd = open(FileName.c_str(),O_RDWR | O_CREAT | O_EXCL,Perms); - break; - } + } + if ((Mode & Empty) == Empty) + { + struct stat Buf; + if (lstat(FileName.c_str(),&Buf) == 0 && S_ISLNK(Buf.st_mode)) + unlink(FileName.c_str()); + } + + int fileflags = 0; + #define if_FLAGGED_SET(FLAG, MODE) if ((Mode & FLAG) == FLAG) fileflags |= MODE + if_FLAGGED_SET(ReadWrite, O_RDWR); + else if_FLAGGED_SET(ReadOnly, O_RDONLY); + else if_FLAGGED_SET(WriteOnly, O_WRONLY); + + if_FLAGGED_SET(Create, O_CREAT); + if_FLAGGED_SET(Empty, O_TRUNC); + if_FLAGGED_SET(Exclusive, O_EXCL); + else if_FLAGGED_SET(Atomic, O_EXCL); + #undef if_FLAGGED_SET + + if (TemporaryFileName.empty() == false) + iFd = open(TemporaryFileName.c_str(), fileflags, Perms); + else + iFd = open(FileName.c_str(), fileflags, Perms); - if (iFd < 0) - return _error->Errno("open",_("Could not open file %s"),FileName.c_str()); - this->FileName = FileName; + if (iFd == -1 || OpenInternDescriptor(Mode, compressor) == false) + { + if (iFd != -1) + { + close (iFd); + iFd = -1; + } + return _error->Errno("open",_("Could not open file %s"), FileName.c_str()); + } + SetCloseExec(iFd,true); return true; } /*}}}*/ +// FileFd::OpenDescriptor - Open a filedescriptor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool FileFd::OpenDescriptor(int Fd, unsigned int const Mode, CompressMode Compress, bool AutoClose) +{ + std::vector const compressors = APT::Configuration::getCompressors(); + std::vector::const_iterator compressor = compressors.begin(); + std::string name; + + // compat with the old API + if (Mode == ReadOnlyGzip && Compress == None) + Compress = Gzip; + + switch (Compress) + { + case None: name = "."; break; + case Gzip: name = "gzip"; break; + case Bzip2: name = "bzip2"; break; + case Lzma: name = "lzma"; break; + case Xz: name = "xz"; break; + case Auto: + case Extension: + return _error->Error("Opening Fd %d in Auto or Extension compression mode is not supported", Fd); + } + for (; compressor != compressors.end(); ++compressor) + if (compressor->Name == name) + break; + if (compressor == compressors.end()) + return _error->Error("Can't find a configured compressor %s for file %s", name.c_str(), FileName.c_str()); + + return OpenDescriptor(Fd, Mode, *compressor, AutoClose); +} +bool FileFd::OpenDescriptor(int Fd, unsigned int const Mode, APT::Configuration::Compressor const &compressor, bool AutoClose) +{ + Close(); + Flags = (AutoClose) ? FileFd::AutoClose : 0; + iFd = Fd; + this->FileName = ""; + if (OpenInternDescriptor(Mode, compressor) == false) + { + if (AutoClose) + close (iFd); + return _error->Errno("gzdopen",_("Could not open file descriptor %d"), Fd); + } + return true; +} +bool FileFd::OpenInternDescriptor(unsigned int const Mode, APT::Configuration::Compressor const &compressor) +{ + if (compressor.Name == "." || compressor.Binary.empty() == true) + return true; + + if (d == NULL) + { + d = new FileFdPrivate(); + d->openmode = Mode; + d->compressor = compressor; + } + +#ifdef HAVE_ZLIB + if (compressor.Name == "gzip") + { + if (d->gz != NULL) + { + gzclose(d->gz); + d->gz = NULL; + } + if ((Mode & ReadWrite) == ReadWrite) + d->gz = gzdopen(iFd, "r+"); + else if ((Mode & WriteOnly) == WriteOnly) + d->gz = gzdopen(iFd, "w"); + else + d->gz = gzdopen(iFd, "r"); + if (d->gz == NULL) + return false; + Flags |= Compressed; + return true; + } +#endif +#ifdef HAVE_BZ2 + if (compressor.Name == "bzip2") + { + if (d->bz2 != NULL) + { + BZ2_bzclose(d->bz2); + d->bz2 = NULL; + } + if ((Mode & ReadWrite) == ReadWrite) + d->bz2 = BZ2_bzdopen(iFd, "r+"); + else if ((Mode & WriteOnly) == WriteOnly) + d->bz2 = BZ2_bzdopen(iFd, "w"); + else + d->bz2 = BZ2_bzdopen(iFd, "r"); + if (d->bz2 == NULL) + return false; + Flags |= Compressed; + return true; + } +#endif + + // collect zombies here in case we reopen + if (d->compressor_pid > 0) + ExecWait(d->compressor_pid, "FileFdCompressor", true); + + if ((Mode & ReadWrite) == ReadWrite) + { + Flags |= Fail; + return _error->Error("ReadWrite mode is not supported for file %s", FileName.c_str()); + } + + bool const Comp = (Mode & WriteOnly) == WriteOnly; + if (Comp == false) + { + // Handle 'decompression' of empty files + struct stat Buf; + fstat(iFd, &Buf); + if (Buf.st_size == 0 && S_ISFIFO(Buf.st_mode) == false) + return true; + + // We don't need the file open - instead let the compressor open it + // as he properly knows better how to efficiently read from 'his' file + if (FileName.empty() == false) + { + close(iFd); + iFd = -1; + } + } + + // Create a data pipe + int Pipe[2] = {-1,-1}; + if (pipe(Pipe) != 0) + { + Flags |= Fail; + return _error->Errno("pipe",_("Failed to create subprocess IPC")); + } + for (int J = 0; J != 2; J++) + SetCloseExec(Pipe[J],true); + + d->compressed_fd = iFd; + d->pipe = true; + + if (Comp == true) + iFd = Pipe[1]; + else + iFd = Pipe[0]; + + // The child.. + d->compressor_pid = ExecFork(); + if (d->compressor_pid == 0) + { + if (Comp == true) + { + dup2(d->compressed_fd,STDOUT_FILENO); + dup2(Pipe[0],STDIN_FILENO); + } + else + { + if (FileName.empty() == true) + dup2(d->compressed_fd,STDIN_FILENO); + dup2(Pipe[1],STDOUT_FILENO); + } + int const nullfd = open("/dev/null", O_WRONLY); + if (nullfd != -1) + { + dup2(nullfd,STDERR_FILENO); + close(nullfd); + } + + SetCloseExec(STDOUT_FILENO,false); + SetCloseExec(STDIN_FILENO,false); + + std::vector Args; + Args.push_back(compressor.Binary.c_str()); + std::vector const * const addArgs = + (Comp == true) ? &(compressor.CompressArgs) : &(compressor.UncompressArgs); + for (std::vector::const_iterator a = addArgs->begin(); + a != addArgs->end(); ++a) + Args.push_back(a->c_str()); + if (Comp == false && FileName.empty() == false) + { + Args.push_back("--stdout"); + if (TemporaryFileName.empty() == false) + Args.push_back(TemporaryFileName.c_str()); + else + Args.push_back(FileName.c_str()); + } + Args.push_back(NULL); + + execvp(Args[0],(char **)&Args[0]); + cerr << _("Failed to exec compressor ") << Args[0] << endl; + _exit(100); + } + if (Comp == true) + close(Pipe[0]); + else + close(Pipe[1]); + if ((Comp == true || FileName.empty() == true) && d->compressed_fd != -1) + close(d->compressed_fd); + + return true; +} + /*}}}*/ // FileFd::~File - Closes the file /*{{{*/ // --------------------------------------------------------------------- /* If the proper modes are selected then we close the Fd and possibly @@ -434,26 +1177,57 @@ FileFd::~FileFd() // --------------------------------------------------------------------- /* We are carefull to handle interruption by a signal while reading gracefully. */ -bool FileFd::Read(void *To,unsigned long Size,unsigned long *Actual) +bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual) { int Res; errno = 0; if (Actual != 0) *Actual = 0; - + *((char *)To) = '\0'; do { - Res = read(iFd,To,Size); - if (Res < 0 && errno == EINTR) - continue; +#ifdef HAVE_ZLIB + if (d != NULL && d->gz != NULL) + Res = gzread(d->gz,To,Size); + else +#endif +#ifdef HAVE_BZ2 + if (d != NULL && d->bz2 != NULL) + Res = BZ2_bzread(d->bz2,To,Size); + else +#endif + Res = read(iFd,To,Size); + if (Res < 0) { + if (errno == EINTR) + continue; Flags |= Fail; +#ifdef HAVE_ZLIB + if (d != NULL && d->gz != NULL) + { + int err; + char const * const errmsg = gzerror(d->gz, &err); + if (err != Z_ERRNO) + return _error->Error("gzread: %s (%d: %s)", _("Read error"), err, errmsg); + } +#endif +#ifdef HAVE_BZ2 + if (d != NULL && d->bz2 != NULL) + { + int err; + char const * const errmsg = BZ2_bzerror(d->bz2, &err); + if (err != BZ_IO_ERROR) + return _error->Error("BZ2_bzread: %s (%d: %s)", _("Read error"), err, errmsg); + } +#endif return _error->Errno("read",_("Read error")); } To = (char *)To + Res; Size -= Res; + if (d != NULL) + d->seekpos += Res; if (Actual != 0) *Actual += Res; } @@ -470,29 +1244,88 @@ bool FileFd::Read(void *To,unsigned long Size,unsigned long *Actual) } Flags |= Fail; - return _error->Error(_("read, still have %lu to read but none left"),Size); + return _error->Error(_("read, still have %llu to read but none left"), Size); +} + /*}}}*/ +// FileFd::ReadLine - Read a complete line from the file /*{{{*/ +// --------------------------------------------------------------------- +/* Beware: This method can be quiet slow for big buffers on UNcompressed + files because of the naive implementation! */ +char* FileFd::ReadLine(char *To, unsigned long long const Size) +{ + *To = '\0'; +#ifdef HAVE_ZLIB + if (d != NULL && d->gz != NULL) + return gzgets(d->gz, To, Size); +#endif + + unsigned long long read = 0; + while ((Size - 1) != read) + { + unsigned long long done = 0; + if (Read(To + read, 1, &done) == false) + return NULL; + if (done == 0) + break; + if (To[read++] == '\n') + break; + } + if (read == 0) + return NULL; + To[read] = '\0'; + return To; } /*}}}*/ // FileFd::Write - Write to the file /*{{{*/ // --------------------------------------------------------------------- /* */ -bool FileFd::Write(const void *From,unsigned long Size) +bool FileFd::Write(const void *From,unsigned long long Size) { int Res; errno = 0; do { - Res = write(iFd,From,Size); +#ifdef HAVE_ZLIB + if (d != NULL && d->gz != NULL) + Res = gzwrite(d->gz,From,Size); + else +#endif +#ifdef HAVE_BZ2 + if (d != NULL && d->bz2 != NULL) + Res = BZ2_bzwrite(d->bz2,(void*)From,Size); + else +#endif + Res = write(iFd,From,Size); if (Res < 0 && errno == EINTR) continue; if (Res < 0) { Flags |= Fail; +#ifdef HAVE_ZLIB + if (d != NULL && d->gz != NULL) + { + int err; + char const * const errmsg = gzerror(d->gz, &err); + if (err != Z_ERRNO) + return _error->Error("gzwrite: %s (%d: %s)", _("Write error"), err, errmsg); + } +#endif +#ifdef HAVE_BZ2 + if (d != NULL && d->bz2 != NULL) + { + int err; + char const * const errmsg = BZ2_bzerror(d->bz2, &err); + if (err != BZ_IO_ERROR) + return _error->Error("BZ2_bzwrite: %s (%d: %s)", _("Write error"), err, errmsg); + } +#endif return _error->Errno("write",_("Write error")); } From = (char *)From + Res; Size -= Res; + if (d != NULL) + d->seekpos += Res; } while (Res > 0 && Size > 0); @@ -500,46 +1333,167 @@ bool FileFd::Write(const void *From,unsigned long Size) return true; Flags |= Fail; - return _error->Error(_("write, still have %lu to write but couldn't"),Size); + return _error->Error(_("write, still have %llu to write but couldn't"), Size); +} +bool FileFd::Write(int Fd, const void *From, unsigned long long Size) +{ + int Res; + errno = 0; + do + { + Res = write(Fd,From,Size); + if (Res < 0 && errno == EINTR) + continue; + if (Res < 0) + return _error->Errno("write",_("Write error")); + + From = (char *)From + Res; + Size -= Res; + } + while (Res > 0 && Size > 0); + + if (Size == 0) + return true; + + return _error->Error(_("write, still have %llu to write but couldn't"), Size); } /*}}}*/ // FileFd::Seek - Seek in the file /*{{{*/ // --------------------------------------------------------------------- /* */ -bool FileFd::Seek(unsigned long To) +bool FileFd::Seek(unsigned long long To) { - if (lseek(iFd,To,SEEK_SET) != (signed)To) + if (d != NULL && (d->pipe == true +#ifdef HAVE_BZ2 + || d->bz2 != NULL +#endif + )) + { + // Our poor man seeking in pipes is costly, so try to avoid it + unsigned long long seekpos = Tell(); + if (seekpos == To) + return true; + else if (seekpos < To) + return Skip(To - seekpos); + + if ((d->openmode & ReadOnly) != ReadOnly) + { + Flags |= Fail; + return _error->Error("Reopen is only implemented for read-only files!"); + } +#ifdef HAVE_BZ2 + if (d->bz2 != NULL) + BZ2_bzclose(d->bz2); +#endif + if (iFd != -1) + close(iFd); + iFd = -1; + if (TemporaryFileName.empty() == false) + iFd = open(TemporaryFileName.c_str(), O_RDONLY); + else if (FileName.empty() == false) + iFd = open(FileName.c_str(), O_RDONLY); + else + { + if (d->compressed_fd > 0) + if (lseek(d->compressed_fd, 0, SEEK_SET) != 0) + iFd = d->compressed_fd; + if (iFd <= 0) + { + Flags |= Fail; + return _error->Error("Reopen is not implemented for pipes opened with FileFd::OpenDescriptor()!"); + } + } + + if (OpenInternDescriptor(d->openmode, d->compressor) == false) + { + Flags |= Fail; + return _error->Error("Seek on file %s because it couldn't be reopened", FileName.c_str()); + } + + if (To != 0) + return Skip(To); + + d->seekpos = To; + return true; + } + int res; +#ifdef HAVE_ZLIB + if (d != NULL && d->gz) + res = gzseek(d->gz,To,SEEK_SET); + else +#endif + res = lseek(iFd,To,SEEK_SET); + if (res != (signed)To) { Flags |= Fail; - return _error->Error("Unable to seek to %lu",To); + return _error->Error("Unable to seek to %llu", To); } - + + if (d != NULL) + d->seekpos = To; return true; } /*}}}*/ // FileFd::Skip - Seek in the file /*{{{*/ // --------------------------------------------------------------------- /* */ -bool FileFd::Skip(unsigned long Over) +bool FileFd::Skip(unsigned long long Over) { - if (lseek(iFd,Over,SEEK_CUR) < 0) + if (d != NULL && (d->pipe == true +#ifdef HAVE_BZ2 + || d->bz2 != NULL +#endif + )) + { + d->seekpos += Over; + char buffer[1024]; + while (Over != 0) + { + unsigned long long toread = std::min((unsigned long long) sizeof(buffer), Over); + if (Read(buffer, toread) == false) + { + Flags |= Fail; + return _error->Error("Unable to seek ahead %llu",Over); + } + Over -= toread; + } + return true; + } + + int res; +#ifdef HAVE_ZLIB + if (d != NULL && d->gz != NULL) + res = gzseek(d->gz,Over,SEEK_CUR); + else +#endif + res = lseek(iFd,Over,SEEK_CUR); + if (res < 0) { Flags |= Fail; - return _error->Error("Unable to seek ahead %lu",Over); + return _error->Error("Unable to seek ahead %llu",Over); } - + if (d != NULL) + d->seekpos = res; + return true; } /*}}}*/ // FileFd::Truncate - Truncate the file /*{{{*/ // --------------------------------------------------------------------- /* */ -bool FileFd::Truncate(unsigned long To) +bool FileFd::Truncate(unsigned long long To) { +#if defined HAVE_ZLIB || defined HAVE_BZ2 + if (d != NULL && (d->gz != NULL || d->bz2 != NULL)) + { + Flags |= Fail; + return _error->Error("Truncating compressed files is not implemented (%s)", FileName.c_str()); + } +#endif if (ftruncate(iFd,To) != 0) { Flags |= Fail; - return _error->Error("Unable to truncate to %lu",To); + return _error->Error("Unable to truncate to %llu",To); } return true; @@ -548,40 +1502,216 @@ bool FileFd::Truncate(unsigned long To) // FileFd::Tell - Current seek position /*{{{*/ // --------------------------------------------------------------------- /* */ -unsigned long FileFd::Tell() +unsigned long long FileFd::Tell() { - off_t Res = lseek(iFd,0,SEEK_CUR); + // In theory, we could just return seekpos here always instead of + // seeking around, but not all users of FileFd use always Seek() and co + // so d->seekpos isn't always true and we can just use it as a hint if + // we have nothing else, but not always as an authority… + if (d != NULL && (d->pipe == true +#ifdef HAVE_BZ2 + || d->bz2 != NULL +#endif + )) + return d->seekpos; + + off_t Res; +#ifdef HAVE_ZLIB + if (d != NULL && d->gz != NULL) + Res = gztell(d->gz); + else +#endif + Res = lseek(iFd,0,SEEK_CUR); if (Res == (off_t)-1) + { + Flags |= Fail; _error->Errno("lseek","Failed to determine the current file position"); + } + if (d != NULL) + d->seekpos = Res; return Res; } /*}}}*/ -// FileFd::Size - Return the size of the file /*{{{*/ +// FileFd::FileSize - Return the size of the file /*{{{*/ // --------------------------------------------------------------------- /* */ -unsigned long FileFd::Size() +unsigned long long FileFd::FileSize() { struct stat Buf; - if (fstat(iFd,&Buf) != 0) + if ((d == NULL || d->pipe == false) && fstat(iFd,&Buf) != 0) + { + Flags |= Fail; return _error->Errno("fstat","Unable to determine the file size"); + } + + // for compressor pipes st_size is undefined and at 'best' zero + if ((d != NULL && d->pipe == true) || S_ISFIFO(Buf.st_mode)) + { + // we set it here, too, as we get the info here for free + // in theory the Open-methods should take care of it already + if (d != NULL) + d->pipe = true; + if (stat(FileName.c_str(), &Buf) != 0) + { + Flags |= Fail; + return _error->Errno("stat","Unable to determine the file size"); + } + } + return Buf.st_size; } /*}}}*/ +// FileFd::Size - Return the size of the content in the file /*{{{*/ +// --------------------------------------------------------------------- +/* */ +unsigned long long FileFd::Size() +{ + unsigned long long size = FileSize(); + + // for compressor pipes st_size is undefined and at 'best' zero, + // so we 'read' the content and 'seek' back - see there + if (d != NULL && (d->pipe == true +#ifdef HAVE_BZ2 + || (d->bz2 && size > 0) +#endif + )) + { + unsigned long long const oldSeek = Tell(); + char ignore[1000]; + unsigned long long read = 0; + do { + Read(ignore, sizeof(ignore), &read); + } while(read != 0); + size = Tell(); + Seek(oldSeek); + } +#ifdef HAVE_ZLIB + // only check gzsize if we are actually a gzip file, just checking for + // "gz" is not sufficient as uncompressed files could be opened with + // gzopen in "direct" mode as well + else if (d != NULL && d->gz && !gzdirect(d->gz) && size > 0) + { + off_t const oldPos = lseek(iFd,0,SEEK_CUR); + /* unfortunately zlib.h doesn't provide a gzsize(), so we have to do + * this ourselves; the original (uncompressed) file size is the last 32 + * bits of the file */ + // FIXME: Size for gz-files is limited by 32bit… no largefile support + if (lseek(iFd, -4, SEEK_END) < 0) + { + Flags |= Fail; + return _error->Errno("lseek","Unable to seek to end of gzipped file"); + } + size = 0L; + if (read(iFd, &size, 4) != 4) + { + Flags |= Fail; + return _error->Errno("read","Unable to read original size of gzipped file"); + } + +#ifdef WORDS_BIGENDIAN + uint32_t tmp_size = size; + uint8_t const * const p = (uint8_t const * const) &tmp_size; + tmp_size = (p[3] << 24) | (p[2] << 16) | (p[1] << 8) | p[0]; + size = tmp_size; +#endif + + if (lseek(iFd, oldPos, SEEK_SET) < 0) + { + Flags |= Fail; + return _error->Errno("lseek","Unable to seek in gzipped file"); + } + + return size; + } +#endif + + return size; +} + /*}}}*/ +// FileFd::ModificationTime - Return the time of last touch /*{{{*/ +// --------------------------------------------------------------------- +/* */ +time_t FileFd::ModificationTime() +{ + struct stat Buf; + if ((d == NULL || d->pipe == false) && fstat(iFd,&Buf) != 0) + { + Flags |= Fail; + _error->Errno("fstat","Unable to determine the modification time of file %s", FileName.c_str()); + return 0; + } + + // for compressor pipes st_size is undefined and at 'best' zero + if ((d != NULL && d->pipe == true) || S_ISFIFO(Buf.st_mode)) + { + // we set it here, too, as we get the info here for free + // in theory the Open-methods should take care of it already + if (d != NULL) + d->pipe = true; + if (stat(FileName.c_str(), &Buf) != 0) + { + Flags |= Fail; + _error->Errno("fstat","Unable to determine the modification time of file %s", FileName.c_str()); + return 0; + } + } + + return Buf.st_mtime; +} + /*}}}*/ // FileFd::Close - Close the file if the close flag is set /*{{{*/ // --------------------------------------------------------------------- /* */ bool FileFd::Close() { + if (iFd == -1) + return true; + bool Res = true; if ((Flags & AutoClose) == AutoClose) - if (iFd >= 0 && close(iFd) != 0) - Res &= _error->Errno("close",_("Problem closing the file")); + { +#ifdef HAVE_ZLIB + if (d != NULL && d->gz != NULL) { + int const e = gzclose(d->gz); + // gzdclose() on empty files always fails with "buffer error" here, ignore that + if (e != 0 && e != Z_BUF_ERROR) + Res &= _error->Errno("close",_("Problem closing the gzip file %s"), FileName.c_str()); + } else +#endif +#ifdef HAVE_BZ2 + if (d != NULL && d->bz2 != NULL) + BZ2_bzclose(d->bz2); + else +#endif + if (iFd > 0 && close(iFd) != 0) + Res &= _error->Errno("close",_("Problem closing the file %s"), FileName.c_str()); + } + + if ((Flags & Replace) == Replace && iFd >= 0) { + if (rename(TemporaryFileName.c_str(), FileName.c_str()) != 0) + Res &= _error->Errno("rename",_("Problem renaming the file %s to %s"), TemporaryFileName.c_str(), FileName.c_str()); + + FileName = TemporaryFileName; // for the unlink() below. + TemporaryFileName.clear(); + } + iFd = -1; - + if ((Flags & Fail) == Fail && (Flags & DelOnFail) == DelOnFail && FileName.empty() == false) if (unlink(FileName.c_str()) != 0) - Res &= _error->WarningE("unlnk",_("Problem unlinking the file")); + Res &= _error->WarningE("unlnk",_("Problem unlinking the file %s"), FileName.c_str()); + + if (d != NULL) + { + if (d->compressor_pid > 0) + ExecWait(d->compressor_pid, "FileFdCompressor", true); + delete d; + d = NULL; + } + + if (Res == false) + Flags |= Fail; return Res; } /*}}}*/ @@ -592,8 +1722,13 @@ bool FileFd::Sync() { #ifdef _POSIX_SYNCHRONIZED_IO if (fsync(iFd) != 0) + { + Flags |= Fail; return _error->Errno("sync",_("Problem syncing the file")); + } #endif return true; } /*}}}*/ + +gzFile FileFd::gzFd() { return (gzFile) d->gz; }