apt-pkg/contrib/fileutl.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: fileutl.cc,v 1.42 2002/09/14 05:29:22 jgg Exp $
   4 /* ######################################################################
   5
   6    File Utilities
   7
   8    CopyFile - Buffered copy of a single file
   9    GetLock - dpkg compatible lock file manipulation (fcntl)
  10
  11    Most of this source is placed in the Public Domain, do with it what
  12    you will
  13    It was originally written by Jason Gunthorpe <jgg@debian.org>.
  14    FileFd gzip support added by Martin Pitt <martin.pitt@canonical.com>
  15
  16    The exception is RunScripts() it is under the GPLv2
  17
  18    ##################################################################### */
  19                                                                         /*}}}*/
  20 // Include Files                                                        /*{{{*/
  21 #include <config.h>
  22
  23 #include <apt-pkg/fileutl.h>
  24 #include <apt-pkg/strutl.h>
  25 #include <apt-pkg/error.h>
  26 #include <apt-pkg/sptr.h>
  27 #include <apt-pkg/aptconfiguration.h>
  28 #include <apt-pkg/configuration.h>
  29
  30 #include <cstdlib>
  31 #include <cstring>
  32 #include <cstdio>
  33
  34 #include <iostream>
  35 #include <unistd.h>
  36 #include <fcntl.h>
  37 #include <sys/stat.h>
  38 #include <sys/types.h>
  39 #include <sys/time.h>
  40 #include <sys/wait.h>
  41 #include <dirent.h>
  42 #include <signal.h>
  43 #include <errno.h>
  44 #include <set>
  45 #include <algorithm>
  46
  47 #ifdef HAVE_ZLIB
  48         #include <zlib.h>
  49 #endif
  50 #ifdef HAVE_BZ2
  51         #include <bzlib.h>
  52 #endif
  53
  54 #ifdef WORDS_BIGENDIAN
  55 #include <inttypes.h>
  56 #endif
  57
  58 #include <apti18n.h>
  59                                                                         /*}}}*/
  60
  61 using namespace std;
  62
  63 class FileFdPrivate {
  64         public:
  65 #ifdef HAVE_ZLIB
  66         gzFile gz;
  67 #else
  68         void* gz;
  69 #endif
  70 #ifdef HAVE_BZ2
  71         BZFILE* bz2;
  72 #else
  73         void* bz2;
  74 #endif
  75         int compressed_fd;
  76         pid_t compressor_pid;
  77         bool pipe;
  78         APT::Configuration::Compressor compressor;
  79         unsigned int openmode;
  80         unsigned long long seekpos;
  81         FileFdPrivate() : gz(NULL), bz2(NULL),
  82                           compressed_fd(-1), compressor_pid(-1), pipe(false),
  83                           openmode(0), seekpos(0) {};
  84         bool CloseDown(std::string const &FileName)
  85         {
  86            bool Res = true;
  87 #ifdef HAVE_ZLIB
  88            if (gz != NULL) {
  89               int const e = gzclose(gz);
  90               gz = NULL;
  91               // gzdclose() on empty files always fails with "buffer error" here, ignore that
  92               if (e != 0 && e != Z_BUF_ERROR)
  93                  Res &= _error->Errno("close",_("Problem closing the gzip file %s"), FileName.c_str());
  94            }
  95 #endif
  96 #ifdef HAVE_BZ2
  97            if (bz2 != NULL) {
  98               BZ2_bzclose(bz2);
  99               bz2 = NULL;
 100            }
 101 #endif
 102            if (compressor_pid > 0)
 103               ExecWait(compressor_pid, "FileFdCompressor", true);
 104            compressor_pid = -1;
 105
 106            return Res;
 107         }
 108         ~FileFdPrivate() { CloseDown(""); }
 109 };
 110
 111 // RunScripts - Run a set of scripts from a configuration subtree       /*{{{*/
 112 // ---------------------------------------------------------------------
 113 /* */
 114 bool RunScripts(const char *Cnf)
 115 {
 116    Configuration::Item const *Opts = _config->Tree(Cnf);
 117    if (Opts == 0 || Opts->Child == 0)
 118       return true;
 119    Opts = Opts->Child;
 120
 121    // Fork for running the system calls
 122    pid_t Child = ExecFork();
 123
 124    // This is the child
 125    if (Child == 0)
 126    {
 127       if (_config->FindDir("DPkg::Chroot-Directory","/") != "/")
 128       {
 129          std::cerr << "Chrooting into "
 130                    << _config->FindDir("DPkg::Chroot-Directory")
 131                    << std::endl;
 132          if (chroot(_config->FindDir("DPkg::Chroot-Directory","/").c_str()) != 0)
 133             _exit(100);
 134       }
 135
 136       if (chdir("/tmp/") != 0)
 137          _exit(100);
 138
 139       unsigned int Count = 1;
 140       for (; Opts != 0; Opts = Opts->Next, Count++)
 141       {
 142          if (Opts->Value.empty() == true)
 143             continue;
 144
 145          if (system(Opts->Value.c_str()) != 0)
 146             _exit(100+Count);
 147       }
 148       _exit(0);
 149    }
 150
 151    // Wait for the child
 152    int Status = 0;
 153    while (waitpid(Child,&Status,0) != Child)
 154    {
 155       if (errno == EINTR)
 156          continue;
 157       return _error->Errno("waitpid","Couldn't wait for subprocess");
 158    }
 159
 160    // Restore sig int/quit
 161    signal(SIGQUIT,SIG_DFL);
 162    signal(SIGINT,SIG_DFL);
 163
 164    // Check for an error code.
 165    if (WIFEXITED(Status) == 0 || WEXITSTATUS(Status) != 0)
 166    {
 167       unsigned int Count = WEXITSTATUS(Status);
 168       if (Count > 100)
 169       {
 170          Count -= 100;
 171          for (; Opts != 0 && Count != 1; Opts = Opts->Next, Count--);
 172          _error->Error("Problem executing scripts %s '%s'",Cnf,Opts->Value.c_str());
 173       }
 174
 175       return _error->Error("Sub-process returned an error code");
 176    }
 177
 178    return true;
 179 }
 180                                                                         /*}}}*/
 181
 182 // CopyFile - Buffered copy of a file                                   /*{{{*/
 183 // ---------------------------------------------------------------------
 184 /* The caller is expected to set things so that failure causes erasure */
 185 bool CopyFile(FileFd &From,FileFd &To)
 186 {
 187    if (From.IsOpen() == false || To.IsOpen() == false ||
 188          From.Failed() == true || To.Failed() == true)
 189       return false;
 190
 191    // Buffered copy between fds
 192    SPtrArray<unsigned char> Buf = new unsigned char[64000];
 193    unsigned long long Size = From.Size();
 194    while (Size != 0)
 195    {
 196       unsigned long long ToRead = Size;
 197       if (Size > 64000)
 198          ToRead = 64000;
 199
 200       if (From.Read(Buf,ToRead) == false ||
 201           To.Write(Buf,ToRead) == false)
 202          return false;
 203
 204       Size -= ToRead;
 205    }
 206
 207    return true;
 208 }
 209                                                                         /*}}}*/
 210 // GetLock - Gets a lock file                                           /*{{{*/
 211 // ---------------------------------------------------------------------
 212 /* This will create an empty file of the given name and lock it. Once this
 213    is done all other calls to GetLock in any other process will fail with
 214    -1. The return result is the fd of the file, the call should call
 215    close at some time. */
 216 int GetLock(string File,bool Errors)
 217 {
 218    // GetLock() is used in aptitude on directories with public-write access
 219    // Use O_NOFOLLOW here to prevent symlink traversal attacks
 220    int FD = open(File.c_str(),O_RDWR | O_CREAT | O_NOFOLLOW,0640);
 221    if (FD < 0)
 222    {
 223       // Read only .. cant have locking problems there.
 224       if (errno == EROFS)
 225       {
 226          _error->Warning(_("Not using locking for read only lock file %s"),File.c_str());
 227          return dup(0);       // Need something for the caller to close
 228       }
 229
 230       if (Errors == true)
 231          _error->Errno("open",_("Could not open lock file %s"),File.c_str());
 232
 233       // Feh.. We do this to distinguish the lock vs open case..
 234       errno = EPERM;
 235       return -1;
 236    }
 237    SetCloseExec(FD,true);
 238
 239    // Aquire a write lock
 240    struct flock fl;
 241    fl.l_type = F_WRLCK;
 242    fl.l_whence = SEEK_SET;
 243    fl.l_start = 0;
 244    fl.l_len = 0;
 245    if (fcntl(FD,F_SETLK,&fl) == -1)
 246    {
 247       // always close to not leak resources
 248       int Tmp = errno;
 249       close(FD);
 250       errno = Tmp;
 251
 252       if (errno == ENOLCK)
 253       {
 254          _error->Warning(_("Not using locking for nfs mounted lock file %s"),File.c_str());
 255          return dup(0);       // Need something for the caller to close
 256       }
 257
 258       if (Errors == true)
 259          _error->Errno("open",_("Could not get lock %s"),File.c_str());
 260
 261       return -1;
 262    }
 263
 264    return FD;
 265 }
 266                                                                         /*}}}*/
 267 // FileExists - Check if a file exists                                  /*{{{*/
 268 // ---------------------------------------------------------------------
 269 /* Beware: Directories are also files! */
 270 bool FileExists(string File)
 271 {
 272    struct stat Buf;
 273    if (stat(File.c_str(),&Buf) != 0)
 274       return false;
 275    return true;
 276 }
 277                                                                         /*}}}*/
 278 // RealFileExists - Check if a file exists and if it is really a file   /*{{{*/
 279 // ---------------------------------------------------------------------
 280 /* */
 281 bool RealFileExists(string File)
 282 {
 283    struct stat Buf;
 284    if (stat(File.c_str(),&Buf) != 0)
 285       return false;
 286    return ((Buf.st_mode & S_IFREG) != 0);
 287 }
 288                                                                         /*}}}*/
 289 // DirectoryExists - Check if a directory exists and is really one      /*{{{*/
 290 // ---------------------------------------------------------------------
 291 /* */
 292 bool DirectoryExists(string const &Path)
 293 {
 294    struct stat Buf;
 295    if (stat(Path.c_str(),&Buf) != 0)
 296       return false;
 297    return ((Buf.st_mode & S_IFDIR) != 0);
 298 }
 299                                                                         /*}}}*/
 300 // CreateDirectory - poor man's mkdir -p guarded by a parent directory  /*{{{*/
 301 // ---------------------------------------------------------------------
 302 /* This method will create all directories needed for path in good old
 303    mkdir -p style but refuses to do this if Parent is not a prefix of
 304    this Path. Example: /var/cache/ and /var/cache/apt/archives are given,
 305    so it will create apt/archives if /var/cache exists - on the other
 306    hand if the parent is /var/lib the creation will fail as this path
 307    is not a parent of the path to be generated. */
 308 bool CreateDirectory(string const &Parent, string const &Path)
 309 {
 310    if (Parent.empty() == true || Path.empty() == true)
 311       return false;
 312
 313    if (DirectoryExists(Path) == true)
 314       return true;
 315
 316    if (DirectoryExists(Parent) == false)
 317       return false;
 318
 319    // we are not going to create directories "into the blue"
 320    if (Path.find(Parent, 0) != 0)
 321       return false;
 322
 323    vector<string> const dirs = VectorizeString(Path.substr(Parent.size()), '/');
 324    string progress = Parent;
 325    for (vector<string>::const_iterator d = dirs.begin(); d != dirs.end(); ++d)
 326    {
 327       if (d->empty() == true)
 328          continue;
 329
 330       progress.append("/").append(*d);
 331       if (DirectoryExists(progress) == true)
 332          continue;
 333
 334       if (mkdir(progress.c_str(), 0755) != 0)
 335          return false;
 336    }
 337    return true;
 338 }
 339                                                                         /*}}}*/
 340 // CreateAPTDirectoryIfNeeded - ensure that the given directory exists          /*{{{*/
 341 // ---------------------------------------------------------------------
 342 /* a small wrapper around CreateDirectory to check if it exists and to
 343    remove the trailing "/apt/" from the parent directory if needed */
 344 bool CreateAPTDirectoryIfNeeded(string const &Parent, string const &Path)
 345 {
 346    if (DirectoryExists(Path) == true)
 347       return true;
 348
 349    size_t const len = Parent.size();
 350    if (len > 5 && Parent.find("/apt/", len - 6, 5) == len - 5)
 351    {
 352       if (CreateDirectory(Parent.substr(0,len-5), Path) == true)
 353          return true;
 354    }
 355    else if (CreateDirectory(Parent, Path) == true)
 356       return true;
 357
 358    return false;
 359 }
 360                                                                         /*}}}*/
 361 // GetListOfFilesInDir - returns a vector of files in the given dir     /*{{{*/
 362 // ---------------------------------------------------------------------
 363 /* If an extension is given only files with this extension are included
 364    in the returned vector, otherwise every "normal" file is included. */
 365 std::vector<string> GetListOfFilesInDir(string const &Dir, string const &Ext,
 366                                         bool const &SortList, bool const &AllowNoExt)
 367 {
 368    std::vector<string> ext;
 369    ext.reserve(2);
 370    if (Ext.empty() == false)
 371       ext.push_back(Ext);
 372    if (AllowNoExt == true && ext.empty() == false)
 373       ext.push_back("");
 374    return GetListOfFilesInDir(Dir, ext, SortList);
 375 }
 376 std::vector<string> GetListOfFilesInDir(string const &Dir, std::vector<string> const &Ext,
 377                                         bool const &SortList)
 378 {
 379    // Attention debuggers: need to be set with the environment config file!
 380    bool const Debug = _config->FindB("Debug::GetListOfFilesInDir", false);
 381    if (Debug == true)
 382    {
 383       std::clog << "Accept in " << Dir << " only files with the following " << Ext.size() << " extensions:" << std::endl;
 384       if (Ext.empty() == true)
 385          std::clog << "\tNO extension" << std::endl;
 386       else
 387          for (std::vector<string>::const_iterator e = Ext.begin();
 388               e != Ext.end(); ++e)
 389             std::clog << '\t' << (e->empty() == true ? "NO" : *e) << " extension" << std::endl;
 390    }
 391
 392    std::vector<string> List;
 393
 394    if (DirectoryExists(Dir) == false)
 395    {
 396       _error->Error(_("List of files can't be created as '%s' is not a directory"), Dir.c_str());
 397       return List;
 398    }
 399
 400    Configuration::MatchAgainstConfig SilentIgnore("Dir::Ignore-Files-Silently");
 401    DIR *D = opendir(Dir.c_str());
 402    if (D == 0)
 403    {
 404       _error->Errno("opendir",_("Unable to read %s"),Dir.c_str());
 405       return List;
 406    }
 407
 408    for (struct dirent *Ent = readdir(D); Ent != 0; Ent = readdir(D))
 409    {
 410       // skip "hidden" files
 411       if (Ent->d_name[0] == '.')
 412          continue;
 413
 414       // Make sure it is a file and not something else
 415       string const File = flCombine(Dir,Ent->d_name);
 416 #ifdef _DIRENT_HAVE_D_TYPE
 417       if (Ent->d_type != DT_REG)
 418 #endif
 419       {
 420          if (RealFileExists(File) == false)
 421          {
 422             // do not show ignoration warnings for directories
 423             if (
 424 #ifdef _DIRENT_HAVE_D_TYPE
 425                 Ent->d_type == DT_DIR ||
 426 #endif
 427                 DirectoryExists(File) == true)
 428                continue;
 429             if (SilentIgnore.Match(Ent->d_name) == false)
 430                _error->Notice(_("Ignoring '%s' in directory '%s' as it is not a regular file"), Ent->d_name, Dir.c_str());
 431             continue;
 432          }
 433       }
 434
 435       // check for accepted extension:
 436       // no extension given -> periods are bad as hell!
 437       // extensions given -> "" extension allows no extension
 438       if (Ext.empty() == false)
 439       {
 440          string d_ext = flExtension(Ent->d_name);
 441          if (d_ext == Ent->d_name) // no extension
 442          {
 443             if (std::find(Ext.begin(), Ext.end(), "") == Ext.end())
 444             {
 445                if (Debug == true)
 446                   std::clog << "Bad file: " << Ent->d_name << " → no extension" << std::endl;
 447                if (SilentIgnore.Match(Ent->d_name) == false)
 448                   _error->Notice(_("Ignoring file '%s' in directory '%s' as it has no filename extension"), Ent->d_name, Dir.c_str());
 449                continue;
 450             }
 451          }
 452          else if (std::find(Ext.begin(), Ext.end(), d_ext) == Ext.end())
 453          {
 454             if (Debug == true)
 455                std::clog << "Bad file: " << Ent->d_name << " → bad extension »" << flExtension(Ent->d_name) << "«" << std::endl;
 456             if (SilentIgnore.Match(Ent->d_name) == false)
 457                _error->Notice(_("Ignoring file '%s' in directory '%s' as it has an invalid filename extension"), Ent->d_name, Dir.c_str());
 458             continue;
 459          }
 460       }
 461
 462       // Skip bad filenames ala run-parts
 463       const char *C = Ent->d_name;
 464       for (; *C != 0; ++C)
 465          if (isalpha(*C) == 0 && isdigit(*C) == 0
 466              && *C != '_' && *C != '-') {
 467             // no required extension -> dot is a bad character
 468             if (*C == '.' && Ext.empty() == false)
 469                continue;
 470             break;
 471          }
 472
 473       // we don't reach the end of the name -> bad character included
 474       if (*C != 0)
 475       {
 476          if (Debug == true)
 477             std::clog << "Bad file: " << Ent->d_name << " → bad character »"
 478                << *C << "« in filename (period allowed: " << (Ext.empty() ? "no" : "yes") << ")" << std::endl;
 479          continue;
 480       }
 481
 482       // skip filenames which end with a period. These are never valid
 483       if (*(C - 1) == '.')
 484       {
 485          if (Debug == true)
 486             std::clog << "Bad file: " << Ent->d_name << " → Period as last character" << std::endl;
 487          continue;
 488       }
 489
 490       if (Debug == true)
 491          std::clog << "Accept file: " << Ent->d_name << " in " << Dir << std::endl;
 492       List.push_back(File);
 493    }
 494    closedir(D);
 495
 496    if (SortList == true)
 497       std::sort(List.begin(),List.end());
 498    return List;
 499 }
 500 std::vector<string> GetListOfFilesInDir(string const &Dir, bool SortList)
 501 {
 502    bool const Debug = _config->FindB("Debug::GetListOfFilesInDir", false);
 503    if (Debug == true)
 504       std::clog << "Accept in " << Dir << " all regular files" << std::endl;
 505
 506    std::vector<string> List;
 507
 508    if (DirectoryExists(Dir) == false)
 509    {
 510       _error->Error(_("List of files can't be created as '%s' is not a directory"), Dir.c_str());
 511       return List;
 512    }
 513
 514    DIR *D = opendir(Dir.c_str());
 515    if (D == 0)
 516    {
 517       _error->Errno("opendir",_("Unable to read %s"),Dir.c_str());
 518       return List;
 519    }
 520
 521    for (struct dirent *Ent = readdir(D); Ent != 0; Ent = readdir(D))
 522    {
 523       // skip "hidden" files
 524       if (Ent->d_name[0] == '.')
 525          continue;
 526
 527       // Make sure it is a file and not something else
 528       string const File = flCombine(Dir,Ent->d_name);
 529 #ifdef _DIRENT_HAVE_D_TYPE
 530       if (Ent->d_type != DT_REG)
 531 #endif
 532       {
 533          if (RealFileExists(File) == false)
 534          {
 535             if (Debug == true)
 536                std::clog << "Bad file: " << Ent->d_name << " → it is not a real file" << std::endl;
 537             continue;
 538          }
 539       }
 540
 541       // Skip bad filenames ala run-parts
 542       const char *C = Ent->d_name;
 543       for (; *C != 0; ++C)
 544          if (isalpha(*C) == 0 && isdigit(*C) == 0
 545              && *C != '_' && *C != '-' && *C != '.')
 546             break;
 547
 548       // we don't reach the end of the name -> bad character included
 549       if (*C != 0)
 550       {
 551          if (Debug == true)
 552             std::clog << "Bad file: " << Ent->d_name << " → bad character »" << *C << "« in filename" << std::endl;
 553          continue;
 554       }
 555
 556       // skip filenames which end with a period. These are never valid
 557       if (*(C - 1) == '.')
 558       {
 559          if (Debug == true)
 560             std::clog << "Bad file: " << Ent->d_name << " → Period as last character" << std::endl;
 561          continue;
 562       }
 563
 564       if (Debug == true)
 565          std::clog << "Accept file: " << Ent->d_name << " in " << Dir << std::endl;
 566       List.push_back(File);
 567    }
 568    closedir(D);
 569
 570    if (SortList == true)
 571       std::sort(List.begin(),List.end());
 572    return List;
 573 }
 574                                                                         /*}}}*/
 575 // SafeGetCWD - This is a safer getcwd that returns a dynamic string    /*{{{*/
 576 // ---------------------------------------------------------------------
 577 /* We return / on failure. */
 578 string SafeGetCWD()
 579 {
 580    // Stash the current dir.
 581    char S[300];
 582    S[0] = 0;
 583    if (getcwd(S,sizeof(S)-2) == 0)
 584       return "/";
 585    unsigned int Len = strlen(S);
 586    S[Len] = '/';
 587    S[Len+1] = 0;
 588    return S;
 589 }
 590                                                                         /*}}}*/
 591 // GetModificationTime - Get the mtime of the given file or -1 on error /*{{{*/
 592 // ---------------------------------------------------------------------
 593 /* We return / on failure. */
 594 time_t GetModificationTime(string const &Path)
 595 {
 596    struct stat St;
 597    if (stat(Path.c_str(), &St) < 0)
 598       return -1;
 599    return  St.st_mtime;
 600 }
 601                                                                         /*}}}*/
 602 // flNotDir - Strip the directory from the filename                     /*{{{*/
 603 // ---------------------------------------------------------------------
 604 /* */
 605 string flNotDir(string File)
 606 {
 607    string::size_type Res = File.rfind('/');
 608    if (Res == string::npos)
 609       return File;
 610    Res++;
 611    return string(File,Res,Res - File.length());
 612 }
 613                                                                         /*}}}*/
 614 // flNotFile - Strip the file from the directory name                   /*{{{*/
 615 // ---------------------------------------------------------------------
 616 /* Result ends in a / */
 617 string flNotFile(string File)
 618 {
 619    string::size_type Res = File.rfind('/');
 620    if (Res == string::npos)
 621       return "./";
 622    Res++;
 623    return string(File,0,Res);
 624 }
 625                                                                         /*}}}*/
 626 // flExtension - Return the extension for the file                      /*{{{*/
 627 // ---------------------------------------------------------------------
 628 /* */
 629 string flExtension(string File)
 630 {
 631    string::size_type Res = File.rfind('.');
 632    if (Res == string::npos)
 633       return File;
 634    Res++;
 635    return string(File,Res,Res - File.length());
 636 }
 637                                                                         /*}}}*/
 638 // flNoLink - If file is a symlink then deref it                        /*{{{*/
 639 // ---------------------------------------------------------------------
 640 /* If the name is not a link then the returned path is the input. */
 641 string flNoLink(string File)
 642 {
 643    struct stat St;
 644    if (lstat(File.c_str(),&St) != 0 || S_ISLNK(St.st_mode) == 0)
 645       return File;
 646    if (stat(File.c_str(),&St) != 0)
 647       return File;
 648
 649    /* Loop resolving the link. There is no need to limit the number of
 650       loops because the stat call above ensures that the symlink is not
 651       circular */
 652    char Buffer[1024];
 653    string NFile = File;
 654    while (1)
 655    {
 656       // Read the link
 657       int Res;
 658       if ((Res = readlink(NFile.c_str(),Buffer,sizeof(Buffer))) <= 0 ||
 659           (unsigned)Res >= sizeof(Buffer))
 660           return File;
 661
 662       // Append or replace the previous path
 663       Buffer[Res] = 0;
 664       if (Buffer[0] == '/')
 665          NFile = Buffer;
 666       else
 667          NFile = flNotFile(NFile) + Buffer;
 668
 669       // See if we are done
 670       if (lstat(NFile.c_str(),&St) != 0)
 671          return File;
 672       if (S_ISLNK(St.st_mode) == 0)
 673          return NFile;
 674    }
 675 }
 676                                                                         /*}}}*/
 677 // flCombine - Combine a file and a directory                           /*{{{*/
 678 // ---------------------------------------------------------------------
 679 /* If the file is an absolute path then it is just returned, otherwise
 680    the directory is pre-pended to it. */
 681 string flCombine(string Dir,string File)
 682 {
 683    if (File.empty() == true)
 684       return string();
 685
 686    if (File[0] == '/' || Dir.empty() == true)
 687       return File;
 688    if (File.length() >= 2 && File[0] == '.' && File[1] == '/')
 689       return File;
 690    if (Dir[Dir.length()-1] == '/')
 691       return Dir + File;
 692    return Dir + '/' + File;
 693 }
 694                                                                         /*}}}*/
 695 // SetCloseExec - Set the close on exec flag                            /*{{{*/
 696 // ---------------------------------------------------------------------
 697 /* */
 698 void SetCloseExec(int Fd,bool Close)
 699 {
 700    if (fcntl(Fd,F_SETFD,(Close == false)?0:FD_CLOEXEC) != 0)
 701    {
 702       cerr << "FATAL -> Could not set close on exec " << strerror(errno) << endl;
 703       exit(100);
 704    }
 705 }
 706                                                                         /*}}}*/
 707 // SetNonBlock - Set the nonblocking flag                               /*{{{*/
 708 // ---------------------------------------------------------------------
 709 /* */
 710 void SetNonBlock(int Fd,bool Block)
 711 {
 712    int Flags = fcntl(Fd,F_GETFL) & (~O_NONBLOCK);
 713    if (fcntl(Fd,F_SETFL,Flags | ((Block == false)?0:O_NONBLOCK)) != 0)
 714    {
 715       cerr << "FATAL -> Could not set non-blocking flag " << strerror(errno) << endl;
 716       exit(100);
 717    }
 718 }
 719                                                                         /*}}}*/
 720 // WaitFd - Wait for a FD to become readable                            /*{{{*/
 721 // ---------------------------------------------------------------------
 722 /* This waits for a FD to become readable using select. It is useful for
 723    applications making use of non-blocking sockets. The timeout is
 724    in seconds. */
 725 bool WaitFd(int Fd,bool write,unsigned long timeout)
 726 {
 727    fd_set Set;
 728    struct timeval tv;
 729    FD_ZERO(&Set);
 730    FD_SET(Fd,&Set);
 731    tv.tv_sec = timeout;
 732    tv.tv_usec = 0;
 733    if (write == true)
 734    {
 735       int Res;
 736       do
 737       {
 738          Res = select(Fd+1,0,&Set,0,(timeout != 0?&tv:0));
 739       }
 740       while (Res < 0 && errno == EINTR);
 741
 742       if (Res <= 0)
 743          return false;
 744    }
 745    else
 746    {
 747       int Res;
 748       do
 749       {
 750          Res = select(Fd+1,&Set,0,0,(timeout != 0?&tv:0));
 751       }
 752       while (Res < 0 && errno == EINTR);
 753
 754       if (Res <= 0)
 755          return false;
 756    }
 757
 758    return true;
 759 }
 760                                                                         /*}}}*/
 761 // ExecFork - Magical fork that sanitizes the context before execing    /*{{{*/
 762 // ---------------------------------------------------------------------
 763 /* This is used if you want to cleanse the environment for the forked
 764    child, it fixes up the important signals and nukes all of the fds,
 765    otherwise acts like normal fork. */
 766 pid_t ExecFork()
 767 {
 768    // Fork off the process
 769    pid_t Process = fork();
 770    if (Process < 0)
 771    {
 772       cerr << "FATAL -> Failed to fork." << endl;
 773       exit(100);
 774    }
 775
 776    // Spawn the subprocess
 777    if (Process == 0)
 778    {
 779       // Setup the signals
 780       signal(SIGPIPE,SIG_DFL);
 781       signal(SIGQUIT,SIG_DFL);
 782       signal(SIGINT,SIG_DFL);
 783       signal(SIGWINCH,SIG_DFL);
 784       signal(SIGCONT,SIG_DFL);
 785       signal(SIGTSTP,SIG_DFL);
 786
 787       set<int> KeepFDs;
 788       Configuration::Item const *Opts = _config->Tree("APT::Keep-Fds");
 789       if (Opts != 0 && Opts->Child != 0)
 790       {
 791          Opts = Opts->Child;
 792          for (; Opts != 0; Opts = Opts->Next)
 793          {
 794             if (Opts->Value.empty() == true)
 795                continue;
 796             int fd = atoi(Opts->Value.c_str());
 797             KeepFDs.insert(fd);
 798          }
 799       }
 800
 801       // Close all of our FDs - just in case
 802       for (int K = 3; K != 40; K++)
 803       {
 804          if(KeepFDs.find(K) == KeepFDs.end())
 805             fcntl(K,F_SETFD,FD_CLOEXEC);
 806       }
 807    }
 808
 809    return Process;
 810 }
 811                                                                         /*}}}*/
 812 // ExecWait - Fancy waitpid                                             /*{{{*/
 813 // ---------------------------------------------------------------------
 814 /* Waits for the given sub process. If Reap is set then no errors are
 815    generated. Otherwise a failed subprocess will generate a proper descriptive
 816    message */
 817 bool ExecWait(pid_t Pid,const char *Name,bool Reap)
 818 {
 819    if (Pid <= 1)
 820       return true;
 821
 822    // Wait and collect the error code
 823    int Status;
 824    while (waitpid(Pid,&Status,0) != Pid)
 825    {
 826       if (errno == EINTR)
 827          continue;
 828
 829       if (Reap == true)
 830          return false;
 831
 832       return _error->Error(_("Waited for %s but it wasn't there"),Name);
 833    }
 834
 835
 836    // Check for an error code.
 837    if (WIFEXITED(Status) == 0 || WEXITSTATUS(Status) != 0)
 838    {
 839       if (Reap == true)
 840          return false;
 841       if (WIFSIGNALED(Status) != 0)
 842       {
 843          if( WTERMSIG(Status) == SIGSEGV)
 844             return _error->Error(_("Sub-process %s received a segmentation fault."),Name);
 845          else
 846             return _error->Error(_("Sub-process %s received signal %u."),Name, WTERMSIG(Status));
 847       }
 848
 849       if (WIFEXITED(Status) != 0)
 850          return _error->Error(_("Sub-process %s returned an error code (%u)"),Name,WEXITSTATUS(Status));
 851
 852       return _error->Error(_("Sub-process %s exited unexpectedly"),Name);
 853    }
 854
 855    return true;
 856 }
 857                                                                         /*}}}*/
 858
 859 // FileFd::Open - Open a file                                           /*{{{*/
 860 // ---------------------------------------------------------------------
 861 /* The most commonly used open mode combinations are given with Mode */
 862 bool FileFd::Open(string FileName,unsigned int const Mode,CompressMode Compress, unsigned long const Perms)
 863 {
 864    if (Mode == ReadOnlyGzip)
 865       return Open(FileName, ReadOnly, Gzip, Perms);
 866
 867    if (Compress == Auto && (Mode & WriteOnly) == WriteOnly)
 868       return FileFdError("Autodetection on %s only works in ReadOnly openmode!", FileName.c_str());
 869
 870    std::vector<APT::Configuration::Compressor> const compressors = APT::Configuration::getCompressors();
 871    std::vector<APT::Configuration::Compressor>::const_iterator compressor = compressors.begin();
 872    if (Compress == Auto)
 873    {
 874       for (; compressor != compressors.end(); ++compressor)
 875       {
 876          std::string file = std::string(FileName).append(compressor->Extension);
 877          if (FileExists(file) == false)
 878             continue;
 879          FileName = file;
 880          break;
 881       }
 882    }
 883    else if (Compress == Extension)
 884    {
 885       std::string::size_type const found = FileName.find_last_of('.');
 886       std::string ext;
 887       if (found != std::string::npos)
 888       {
 889          ext = FileName.substr(found);
 890          if (ext == ".new" || ext == ".bak")
 891          {
 892             std::string::size_type const found2 = FileName.find_last_of('.', found - 1);
 893             if (found2 != std::string::npos)
 894                ext = FileName.substr(found2, found - found2);
 895             else
 896                ext.clear();
 897          }
 898       }
 899       for (; compressor != compressors.end(); ++compressor)
 900          if (ext == compressor->Extension)
 901             break;
 902       // no matching extension - assume uncompressed (imagine files like 'example.org_Packages')
 903       if (compressor == compressors.end())
 904          for (compressor = compressors.begin(); compressor != compressors.end(); ++compressor)
 905             if (compressor->Name == ".")
 906                break;
 907    }
 908    else
 909    {
 910       std::string name;
 911       switch (Compress)
 912       {
 913       case None: name = "."; break;
 914       case Gzip: name = "gzip"; break;
 915       case Bzip2: name = "bzip2"; break;
 916       case Lzma: name = "lzma"; break;
 917       case Xz: name = "xz"; break;
 918       case Auto:
 919       case Extension:
 920          // Unreachable
 921          return FileFdError("Opening File %s in None, Auto or Extension should be already handled?!?", FileName.c_str());
 922       }
 923       for (; compressor != compressors.end(); ++compressor)
 924          if (compressor->Name == name)
 925             break;
 926       if (compressor == compressors.end())
 927          return FileFdError("Can't find a configured compressor %s for file %s", name.c_str(), FileName.c_str());
 928    }
 929
 930    if (compressor == compressors.end())
 931       return FileFdError("Can't find a match for specified compressor mode for file %s", FileName.c_str());
 932    return Open(FileName, Mode, *compressor, Perms);
 933 }
 934 bool FileFd::Open(string FileName,unsigned int const Mode,APT::Configuration::Compressor const &compressor, unsigned long const Perms)
 935 {
 936    Close();
 937    Flags = AutoClose;
 938
 939    if ((Mode & WriteOnly) != WriteOnly && (Mode & (Atomic | Create | Empty | Exclusive)) != 0)
 940       return FileFdError("ReadOnly mode for %s doesn't accept additional flags!", FileName.c_str());
 941    if ((Mode & ReadWrite) == 0)
 942       return FileFdError("No openmode provided in FileFd::Open for %s", FileName.c_str());
 943
 944    if ((Mode & Atomic) == Atomic)
 945    {
 946       Flags |= Replace;
 947       char *name = strdup((FileName + ".XXXXXX").c_str());
 948       TemporaryFileName = string(mktemp(name));
 949       free(name);
 950    }
 951    else if ((Mode & (Exclusive | Create)) == (Exclusive | Create))
 952    {
 953       // for atomic, this will be done by rename in Close()
 954       unlink(FileName.c_str());
 955    }
 956    if ((Mode & Empty) == Empty)
 957    {
 958       struct stat Buf;
 959       if (lstat(FileName.c_str(),&Buf) == 0 && S_ISLNK(Buf.st_mode))
 960          unlink(FileName.c_str());
 961    }
 962
 963    int fileflags = 0;
 964    #define if_FLAGGED_SET(FLAG, MODE) if ((Mode & FLAG) == FLAG) fileflags |= MODE
 965    if_FLAGGED_SET(ReadWrite, O_RDWR);
 966    else if_FLAGGED_SET(ReadOnly, O_RDONLY);
 967    else if_FLAGGED_SET(WriteOnly, O_WRONLY);
 968
 969    if_FLAGGED_SET(Create, O_CREAT);
 970    if_FLAGGED_SET(Empty, O_TRUNC);
 971    if_FLAGGED_SET(Exclusive, O_EXCL);
 972    else if_FLAGGED_SET(Atomic, O_EXCL);
 973    #undef if_FLAGGED_SET
 974
 975    if (TemporaryFileName.empty() == false)
 976       iFd = open(TemporaryFileName.c_str(), fileflags, Perms);
 977    else
 978       iFd = open(FileName.c_str(), fileflags, Perms);
 979
 980    this->FileName = FileName;
 981    if (iFd == -1 || OpenInternDescriptor(Mode, compressor) == false)
 982    {
 983       if (iFd != -1)
 984       {
 985          close (iFd);
 986          iFd = -1;
 987       }
 988       return FileFdErrno("open",_("Could not open file %s"), FileName.c_str());
 989    }
 990
 991    SetCloseExec(iFd,true);
 992    return true;
 993 }
 994                                                                         /*}}}*/
 995 // FileFd::OpenDescriptor - Open a filedescriptor                       /*{{{*/
 996 // ---------------------------------------------------------------------
 997 /* */
 998 bool FileFd::OpenDescriptor(int Fd, unsigned int const Mode, CompressMode Compress, bool AutoClose)
 999 {
1000    std::vector<APT::Configuration::Compressor> const compressors = APT::Configuration::getCompressors();
1001    std::vector<APT::Configuration::Compressor>::const_iterator compressor = compressors.begin();
1002    std::string name;
1003
1004    // compat with the old API
1005    if (Mode == ReadOnlyGzip && Compress == None)
1006       Compress = Gzip;
1007
1008    switch (Compress)
1009    {
1010    case None: name = "."; break;
1011    case Gzip: name = "gzip"; break;
1012    case Bzip2: name = "bzip2"; break;
1013    case Lzma: name = "lzma"; break;
1014    case Xz: name = "xz"; break;
1015    case Auto:
1016    case Extension:
1017       if (AutoClose == true && Fd != -1)
1018          close(Fd);
1019       return FileFdError("Opening Fd %d in Auto or Extension compression mode is not supported", Fd);
1020    }
1021    for (; compressor != compressors.end(); ++compressor)
1022       if (compressor->Name == name)
1023          break;
1024    if (compressor == compressors.end())
1025    {
1026       if (AutoClose == true && Fd != -1)
1027          close(Fd);
1028       return FileFdError("Can't find a configured compressor %s for file %s", name.c_str(), FileName.c_str());
1029    }
1030    return OpenDescriptor(Fd, Mode, *compressor, AutoClose);
1031 }
1032 bool FileFd::OpenDescriptor(int Fd, unsigned int const Mode, APT::Configuration::Compressor const &compressor, bool AutoClose)
1033 {
1034    Close();
1035    Flags = (AutoClose) ? FileFd::AutoClose : 0;
1036    if (AutoClose == false && (
1037 #ifdef HAVE_ZLIB
1038         compressor.Name == "gzip" ||
1039 #endif
1040 #ifdef HAVE_BZ2
1041         compressor.Name == "bzip2" ||
1042 #endif
1043         false))
1044    {
1045       // Need to duplicate fd here or gzclose for cleanup will close the fd as well
1046       iFd = dup(Fd);
1047    }
1048    else
1049       iFd = Fd;
1050    this->FileName = "";
1051    if (Fd == -1 || OpenInternDescriptor(Mode, compressor) == false)
1052    {
1053       if (iFd != -1 && (
1054 #ifdef HAVE_ZLIB
1055         compressor.Name == "gzip" ||
1056 #endif
1057 #ifdef HAVE_BZ2
1058         compressor.Name == "bzip2" ||
1059 #endif
1060         AutoClose == true))
1061       {
1062          close (iFd);
1063          iFd = -1;
1064       }
1065       return FileFdError(_("Could not open file descriptor %d"), Fd);
1066    }
1067    return true;
1068 }
1069 bool FileFd::OpenInternDescriptor(unsigned int const Mode, APT::Configuration::Compressor const &compressor)
1070 {
1071    if (compressor.Name == "." || compressor.Binary.empty() == true)
1072       return true;
1073
1074    if (d == NULL)
1075    {
1076       d = new FileFdPrivate();
1077       d->openmode = Mode;
1078       d->compressor = compressor;
1079    }
1080
1081 #ifdef HAVE_ZLIB
1082    if (compressor.Name == "gzip")
1083    {
1084       if (d->gz != NULL)
1085       {
1086          gzclose(d->gz);
1087          d->gz = NULL;
1088       }
1089       if ((Mode & ReadWrite) == ReadWrite)
1090          d->gz = gzdopen(iFd, "r+");
1091       else if ((Mode & WriteOnly) == WriteOnly)
1092          d->gz = gzdopen(iFd, "w");
1093       else
1094          d->gz = gzdopen(iFd, "r");
1095       if (d->gz == NULL)
1096          return false;
1097       Flags |= Compressed;
1098       return true;
1099    }
1100 #endif
1101 #ifdef HAVE_BZ2
1102    if (compressor.Name == "bzip2")
1103    {
1104       if (d->bz2 != NULL)
1105       {
1106          BZ2_bzclose(d->bz2);
1107          d->bz2 = NULL;
1108       }
1109       if ((Mode & ReadWrite) == ReadWrite)
1110          d->bz2 = BZ2_bzdopen(iFd, "r+");
1111       else if ((Mode & WriteOnly) == WriteOnly)
1112          d->bz2 = BZ2_bzdopen(iFd, "w");
1113       else
1114          d->bz2 = BZ2_bzdopen(iFd, "r");
1115       if (d->bz2 == NULL)
1116          return false;
1117       Flags |= Compressed;
1118       return true;
1119    }
1120 #endif
1121
1122    // collect zombies here in case we reopen
1123    if (d->compressor_pid > 0)
1124       ExecWait(d->compressor_pid, "FileFdCompressor", true);
1125
1126    if ((Mode & ReadWrite) == ReadWrite)
1127       return FileFdError("ReadWrite mode is not supported for file %s", FileName.c_str());
1128
1129    bool const Comp = (Mode & WriteOnly) == WriteOnly;
1130    if (Comp == false)
1131    {
1132       // Handle 'decompression' of empty files
1133       struct stat Buf;
1134       fstat(iFd, &Buf);
1135       if (Buf.st_size == 0 && S_ISFIFO(Buf.st_mode) == false)
1136          return true;
1137
1138       // We don't need the file open - instead let the compressor open it
1139       // as he properly knows better how to efficiently read from 'his' file
1140       if (FileName.empty() == false)
1141       {
1142          close(iFd);
1143          iFd = -1;
1144       }
1145    }
1146
1147    // Create a data pipe
1148    int Pipe[2] = {-1,-1};
1149    if (pipe(Pipe) != 0)
1150       return FileFdErrno("pipe",_("Failed to create subprocess IPC"));
1151    for (int J = 0; J != 2; J++)
1152       SetCloseExec(Pipe[J],true);
1153
1154    d->compressed_fd = iFd;
1155    d->pipe = true;
1156
1157    if (Comp == true)
1158       iFd = Pipe[1];
1159    else
1160       iFd = Pipe[0];
1161
1162    // The child..
1163    d->compressor_pid = ExecFork();
1164    if (d->compressor_pid == 0)
1165    {
1166       if (Comp == true)
1167       {
1168          dup2(d->compressed_fd,STDOUT_FILENO);
1169          dup2(Pipe[0],STDIN_FILENO);
1170       }
1171       else
1172       {
1173          if (FileName.empty() == true)
1174             dup2(d->compressed_fd,STDIN_FILENO);
1175          dup2(Pipe[1],STDOUT_FILENO);
1176       }
1177       int const nullfd = open("/dev/null", O_WRONLY);
1178       if (nullfd != -1)
1179       {
1180          dup2(nullfd,STDERR_FILENO);
1181          close(nullfd);
1182       }
1183
1184       SetCloseExec(STDOUT_FILENO,false);
1185       SetCloseExec(STDIN_FILENO,false);
1186
1187       std::vector<char const*> Args;
1188       Args.push_back(compressor.Binary.c_str());
1189       std::vector<std::string> const * const addArgs =
1190                 (Comp == true) ? &(compressor.CompressArgs) : &(compressor.UncompressArgs);
1191       for (std::vector<std::string>::const_iterator a = addArgs->begin();
1192            a != addArgs->end(); ++a)
1193          Args.push_back(a->c_str());
1194       if (Comp == false && FileName.empty() == false)
1195       {
1196          Args.push_back("--stdout");
1197          if (TemporaryFileName.empty() == false)
1198             Args.push_back(TemporaryFileName.c_str());
1199          else
1200             Args.push_back(FileName.c_str());
1201       }
1202       Args.push_back(NULL);
1203
1204       execvp(Args[0],(char **)&Args[0]);
1205       cerr << _("Failed to exec compressor ") << Args[0] << endl;
1206       _exit(100);
1207    }
1208    if (Comp == true)
1209       close(Pipe[0]);
1210    else
1211       close(Pipe[1]);
1212
1213    return true;
1214 }
1215                                                                         /*}}}*/
1216 // FileFd::~File - Closes the file                                      /*{{{*/
1217 // ---------------------------------------------------------------------
1218 /* If the proper modes are selected then we close the Fd and possibly
1219    unlink the file on error. */
1220 FileFd::~FileFd()
1221 {
1222    Close();
1223    if (d != NULL)
1224       d->CloseDown(FileName);
1225    delete d;
1226    d = NULL;
1227 }
1228                                                                         /*}}}*/
1229 // FileFd::Read - Read a bit of the file                                /*{{{*/
1230 // ---------------------------------------------------------------------
1231 /* We are carefull to handle interruption by a signal while reading
1232    gracefully. */
1233 bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual)
1234 {
1235    int Res;
1236    errno = 0;
1237    if (Actual != 0)
1238       *Actual = 0;
1239    *((char *)To) = '\0';
1240    do
1241    {
1242 #ifdef HAVE_ZLIB
1243       if (d != NULL && d->gz != NULL)
1244          Res = gzread(d->gz,To,Size);
1245       else
1246 #endif
1247 #ifdef HAVE_BZ2
1248       if (d != NULL && d->bz2 != NULL)
1249          Res = BZ2_bzread(d->bz2,To,Size);
1250       else
1251 #endif
1252          Res = read(iFd,To,Size);
1253
1254       if (Res < 0)
1255       {
1256          if (errno == EINTR)
1257             continue;
1258 #ifdef HAVE_ZLIB
1259          if (d != NULL && d->gz != NULL)
1260          {
1261             int err;
1262             char const * const errmsg = gzerror(d->gz, &err);
1263             if (err != Z_ERRNO)
1264                return FileFdError("gzread: %s (%d: %s)", _("Read error"), err, errmsg);
1265          }
1266 #endif
1267 #ifdef HAVE_BZ2
1268          if (d != NULL && d->bz2 != NULL)
1269          {
1270             int err;
1271             char const * const errmsg = BZ2_bzerror(d->bz2, &err);
1272             if (err != BZ_IO_ERROR)
1273                return FileFdError("BZ2_bzread: %s (%d: %s)", _("Read error"), err, errmsg);
1274          }
1275 #endif
1276          return FileFdErrno("read",_("Read error"));
1277       }
1278
1279       To = (char *)To + Res;
1280       Size -= Res;
1281       if (d != NULL)
1282          d->seekpos += Res;
1283       if (Actual != 0)
1284          *Actual += Res;
1285    }
1286    while (Res > 0 && Size > 0);
1287
1288    if (Size == 0)
1289       return true;
1290
1291    // Eof handling
1292    if (Actual != 0)
1293    {
1294       Flags |= HitEof;
1295       return true;
1296    }
1297
1298    return FileFdError(_("read, still have %llu to read but none left"), Size);
1299 }
1300                                                                         /*}}}*/
1301 // FileFd::ReadLine - Read a complete line from the file                /*{{{*/
1302 // ---------------------------------------------------------------------
1303 /* Beware: This method can be quiet slow for big buffers on UNcompressed
1304    files because of the naive implementation! */
1305 char* FileFd::ReadLine(char *To, unsigned long long const Size)
1306 {
1307    *To = '\0';
1308 #ifdef HAVE_ZLIB
1309    if (d != NULL && d->gz != NULL)
1310       return gzgets(d->gz, To, Size);
1311 #endif
1312
1313    unsigned long long read = 0;
1314    while ((Size - 1) != read)
1315    {
1316       unsigned long long done = 0;
1317       if (Read(To + read, 1, &done) == false)
1318          return NULL;
1319       if (done == 0)
1320          break;
1321       if (To[read++] == '\n')
1322          break;
1323    }
1324    if (read == 0)
1325       return NULL;
1326    To[read] = '\0';
1327    return To;
1328 }
1329                                                                         /*}}}*/
1330 // FileFd::Write - Write to the file                                    /*{{{*/
1331 // ---------------------------------------------------------------------
1332 /* */
1333 bool FileFd::Write(const void *From,unsigned long long Size)
1334 {
1335    int Res;
1336    errno = 0;
1337    do
1338    {
1339 #ifdef HAVE_ZLIB
1340       if (d != NULL && d->gz != NULL)
1341          Res = gzwrite(d->gz,From,Size);
1342       else
1343 #endif
1344 #ifdef HAVE_BZ2
1345       if (d != NULL && d->bz2 != NULL)
1346          Res = BZ2_bzwrite(d->bz2,(void*)From,Size);
1347       else
1348 #endif
1349          Res = write(iFd,From,Size);
1350       if (Res < 0 && errno == EINTR)
1351          continue;
1352       if (Res < 0)
1353       {
1354 #ifdef HAVE_ZLIB
1355          if (d != NULL && d->gz != NULL)
1356          {
1357             int err;
1358             char const * const errmsg = gzerror(d->gz, &err);
1359             if (err != Z_ERRNO)
1360                return FileFdError("gzwrite: %s (%d: %s)", _("Write error"), err, errmsg);
1361          }
1362 #endif
1363 #ifdef HAVE_BZ2
1364          if (d != NULL && d->bz2 != NULL)
1365          {
1366             int err;
1367             char const * const errmsg = BZ2_bzerror(d->bz2, &err);
1368             if (err != BZ_IO_ERROR)
1369                return FileFdError("BZ2_bzwrite: %s (%d: %s)", _("Write error"), err, errmsg);
1370          }
1371 #endif
1372          return FileFdErrno("write",_("Write error"));
1373       }
1374
1375       From = (char *)From + Res;
1376       Size -= Res;
1377       if (d != NULL)
1378          d->seekpos += Res;
1379    }
1380    while (Res > 0 && Size > 0);
1381
1382    if (Size == 0)
1383       return true;
1384
1385    return FileFdError(_("write, still have %llu to write but couldn't"), Size);
1386 }
1387 bool FileFd::Write(int Fd, const void *From, unsigned long long Size)
1388 {
1389    int Res;
1390    errno = 0;
1391    do
1392    {
1393       Res = write(Fd,From,Size);
1394       if (Res < 0 && errno == EINTR)
1395          continue;
1396       if (Res < 0)
1397          return _error->Errno("write",_("Write error"));
1398
1399       From = (char *)From + Res;
1400       Size -= Res;
1401    }
1402    while (Res > 0 && Size > 0);
1403
1404    if (Size == 0)
1405       return true;
1406
1407    return _error->Error(_("write, still have %llu to write but couldn't"), Size);
1408 }
1409                                                                         /*}}}*/
1410 // FileFd::Seek - Seek in the file                                      /*{{{*/
1411 // ---------------------------------------------------------------------
1412 /* */
1413 bool FileFd::Seek(unsigned long long To)
1414 {
1415    if (d != NULL && (d->pipe == true
1416 #ifdef HAVE_BZ2
1417                         || d->bz2 != NULL
1418 #endif
1419         ))
1420    {
1421       // Our poor man seeking in pipes is costly, so try to avoid it
1422       unsigned long long seekpos = Tell();
1423       if (seekpos == To)
1424          return true;
1425       else if (seekpos < To)
1426          return Skip(To - seekpos);
1427
1428       if ((d->openmode & ReadOnly) != ReadOnly)
1429          return FileFdError("Reopen is only implemented for read-only files!");
1430 #ifdef HAVE_BZ2
1431      if (d->bz2 != NULL)
1432      {
1433         BZ2_bzclose(d->bz2);
1434         d->bz2 = NULL;
1435      }
1436 #endif
1437       if (iFd != -1)
1438          close(iFd);
1439       iFd = -1;
1440       if (TemporaryFileName.empty() == false)
1441          iFd = open(TemporaryFileName.c_str(), O_RDONLY);
1442       else if (FileName.empty() == false)
1443          iFd = open(FileName.c_str(), O_RDONLY);
1444       else
1445       {
1446          if (d->compressed_fd > 0)
1447             if (lseek(d->compressed_fd, 0, SEEK_SET) != 0)
1448                iFd = d->compressed_fd;
1449          if (iFd < 0)
1450             return FileFdError("Reopen is not implemented for pipes opened with FileFd::OpenDescriptor()!");
1451       }
1452
1453       if (OpenInternDescriptor(d->openmode, d->compressor) == false)
1454          return FileFdError("Seek on file %s because it couldn't be reopened", FileName.c_str());
1455
1456       if (To != 0)
1457          return Skip(To);
1458
1459       d->seekpos = To;
1460       return true;
1461    }
1462    int res;
1463 #ifdef HAVE_ZLIB
1464    if (d != NULL && d->gz)
1465       res = gzseek(d->gz,To,SEEK_SET);
1466    else
1467 #endif
1468       res = lseek(iFd,To,SEEK_SET);
1469    if (res != (signed)To)
1470       return FileFdError("Unable to seek to %llu", To);
1471
1472    if (d != NULL)
1473       d->seekpos = To;
1474    return true;
1475 }
1476                                                                         /*}}}*/
1477 // FileFd::Skip - Seek in the file                                      /*{{{*/
1478 // ---------------------------------------------------------------------
1479 /* */
1480 bool FileFd::Skip(unsigned long long Over)
1481 {
1482    if (d != NULL && (d->pipe == true
1483 #ifdef HAVE_BZ2
1484                         || d->bz2 != NULL
1485 #endif
1486         ))
1487    {
1488       d->seekpos += Over;
1489       char buffer[1024];
1490       while (Over != 0)
1491       {
1492          unsigned long long toread = std::min((unsigned long long) sizeof(buffer), Over);
1493          if (Read(buffer, toread) == false)
1494             return FileFdError("Unable to seek ahead %llu",Over);
1495          Over -= toread;
1496       }
1497       return true;
1498    }
1499
1500    int res;
1501 #ifdef HAVE_ZLIB
1502    if (d != NULL && d->gz != NULL)
1503       res = gzseek(d->gz,Over,SEEK_CUR);
1504    else
1505 #endif
1506       res = lseek(iFd,Over,SEEK_CUR);
1507    if (res < 0)
1508       return FileFdError("Unable to seek ahead %llu",Over);
1509    if (d != NULL)
1510       d->seekpos = res;
1511
1512    return true;
1513 }
1514                                                                         /*}}}*/
1515 // FileFd::Truncate - Truncate the file                                 /*{{{*/
1516 // ---------------------------------------------------------------------
1517 /* */
1518 bool FileFd::Truncate(unsigned long long To)
1519 {
1520 #if defined HAVE_ZLIB || defined HAVE_BZ2
1521    if (d != NULL && (d->gz != NULL || d->bz2 != NULL))
1522       return FileFdError("Truncating compressed files is not implemented (%s)", FileName.c_str());
1523 #endif
1524    if (ftruncate(iFd,To) != 0)
1525       return FileFdError("Unable to truncate to %llu",To);
1526
1527    return true;
1528 }
1529                                                                         /*}}}*/
1530 // FileFd::Tell - Current seek position                                 /*{{{*/
1531 // ---------------------------------------------------------------------
1532 /* */
1533 unsigned long long FileFd::Tell()
1534 {
1535    // In theory, we could just return seekpos here always instead of
1536    // seeking around, but not all users of FileFd use always Seek() and co
1537    // so d->seekpos isn't always true and we can just use it as a hint if
1538    // we have nothing else, but not always as an authority…
1539    if (d != NULL && (d->pipe == true
1540 #ifdef HAVE_BZ2
1541                         || d->bz2 != NULL
1542 #endif
1543         ))
1544       return d->seekpos;
1545
1546    off_t Res;
1547 #ifdef HAVE_ZLIB
1548    if (d != NULL && d->gz != NULL)
1549      Res = gztell(d->gz);
1550    else
1551 #endif
1552      Res = lseek(iFd,0,SEEK_CUR);
1553    if (Res == (off_t)-1)
1554       FileFdErrno("lseek","Failed to determine the current file position");
1555    if (d != NULL)
1556       d->seekpos = Res;
1557    return Res;
1558 }
1559                                                                         /*}}}*/
1560 // FileFd::FileSize - Return the size of the file                       /*{{{*/
1561 // ---------------------------------------------------------------------
1562 /* */
1563 unsigned long long FileFd::FileSize()
1564 {
1565    struct stat Buf;
1566    if ((d == NULL || d->pipe == false) && fstat(iFd,&Buf) != 0)
1567       return FileFdErrno("fstat","Unable to determine the file size");
1568
1569    // for compressor pipes st_size is undefined and at 'best' zero
1570    if ((d != NULL && d->pipe == true) || S_ISFIFO(Buf.st_mode))
1571    {
1572       // we set it here, too, as we get the info here for free
1573       // in theory the Open-methods should take care of it already
1574       if (d != NULL)
1575          d->pipe = true;
1576       if (stat(FileName.c_str(), &Buf) != 0)
1577          return FileFdErrno("stat","Unable to determine the file size");
1578    }
1579
1580    return Buf.st_size;
1581 }
1582                                                                         /*}}}*/
1583 // FileFd::Size - Return the size of the content in the file            /*{{{*/
1584 // ---------------------------------------------------------------------
1585 /* */
1586 unsigned long long FileFd::Size()
1587 {
1588    unsigned long long size = FileSize();
1589
1590    // for compressor pipes st_size is undefined and at 'best' zero,
1591    // so we 'read' the content and 'seek' back - see there
1592    if (d != NULL && (d->pipe == true
1593 #ifdef HAVE_BZ2
1594                         || (d->bz2 && size > 0)
1595 #endif
1596         ))
1597    {
1598       unsigned long long const oldSeek = Tell();
1599       char ignore[1000];
1600       unsigned long long read = 0;
1601       do {
1602          Read(ignore, sizeof(ignore), &read);
1603       } while(read != 0);
1604       size = Tell();
1605       Seek(oldSeek);
1606    }
1607 #ifdef HAVE_ZLIB
1608    // only check gzsize if we are actually a gzip file, just checking for
1609    // "gz" is not sufficient as uncompressed files could be opened with
1610    // gzopen in "direct" mode as well
1611    else if (d != NULL && d->gz && !gzdirect(d->gz) && size > 0)
1612    {
1613        off_t const oldPos = lseek(iFd,0,SEEK_CUR);
1614        /* unfortunately zlib.h doesn't provide a gzsize(), so we have to do
1615         * this ourselves; the original (uncompressed) file size is the last 32
1616         * bits of the file */
1617        // FIXME: Size for gz-files is limited by 32bit… no largefile support
1618        if (lseek(iFd, -4, SEEK_END) < 0)
1619           return FileFdErrno("lseek","Unable to seek to end of gzipped file");
1620        size = 0L;
1621        if (read(iFd, &size, 4) != 4)
1622           return FileFdErrno("read","Unable to read original size of gzipped file");
1623
1624 #ifdef WORDS_BIGENDIAN
1625        uint32_t tmp_size = size;
1626        uint8_t const * const p = (uint8_t const * const) &tmp_size;
1627        tmp_size = (p[3] << 24) | (p[2] << 16) | (p[1] << 8) | p[0];
1628        size = tmp_size;
1629 #endif
1630
1631        if (lseek(iFd, oldPos, SEEK_SET) < 0)
1632           return FileFdErrno("lseek","Unable to seek in gzipped file");
1633
1634        return size;
1635    }
1636 #endif
1637
1638    return size;
1639 }
1640                                                                         /*}}}*/
1641 // FileFd::ModificationTime - Return the time of last touch             /*{{{*/
1642 // ---------------------------------------------------------------------
1643 /* */
1644 time_t FileFd::ModificationTime()
1645 {
1646    struct stat Buf;
1647    if ((d == NULL || d->pipe == false) && fstat(iFd,&Buf) != 0)
1648    {
1649       FileFdErrno("fstat","Unable to determine the modification time of file %s", FileName.c_str());
1650       return 0;
1651    }
1652
1653    // for compressor pipes st_size is undefined and at 'best' zero
1654    if ((d != NULL && d->pipe == true) || S_ISFIFO(Buf.st_mode))
1655    {
1656       // we set it here, too, as we get the info here for free
1657       // in theory the Open-methods should take care of it already
1658       if (d != NULL)
1659          d->pipe = true;
1660       if (stat(FileName.c_str(), &Buf) != 0)
1661       {
1662          FileFdErrno("fstat","Unable to determine the modification time of file %s", FileName.c_str());
1663          return 0;
1664       }
1665    }
1666
1667    return Buf.st_mtime;
1668 }
1669                                                                         /*}}}*/
1670 // FileFd::Close - Close the file if the close flag is set              /*{{{*/
1671 // ---------------------------------------------------------------------
1672 /* */
1673 bool FileFd::Close()
1674 {
1675    if (iFd == -1)
1676       return true;
1677
1678    bool Res = true;
1679    if ((Flags & AutoClose) == AutoClose)
1680    {
1681       if ((Flags & Compressed) != Compressed && iFd > 0 && close(iFd) != 0)
1682          Res &= _error->Errno("close",_("Problem closing the file %s"), FileName.c_str());
1683
1684       if (d != NULL)
1685       {
1686          Res &= d->CloseDown(FileName);
1687          delete d;
1688          d = NULL;
1689       }
1690    }
1691
1692    if ((Flags & Replace) == Replace) {
1693       if (rename(TemporaryFileName.c_str(), FileName.c_str()) != 0)
1694          Res &= _error->Errno("rename",_("Problem renaming the file %s to %s"), TemporaryFileName.c_str(), FileName.c_str());
1695
1696       FileName = TemporaryFileName; // for the unlink() below.
1697       TemporaryFileName.clear();
1698    }
1699
1700    iFd = -1;
1701
1702    if ((Flags & Fail) == Fail && (Flags & DelOnFail) == DelOnFail &&
1703        FileName.empty() == false)
1704       if (unlink(FileName.c_str()) != 0)
1705          Res &= _error->WarningE("unlnk",_("Problem unlinking the file %s"), FileName.c_str());
1706
1707    if (Res == false)
1708       Flags |= Fail;
1709    return Res;
1710 }
1711                                                                         /*}}}*/
1712 // FileFd::Sync - Sync the file                                         /*{{{*/
1713 // ---------------------------------------------------------------------
1714 /* */
1715 bool FileFd::Sync()
1716 {
1717    if (fsync(iFd) != 0)
1718       return FileFdErrno("sync",_("Problem syncing the file"));
1719    return true;
1720 }
1721                                                                         /*}}}*/
1722 // FileFd::FileFdErrno - set Fail and call _error->Errno                *{{{*/
1723 bool FileFd::FileFdErrno(const char *Function, const char *Description,...)
1724 {
1725    Flags |= Fail;
1726    va_list args;
1727    size_t msgSize = 400;
1728    int const errsv = errno;
1729    while (true)
1730    {
1731       va_start(args,Description);
1732       if (_error->InsertErrno(GlobalError::ERROR, Function, Description, args, errsv, msgSize) == false)
1733          break;
1734       va_end(args);
1735    }
1736    return false;
1737 }
1738                                                                         /*}}}*/
1739 // FileFd::FileFdError - set Fail and call _error->Error                *{{{*/
1740 bool FileFd::FileFdError(const char *Description,...) {
1741    Flags |= Fail;
1742    va_list args;
1743    size_t msgSize = 400;
1744    while (true)
1745    {
1746       va_start(args,Description);
1747       if (_error->Insert(GlobalError::ERROR, Description, args, msgSize) == false)
1748          break;
1749       va_end(args);
1750    }
1751    return false;
1752 }
1753                                                                         /*}}}*/
1754
1755 gzFile FileFd::gzFd() { return (gzFile) d->gz; }