apt-pkg/acquire.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire.cc,v 1.50 2004/03/17 05:17:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire - File Acquiration
   7
   8    The core element for the schedule system is the concept of a named
   9    queue. Each queue is unique and each queue has a name derived from the
  10    URI. The degree of paralization can be controlled by how the queue
  11    name is derived from the URI.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <config.h>
  17
  18 #include <apt-pkg/acquire.h>
  19 #include <apt-pkg/acquire-item.h>
  20 #include <apt-pkg/acquire-worker.h>
  21 #include <apt-pkg/configuration.h>
  22 #include <apt-pkg/error.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/fileutl.h>
  25
  26 #include <algorithm>
  27 #include <numeric>
  28 #include <string>
  29 #include <vector>
  30 #include <iostream>
  31 #include <sstream>
  32 #include <iomanip>
  33 #include <memory>
  34
  35 #include <stdio.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38 #include <unistd.h>
  39 #include <fcntl.h>
  40 #include <pwd.h>
  41 #include <grp.h>
  42 #include <dirent.h>
  43 #include <sys/time.h>
  44 #include <sys/select.h>
  45 #include <errno.h>
  46 #include <sys/stat.h>
  47
  48 #include <apti18n.h>
  49                                                                         /*}}}*/
  50
  51 using namespace std;
  52
  53 // Acquire::pkgAcquire - Constructor                                    /*{{{*/
  54 // ---------------------------------------------------------------------
  55 /* We grab some runtime state from the configuration space */
  56 pkgAcquire::pkgAcquire() : LockFD(-1), d(NULL), Queues(0), Workers(0), Configs(0), Log(NULL), ToFetch(0),
  57                            Debug(_config->FindB("Debug::pkgAcquire",false)),
  58                            Running(false)
  59 {
  60    Initialize();
  61 }
  62 pkgAcquire::pkgAcquire(pkgAcquireStatus *Progress) : LockFD(-1), d(NULL), Queues(0), Workers(0),
  63                            Configs(0), Log(NULL), ToFetch(0),
  64                            Debug(_config->FindB("Debug::pkgAcquire",false)),
  65                            Running(false)
  66 {
  67    Initialize();
  68    SetLog(Progress);
  69 }
  70 void pkgAcquire::Initialize()
  71 {
  72    string const Mode = _config->Find("Acquire::Queue-Mode","host");
  73    if (strcasecmp(Mode.c_str(),"host") == 0)
  74       QueueMode = QueueHost;
  75    if (strcasecmp(Mode.c_str(),"access") == 0)
  76       QueueMode = QueueAccess;
  77
  78    // chown the auth.conf file as it will be accessed by our methods
  79    std::string const SandboxUser = _config->Find("APT::Sandbox::User");
  80    if (getuid() == 0 && SandboxUser.empty() == false && SandboxUser != "root") // if we aren't root, we can't chown, so don't try it
  81    {
  82       struct passwd const * const pw = getpwnam(SandboxUser.c_str());
  83       struct group const * const gr = getgrnam(ROOT_GROUP);
  84       if (pw != NULL && gr != NULL)
  85       {
  86          std::string const AuthConf = _config->FindFile("Dir::Etc::netrc");
  87          if(AuthConf.empty() == false && RealFileExists(AuthConf) &&
  88                chown(AuthConf.c_str(), pw->pw_uid, gr->gr_gid) != 0)
  89             _error->WarningE("SetupAPTPartialDirectory", "chown to %s:root of file %s failed", SandboxUser.c_str(), AuthConf.c_str());
  90       }
  91    }
  92 }
  93                                                                         /*}}}*/
  94 // Acquire::GetLock - lock directory and prepare for action             /*{{{*/
  95 static bool SetupAPTPartialDirectory(std::string const &grand, std::string const &parent)
  96 {
  97    std::string const partial = parent + "partial";
  98    mode_t const mode = umask(S_IWGRP | S_IWOTH);
  99    bool const creation_fail = (CreateAPTDirectoryIfNeeded(grand, partial) == false &&
 100          CreateAPTDirectoryIfNeeded(parent, partial) == false);
 101    umask(mode);
 102    if (creation_fail == true)
 103       return false;
 104
 105    std::string const SandboxUser = _config->Find("APT::Sandbox::User");
 106    if (getuid() == 0 && SandboxUser.empty() == false && SandboxUser != "root") // if we aren't root, we can't chown, so don't try it
 107    {
 108       struct passwd const * const pw = getpwnam(SandboxUser.c_str());
 109       struct group const * const gr = getgrnam(ROOT_GROUP);
 110       if (pw != NULL && gr != NULL)
 111       {
 112          // chown the partial dir
 113          if(chown(partial.c_str(), pw->pw_uid, gr->gr_gid) != 0)
 114             _error->WarningE("SetupAPTPartialDirectory", "chown to %s:root of directory %s failed", SandboxUser.c_str(), partial.c_str());
 115       }
 116    }
 117    if (chmod(partial.c_str(), 0700) != 0)
 118       _error->WarningE("SetupAPTPartialDirectory", "chmod 0700 of directory %s failed", partial.c_str());
 119
 120    return true;
 121 }
 122 bool pkgAcquire::Setup(pkgAcquireStatus *Progress, string const &Lock)
 123 {
 124    Log = Progress;
 125    if (Lock.empty())
 126    {
 127       string const listDir = _config->FindDir("Dir::State::lists");
 128       if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir) == false)
 129          return _error->Errno("Acquire", _("List directory %spartial is missing."), listDir.c_str());
 130       string const archivesDir = _config->FindDir("Dir::Cache::Archives");
 131       if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir) == false)
 132          return _error->Errno("Acquire", _("Archives directory %spartial is missing."), archivesDir.c_str());
 133       return true;
 134    }
 135    return GetLock(Lock);
 136 }
 137 bool pkgAcquire::GetLock(std::string const &Lock)
 138 {
 139    if (Lock.empty() == true)
 140       return false;
 141
 142    // check for existence and possibly create auxiliary directories
 143    string const listDir = _config->FindDir("Dir::State::lists");
 144    string const archivesDir = _config->FindDir("Dir::Cache::Archives");
 145
 146    if (Lock == listDir)
 147    {
 148       if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir) == false)
 149          return _error->Errno("Acquire", _("List directory %spartial is missing."), listDir.c_str());
 150    }
 151    if (Lock == archivesDir)
 152    {
 153       if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir) == false)
 154          return _error->Errno("Acquire", _("Archives directory %spartial is missing."), archivesDir.c_str());
 155    }
 156
 157    if (_config->FindB("Debug::NoLocking", false) == true)
 158       return true;
 159
 160    // Lock the directory this acquire object will work in
 161    if (LockFD != -1)
 162       close(LockFD);
 163    LockFD = ::GetLock(flCombine(Lock, "lock"));
 164    if (LockFD == -1)
 165       return _error->Error(_("Unable to lock directory %s"), Lock.c_str());
 166
 167    return true;
 168 }
 169                                                                         /*}}}*/
 170 // Acquire::~pkgAcquire - Destructor                                    /*{{{*/
 171 // ---------------------------------------------------------------------
 172 /* Free our memory, clean up the queues (destroy the workers) */
 173 pkgAcquire::~pkgAcquire()
 174 {
 175    Shutdown();
 176
 177    if (LockFD != -1)
 178       close(LockFD);
 179
 180    while (Configs != 0)
 181    {
 182       MethodConfig *Jnk = Configs;
 183       Configs = Configs->Next;
 184       delete Jnk;
 185    }
 186 }
 187                                                                         /*}}}*/
 188 // Acquire::Shutdown - Clean out the acquire object                     /*{{{*/
 189 // ---------------------------------------------------------------------
 190 /* */
 191 void pkgAcquire::Shutdown()
 192 {
 193    while (Items.empty() == false)
 194    {
 195       if (Items[0]->Status == Item::StatFetching)
 196          Items[0]->Status = Item::StatError;
 197       delete Items[0];
 198    }
 199
 200    while (Queues != 0)
 201    {
 202       Queue *Jnk = Queues;
 203       Queues = Queues->Next;
 204       delete Jnk;
 205    }
 206 }
 207                                                                         /*}}}*/
 208 // Acquire::Add - Add a new item                                        /*{{{*/
 209 // ---------------------------------------------------------------------
 210 /* This puts an item on the acquire list. This list is mainly for tracking
 211    item status */
 212 void pkgAcquire::Add(Item *Itm)
 213 {
 214    Items.push_back(Itm);
 215 }
 216                                                                         /*}}}*/
 217 // Acquire::Remove - Remove a item                                      /*{{{*/
 218 // ---------------------------------------------------------------------
 219 /* Remove an item from the acquire list. This is usually not used.. */
 220 void pkgAcquire::Remove(Item *Itm)
 221 {
 222    Dequeue(Itm);
 223
 224    for (ItemIterator I = Items.begin(); I != Items.end();)
 225    {
 226       if (*I == Itm)
 227       {
 228          Items.erase(I);
 229          I = Items.begin();
 230       }
 231       else
 232          ++I;
 233    }
 234 }
 235                                                                         /*}}}*/
 236 // Acquire::Add - Add a worker                                          /*{{{*/
 237 // ---------------------------------------------------------------------
 238 /* A list of workers is kept so that the select loop can direct their FD
 239    usage. */
 240 void pkgAcquire::Add(Worker *Work)
 241 {
 242    Work->NextAcquire = Workers;
 243    Workers = Work;
 244 }
 245                                                                         /*}}}*/
 246 // Acquire::Remove - Remove a worker                                    /*{{{*/
 247 // ---------------------------------------------------------------------
 248 /* A worker has died. This can not be done while the select loop is running
 249    as it would require that RunFds could handling a changing list state and
 250    it can't.. */
 251 void pkgAcquire::Remove(Worker *Work)
 252 {
 253    if (Running == true)
 254       abort();
 255
 256    Worker **I = &Workers;
 257    for (; *I != 0;)
 258    {
 259       if (*I == Work)
 260          *I = (*I)->NextAcquire;
 261       else
 262          I = &(*I)->NextAcquire;
 263    }
 264 }
 265                                                                         /*}}}*/
 266 // Acquire::Enqueue - Queue an URI for fetching                         /*{{{*/
 267 // ---------------------------------------------------------------------
 268 /* This is the entry point for an item. An item calls this function when
 269    it is constructed which creates a queue (based on the current queue
 270    mode) and puts the item in that queue. If the system is running then
 271    the queue might be started. */
 272 static bool CheckForBadItemAndFailIt(pkgAcquire::Item * const Item,
 273       pkgAcquire::MethodConfig const * const Config, pkgAcquireStatus * const Log)
 274 {
 275    auto SavedDesc = Item->GetItemDesc();
 276    if (Item->IsRedirectionLoop(SavedDesc.URI))
 277    {
 278       std::string const Message = "400 URI Failure"
 279          "\nURI: " + SavedDesc.URI +
 280          "\nFilename: " + Item->DestFile +
 281          "\nFailReason: RedirectionLoop";
 282
 283       Item->Status = pkgAcquire::Item::StatError;
 284       Item->Failed(Message, Config);
 285       if (Log != nullptr)
 286          Log->Fail(SavedDesc);
 287       return true;
 288    }
 289
 290    HashStringList const hsl = Item->GetExpectedHashes();
 291    if (hsl.usable() == false && Item->HashesRequired() &&
 292          _config->Exists("Acquire::ForceHash") == false)
 293    {
 294       std::string const Message = "400 URI Failure"
 295          "\nURI: " + SavedDesc.URI +
 296          "\nFilename: " + Item->DestFile +
 297          "\nFailReason: WeakHashSums";
 298
 299       auto SavedDesc = Item->GetItemDesc();
 300       Item->Status = pkgAcquire::Item::StatAuthError;
 301       Item->Failed(Message, Config);
 302       if (Log != nullptr)
 303          Log->Fail(SavedDesc);
 304       return true;
 305    }
 306    return false;
 307 }
 308 void pkgAcquire::Enqueue(ItemDesc &Item)
 309 {
 310    // Determine which queue to put the item in
 311    const MethodConfig *Config;
 312    string Name = QueueName(Item.URI,Config);
 313    if (Name.empty() == true)
 314       return;
 315
 316    /* the check for running avoids that we produce errors
 317       in logging before we actually have started, which would
 318       be easier to implement but would confuse users/implementations
 319       so we check the items skipped here in #Startup */
 320    if (Running && CheckForBadItemAndFailIt(Item.Owner, Config, Log))
 321       return;
 322
 323    // Find the queue structure
 324    Queue *I = Queues;
 325    for (; I != 0 && I->Name != Name; I = I->Next);
 326    if (I == 0)
 327    {
 328       I = new Queue(Name,this);
 329       I->Next = Queues;
 330       Queues = I;
 331
 332       if (Running == true)
 333          I->Startup();
 334    }
 335
 336    // See if this is a local only URI
 337    if (Config->LocalOnly == true && Item.Owner->Complete == false)
 338       Item.Owner->Local = true;
 339    Item.Owner->Status = Item::StatIdle;
 340
 341    // Queue it into the named queue
 342    if(I->Enqueue(Item))
 343       ToFetch++;
 344
 345    // Some trace stuff
 346    if (Debug == true)
 347    {
 348       clog << "Fetching " << Item.URI << endl;
 349       clog << " to " << Item.Owner->DestFile << endl;
 350       clog << " Queue is: " << Name << endl;
 351    }
 352 }
 353                                                                         /*}}}*/
 354 // Acquire::Dequeue - Remove an item from all queues                    /*{{{*/
 355 // ---------------------------------------------------------------------
 356 /* This is called when an item is finished being fetched. It removes it
 357    from all the queues */
 358 void pkgAcquire::Dequeue(Item *Itm)
 359 {
 360    Queue *I = Queues;
 361    bool Res = false;
 362    if (Debug == true)
 363       clog << "Dequeuing " << Itm->DestFile << endl;
 364
 365    for (; I != 0; I = I->Next)
 366    {
 367       if (I->Dequeue(Itm))
 368       {
 369          Res = true;
 370          if (Debug == true)
 371             clog << "Dequeued from " << I->Name << endl;
 372       }
 373    }
 374
 375    if (Res == true)
 376       ToFetch--;
 377 }
 378                                                                         /*}}}*/
 379 // Acquire::QueueName - Return the name of the queue for this URI       /*{{{*/
 380 // ---------------------------------------------------------------------
 381 /* The string returned depends on the configuration settings and the
 382    method parameters. Given something like http://foo.org/bar it can
 383    return http://foo.org or http */
 384 string pkgAcquire::QueueName(string Uri,MethodConfig const *&Config)
 385 {
 386    URI U(Uri);
 387
 388    Config = GetConfig(U.Access);
 389    if (Config == 0)
 390       return string();
 391
 392    /* Single-Instance methods get exactly one queue per URI. This is
 393       also used for the Access queue method  */
 394    if (Config->SingleInstance == true || QueueMode == QueueAccess)
 395       return U.Access;
 396
 397    string AccessSchema = U.Access + ':';
 398    string FullQueueName;
 399
 400    if (U.Host.empty())
 401    {
 402       long existing = 0;
 403       // check how many queues exist already and reuse empty ones
 404       for (Queue const *I = Queues; I != 0; I = I->Next)
 405          if (I->Name.compare(0, AccessSchema.length(), AccessSchema) == 0)
 406          {
 407             if (I->Items == nullptr)
 408                return I->Name;
 409             ++existing;
 410          }
 411
 412 #ifdef _SC_NPROCESSORS_ONLN
 413       long cpuCount = sysconf(_SC_NPROCESSORS_ONLN) * 2;
 414 #else
 415       long cpuCount = 10;
 416 #endif
 417       cpuCount = _config->FindI("Acquire::QueueHost::Limit", cpuCount);
 418
 419       if (cpuCount <= 0 || existing < cpuCount)
 420          strprintf(FullQueueName, "%s%ld", AccessSchema.c_str(), existing);
 421       else
 422       {
 423          long const randomQueue = random() % cpuCount;
 424          strprintf(FullQueueName, "%s%ld", AccessSchema.c_str(), randomQueue);
 425       }
 426
 427       if (Debug)
 428          clog << "Chose random queue " << FullQueueName << " for " << Uri << endl;
 429    } else
 430    {
 431       FullQueueName = AccessSchema + U.Host;
 432    }
 433    unsigned int Instances = 0, SchemaLength = AccessSchema.length();
 434
 435    Queue *I = Queues;
 436    for (; I != 0; I = I->Next) {
 437       // if the queue already exists, re-use it
 438       if (I->Name == FullQueueName)
 439          return FullQueueName;
 440
 441       if (I->Name.compare(0, SchemaLength, AccessSchema) == 0)
 442          Instances++;
 443    }
 444
 445    if (Debug) {
 446       clog << "Found " << Instances << " instances of " << U.Access << endl;
 447    }
 448
 449    if (Instances >= (unsigned int)_config->FindI("Acquire::QueueHost::Limit",10))
 450       return U.Access;
 451
 452    return FullQueueName;
 453 }
 454                                                                         /*}}}*/
 455 // Acquire::GetConfig - Fetch the configuration information             /*{{{*/
 456 // ---------------------------------------------------------------------
 457 /* This locates the configuration structure for an access method. If
 458    a config structure cannot be found a Worker will be created to
 459    retrieve it */
 460 pkgAcquire::MethodConfig *pkgAcquire::GetConfig(string Access)
 461 {
 462    // Search for an existing config
 463    MethodConfig *Conf;
 464    for (Conf = Configs; Conf != 0; Conf = Conf->Next)
 465       if (Conf->Access == Access)
 466          return Conf;
 467
 468    // Create the new config class
 469    Conf = new MethodConfig;
 470    Conf->Access = Access;
 471    Conf->Next = Configs;
 472    Configs = Conf;
 473
 474    // Create the worker to fetch the configuration
 475    Worker Work(Conf);
 476    if (Work.Start() == false)
 477       return 0;
 478
 479    /* if a method uses DownloadLimit, we switch to SingleInstance mode */
 480    if(_config->FindI("Acquire::"+Access+"::Dl-Limit",0) > 0)
 481       Conf->SingleInstance = true;
 482
 483    return Conf;
 484 }
 485                                                                         /*}}}*/
 486 // Acquire::SetFds - Deal with readable FDs                             /*{{{*/
 487 // ---------------------------------------------------------------------
 488 /* Collect FDs that have activity monitors into the fd sets */
 489 void pkgAcquire::SetFds(int &Fd,fd_set *RSet,fd_set *WSet)
 490 {
 491    for (Worker *I = Workers; I != 0; I = I->NextAcquire)
 492    {
 493       if (I->InReady == true && I->InFd >= 0)
 494       {
 495          if (Fd < I->InFd)
 496             Fd = I->InFd;
 497          FD_SET(I->InFd,RSet);
 498       }
 499       if (I->OutReady == true && I->OutFd >= 0)
 500       {
 501          if (Fd < I->OutFd)
 502             Fd = I->OutFd;
 503          FD_SET(I->OutFd,WSet);
 504       }
 505    }
 506 }
 507                                                                         /*}}}*/
 508 // Acquire::RunFds - compatibility remove on next abi/api break         /*{{{*/
 509 void pkgAcquire::RunFds(fd_set *RSet,fd_set *WSet)
 510 {
 511    RunFdsSane(RSet, WSet);
 512 }
 513                                                                         /*}}}*/
 514 // Acquire::RunFdsSane - Deal with active FDs                           /*{{{*/
 515 // ---------------------------------------------------------------------
 516 /* Dispatch active FDs over to the proper workers. It is very important
 517    that a worker never be erased while this is running! The queue class
 518    should never erase a worker except during shutdown processing. */
 519 bool pkgAcquire::RunFdsSane(fd_set *RSet,fd_set *WSet)
 520 {
 521    bool Res = true;
 522
 523    for (Worker *I = Workers; I != 0; I = I->NextAcquire)
 524    {
 525       if (I->InFd >= 0 && FD_ISSET(I->InFd,RSet) != 0)
 526          Res &= I->InFdReady();
 527       if (I->OutFd >= 0 && FD_ISSET(I->OutFd,WSet) != 0)
 528          Res &= I->OutFdReady();
 529    }
 530
 531    return Res;
 532 }
 533                                                                         /*}}}*/
 534 // Acquire::Run - Run the fetch sequence                                /*{{{*/
 535 // ---------------------------------------------------------------------
 536 /* This runs the queues. It manages a select loop for all of the
 537    Worker tasks. The workers interact with the queues and items to
 538    manage the actual fetch. */
 539 static bool IsAccessibleBySandboxUser(std::string const &filename, bool const ReadWrite)
 540 {
 541    // you would think this is easily to answer with faccessat, right? Wrong!
 542    // It e.g. gets groups wrong, so the only thing which works reliable is trying
 543    // to open the file we want to open later on…
 544    if (unlikely(filename.empty()))
 545       return true;
 546
 547    if (ReadWrite == false)
 548    {
 549       errno = 0;
 550       // can we read a file? Note that non-existing files are "fine"
 551       int const fd = open(filename.c_str(), O_RDONLY | O_CLOEXEC);
 552       if (fd == -1 && errno == EACCES)
 553          return false;
 554       close(fd);
 555       return true;
 556    }
 557    else
 558    {
 559       // the file might not exist yet and even if it does we will fix permissions,
 560       // so important is here just that the directory it is in allows that
 561       std::string const dirname = flNotFile(filename);
 562       if (unlikely(dirname.empty()))
 563          return true;
 564
 565       char const * const filetag = ".apt-acquire-privs-test.XXXXXX";
 566       std::string const tmpfile_tpl = flCombine(dirname, filetag);
 567       std::unique_ptr<char, decltype(std::free) *> tmpfile { strdup(tmpfile_tpl.c_str()), std::free };
 568       int const fd = mkstemp(tmpfile.get());
 569       if (fd == -1 && errno == EACCES)
 570          return false;
 571       RemoveFile("IsAccessibleBySandboxUser", tmpfile.get());
 572       close(fd);
 573       return true;
 574    }
 575 }
 576 static void CheckDropPrivsMustBeDisabled(pkgAcquire const &Fetcher)
 577 {
 578    if(getuid() != 0)
 579       return;
 580
 581    std::string const SandboxUser = _config->Find("APT::Sandbox::User");
 582    if (SandboxUser.empty() || SandboxUser == "root")
 583       return;
 584
 585    struct passwd const * const pw = getpwnam(SandboxUser.c_str());
 586    if (pw == NULL)
 587    {
 588       _error->Warning(_("No sandbox user '%s' on the system, can not drop privileges"), SandboxUser.c_str());
 589       _config->Set("APT::Sandbox::User", "");
 590       return;
 591    }
 592
 593    gid_t const old_euid = geteuid();
 594    gid_t const old_egid = getegid();
 595
 596    long const ngroups_max = sysconf(_SC_NGROUPS_MAX);
 597    std::unique_ptr<gid_t[]> old_gidlist(new gid_t[ngroups_max]);
 598    if (unlikely(old_gidlist == NULL))
 599       return;
 600    ssize_t old_gidlist_nr;
 601    if ((old_gidlist_nr = getgroups(ngroups_max, old_gidlist.get())) < 0)
 602    {
 603       _error->FatalE("getgroups", "getgroups %lu failed", ngroups_max);
 604       old_gidlist[0] = 0;
 605       old_gidlist_nr = 1;
 606    }
 607    if (setgroups(1, &pw->pw_gid))
 608       _error->FatalE("setgroups", "setgroups %u failed", pw->pw_gid);
 609
 610    if (setegid(pw->pw_gid) != 0)
 611       _error->FatalE("setegid", "setegid %u failed", pw->pw_gid);
 612    if (seteuid(pw->pw_uid) != 0)
 613       _error->FatalE("seteuid", "seteuid %u failed", pw->pw_uid);
 614
 615    for (pkgAcquire::ItemCIterator I = Fetcher.ItemsBegin();
 616         I != Fetcher.ItemsEnd(); ++I)
 617    {
 618       // no need to drop privileges for a complete file
 619       if ((*I)->Complete == true || (*I)->Status != pkgAcquire::Item::StatIdle)
 620          continue;
 621
 622       // if destination file is inaccessible all hope is lost for privilege dropping
 623       if (IsAccessibleBySandboxUser((*I)->DestFile, true) == false)
 624       {
 625          _error->WarningE("pkgAcquire::Run", _("Can't drop privileges for downloading as file '%s' couldn't be accessed by user '%s'."),
 626                (*I)->DestFile.c_str(), SandboxUser.c_str());
 627          _config->Set("APT::Sandbox::User", "");
 628          break;
 629       }
 630
 631       // if its the source file (e.g. local sources) we might be lucky
 632       // by dropping the dropping only for some methods.
 633       URI const source = (*I)->DescURI();
 634       if (source.Access == "file" || source.Access == "copy")
 635       {
 636          std::string const conf = "Binary::" + source.Access + "::APT::Sandbox::User";
 637          if (_config->Exists(conf) == true)
 638             continue;
 639
 640          if (IsAccessibleBySandboxUser(source.Path, false) == false)
 641          {
 642             _error->NoticeE("pkgAcquire::Run", _("Can't drop privileges for downloading as file '%s' couldn't be accessed by user '%s'."),
 643                   source.Path.c_str(), SandboxUser.c_str());
 644             _config->CndSet("Binary::file::APT::Sandbox::User", "root");
 645             _config->CndSet("Binary::copy::APT::Sandbox::User", "root");
 646          }
 647       }
 648    }
 649
 650    if (seteuid(old_euid) != 0)
 651       _error->FatalE("seteuid", "seteuid %u failed", old_euid);
 652    if (setegid(old_egid) != 0)
 653       _error->FatalE("setegid", "setegid %u failed", old_egid);
 654    if (setgroups(old_gidlist_nr, old_gidlist.get()))
 655       _error->FatalE("setgroups", "setgroups %u failed", 0);
 656 }
 657 pkgAcquire::RunResult pkgAcquire::Run(int PulseIntervall)
 658 {
 659    _error->PushToStack();
 660    CheckDropPrivsMustBeDisabled(*this);
 661
 662    Running = true;
 663
 664    for (Queue *I = Queues; I != 0; I = I->Next)
 665       I->Startup();
 666
 667    if (Log != 0)
 668       Log->Start();
 669
 670    bool WasCancelled = false;
 671
 672    // Run till all things have been acquired
 673    struct timeval tv;
 674    tv.tv_sec = 0;
 675    tv.tv_usec = PulseIntervall;
 676    while (ToFetch > 0)
 677    {
 678       fd_set RFds;
 679       fd_set WFds;
 680       int Highest = 0;
 681       FD_ZERO(&RFds);
 682       FD_ZERO(&WFds);
 683       SetFds(Highest,&RFds,&WFds);
 684
 685       int Res;
 686       do
 687       {
 688          Res = select(Highest+1,&RFds,&WFds,0,&tv);
 689       }
 690       while (Res < 0 && errno == EINTR);
 691
 692       if (Res < 0)
 693       {
 694          _error->Errno("select","Select has failed");
 695          break;
 696       }
 697
 698       if(RunFdsSane(&RFds,&WFds) == false)
 699          break;
 700
 701       // Timeout, notify the log class
 702       if (Res == 0 || (Log != 0 && Log->Update == true))
 703       {
 704          tv.tv_usec = PulseIntervall;
 705          for (Worker *I = Workers; I != 0; I = I->NextAcquire)
 706             I->Pulse();
 707          if (Log != 0 && Log->Pulse(this) == false)
 708          {
 709             WasCancelled = true;
 710             break;
 711          }
 712       }
 713    }
 714
 715    if (Log != 0)
 716       Log->Stop();
 717
 718    // Shut down the acquire bits
 719    Running = false;
 720    for (Queue *I = Queues; I != 0; I = I->Next)
 721       I->Shutdown(false);
 722
 723    // Shut down the items
 724    for (ItemIterator I = Items.begin(); I != Items.end(); ++I)
 725       (*I)->Finished();
 726
 727    bool const newError = _error->PendingError();
 728    _error->MergeWithStack();
 729    if (newError)
 730       return Failed;
 731    if (WasCancelled)
 732       return Cancelled;
 733    return Continue;
 734 }
 735                                                                         /*}}}*/
 736 // Acquire::Bump - Called when an item is dequeued                      /*{{{*/
 737 // ---------------------------------------------------------------------
 738 /* This routine bumps idle queues in hopes that they will be able to fetch
 739    the dequeued item */
 740 void pkgAcquire::Bump()
 741 {
 742    for (Queue *I = Queues; I != 0; I = I->Next)
 743       I->Bump();
 744 }
 745                                                                         /*}}}*/
 746 // Acquire::WorkerStep - Step to the next worker                        /*{{{*/
 747 // ---------------------------------------------------------------------
 748 /* Not inlined to advoid including acquire-worker.h */
 749 pkgAcquire::Worker *pkgAcquire::WorkerStep(Worker *I)
 750 {
 751    return I->NextAcquire;
 752 }
 753                                                                         /*}}}*/
 754 // Acquire::Clean - Cleans a directory                                  /*{{{*/
 755 // ---------------------------------------------------------------------
 756 /* This is a bit simplistic, it looks at every file in the dir and sees
 757    if it is part of the download set. */
 758 bool pkgAcquire::Clean(string Dir)
 759 {
 760    // non-existing directories are by definition clean…
 761    if (DirectoryExists(Dir) == false)
 762       return true;
 763
 764    if(Dir == "/")
 765       return _error->Error(_("Clean of %s is not supported"), Dir.c_str());
 766
 767    DIR *D = opendir(Dir.c_str());
 768    if (D == 0)
 769       return _error->Errno("opendir",_("Unable to read %s"),Dir.c_str());
 770
 771    string StartDir = SafeGetCWD();
 772    if (chdir(Dir.c_str()) != 0)
 773    {
 774       closedir(D);
 775       return _error->Errno("chdir",_("Unable to change to %s"),Dir.c_str());
 776    }
 777
 778    for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D))
 779    {
 780       // Skip some files..
 781       if (strcmp(Dir->d_name,"lock") == 0 ||
 782           strcmp(Dir->d_name,"partial") == 0 ||
 783           strcmp(Dir->d_name,"lost+found") == 0 ||
 784           strcmp(Dir->d_name,".") == 0 ||
 785           strcmp(Dir->d_name,"..") == 0)
 786          continue;
 787
 788       // Look in the get list
 789       ItemCIterator I = Items.begin();
 790       for (; I != Items.end(); ++I)
 791          if (flNotDir((*I)->DestFile) == Dir->d_name)
 792             break;
 793
 794       // Nothing found, nuke it
 795       if (I == Items.end())
 796          RemoveFile("Clean", Dir->d_name);
 797    };
 798
 799    closedir(D);
 800    if (chdir(StartDir.c_str()) != 0)
 801       return _error->Errno("chdir",_("Unable to change to %s"),StartDir.c_str());
 802    return true;
 803 }
 804                                                                         /*}}}*/
 805 // Acquire::TotalNeeded - Number of bytes to fetch                      /*{{{*/
 806 // ---------------------------------------------------------------------
 807 /* This is the total number of bytes needed */
 808 APT_PURE unsigned long long pkgAcquire::TotalNeeded()
 809 {
 810    return std::accumulate(ItemsBegin(), ItemsEnd(), 0llu,
 811       [](unsigned long long const T, Item const * const I) {
 812          return T + I->FileSize;
 813    });
 814 }
 815                                                                         /*}}}*/
 816 // Acquire::FetchNeeded - Number of bytes needed to get                 /*{{{*/
 817 // ---------------------------------------------------------------------
 818 /* This is the number of bytes that is not local */
 819 APT_PURE unsigned long long pkgAcquire::FetchNeeded()
 820 {
 821    return std::accumulate(ItemsBegin(), ItemsEnd(), 0llu,
 822       [](unsigned long long const T, Item const * const I) {
 823          if (I->Local == false)
 824             return T + I->FileSize;
 825          else
 826             return T;
 827    });
 828 }
 829                                                                         /*}}}*/
 830 // Acquire::PartialPresent - Number of partial bytes we already have    /*{{{*/
 831 // ---------------------------------------------------------------------
 832 /* This is the number of bytes that is not local */
 833 APT_PURE unsigned long long pkgAcquire::PartialPresent()
 834 {
 835    return std::accumulate(ItemsBegin(), ItemsEnd(), 0llu,
 836       [](unsigned long long const T, Item const * const I) {
 837          if (I->Local == false)
 838             return T + I->PartialSize;
 839          else
 840             return T;
 841    });
 842 }
 843                                                                         /*}}}*/
 844 // Acquire::UriBegin - Start iterator for the uri list                  /*{{{*/
 845 // ---------------------------------------------------------------------
 846 /* */
 847 pkgAcquire::UriIterator pkgAcquire::UriBegin()
 848 {
 849    return UriIterator(Queues);
 850 }
 851                                                                         /*}}}*/
 852 // Acquire::UriEnd - End iterator for the uri list                      /*{{{*/
 853 // ---------------------------------------------------------------------
 854 /* */
 855 pkgAcquire::UriIterator pkgAcquire::UriEnd()
 856 {
 857    return UriIterator(0);
 858 }
 859                                                                         /*}}}*/
 860 // Acquire::MethodConfig::MethodConfig - Constructor                    /*{{{*/
 861 // ---------------------------------------------------------------------
 862 /* */
 863 pkgAcquire::MethodConfig::MethodConfig() : d(NULL), Next(0), SingleInstance(false),
 864    Pipeline(false), SendConfig(false), LocalOnly(false), NeedsCleanup(false),
 865    Removable(false)
 866 {
 867 }
 868                                                                         /*}}}*/
 869 // Queue::Queue - Constructor                                           /*{{{*/
 870 // ---------------------------------------------------------------------
 871 /* */
 872 pkgAcquire::Queue::Queue(string const &name,pkgAcquire * const owner) : d(NULL), Next(0),
 873    Name(name), Items(0), Workers(0), Owner(owner), PipeDepth(0), MaxPipeDepth(1)
 874 {
 875 }
 876                                                                         /*}}}*/
 877 // Queue::~Queue - Destructor                                           /*{{{*/
 878 // ---------------------------------------------------------------------
 879 /* */
 880 pkgAcquire::Queue::~Queue()
 881 {
 882    Shutdown(true);
 883
 884    while (Items != 0)
 885    {
 886       QItem *Jnk = Items;
 887       Items = Items->Next;
 888       delete Jnk;
 889    }
 890 }
 891                                                                         /*}}}*/
 892 // Queue::Enqueue - Queue an item to the queue                          /*{{{*/
 893 // ---------------------------------------------------------------------
 894 /* */
 895 bool pkgAcquire::Queue::Enqueue(ItemDesc &Item)
 896 {
 897    QItem **OptimalI = &Items;
 898    QItem **I = &Items;
 899    // move to the end of the queue and check for duplicates here
 900    for (; *I != 0; ) {
 901       if (Item.URI == (*I)->URI)
 902       {
 903          if (_config->FindB("Debug::pkgAcquire::Worker",false) == true)
 904             std::cerr << " @ Queue: Action combined for " << Item.URI << " and " << (*I)->URI << std::endl;
 905          (*I)->Owners.push_back(Item.Owner);
 906          Item.Owner->Status = (*I)->Owner->Status;
 907          return false;
 908       }
 909       // Determine the optimal position to insert: before anything with a
 910       // higher priority.
 911       int priority = (*I)->GetPriority();
 912
 913       I = &(*I)->Next;
 914       if (priority >= Item.Owner->Priority()) {
 915          OptimalI = I;
 916       }
 917    }
 918
 919
 920    // Create a new item
 921    QItem *Itm = new QItem;
 922    *Itm = Item;
 923    Itm->Next = *OptimalI;
 924    *OptimalI = Itm;
 925
 926    Item.Owner->QueueCounter++;
 927    if (Items->Next == 0)
 928       Cycle();
 929    return true;
 930 }
 931                                                                         /*}}}*/
 932 // Queue::Dequeue - Remove an item from the queue                       /*{{{*/
 933 // ---------------------------------------------------------------------
 934 /* We return true if we hit something */
 935 bool pkgAcquire::Queue::Dequeue(Item *Owner)
 936 {
 937    if (Owner->Status == pkgAcquire::Item::StatFetching)
 938       return _error->Error("Tried to dequeue a fetching object");
 939
 940    bool Res = false;
 941
 942    QItem **I = &Items;
 943    for (; *I != 0;)
 944    {
 945       if (Owner == (*I)->Owner)
 946       {
 947          QItem *Jnk= *I;
 948          *I = (*I)->Next;
 949          Owner->QueueCounter--;
 950          delete Jnk;
 951          Res = true;
 952       }
 953       else
 954          I = &(*I)->Next;
 955    }
 956
 957    return Res;
 958 }
 959                                                                         /*}}}*/
 960 // Queue::Startup - Start the worker processes                          /*{{{*/
 961 // ---------------------------------------------------------------------
 962 /* It is possible for this to be called with a pre-existing set of
 963    workers. */
 964 bool pkgAcquire::Queue::Startup()
 965 {
 966    if (Workers == 0)
 967    {
 968       URI U(Name);
 969       pkgAcquire::MethodConfig * const Cnf = Owner->GetConfig(U.Access);
 970       if (unlikely(Cnf == nullptr))
 971          return false;
 972
 973       // now-running twin of the pkgAcquire::Enqueue call
 974       for (QItem *I = Items; I != 0; )
 975       {
 976          auto const INext = I->Next;
 977          for (auto &&O: I->Owners)
 978             CheckForBadItemAndFailIt(O, Cnf, Owner->Log);
 979          // if an item failed, it will be auto-dequeued invalidation our I here
 980          I = INext;
 981       }
 982
 983       Workers = new Worker(this,Cnf,Owner->Log);
 984       Owner->Add(Workers);
 985       if (Workers->Start() == false)
 986          return false;
 987
 988       /* When pipelining we commit 10 items. This needs to change when we
 989          added other source retry to have cycle maintain a pipeline depth
 990          on its own. */
 991       if (Cnf->Pipeline == true)
 992          MaxPipeDepth = _config->FindI("Acquire::Max-Pipeline-Depth",10);
 993       else
 994          MaxPipeDepth = 1;
 995    }
 996
 997    return Cycle();
 998 }
 999                                                                         /*}}}*/
1000 // Queue::Shutdown - Shutdown the worker processes                      /*{{{*/
1001 // ---------------------------------------------------------------------
1002 /* If final is true then all workers are eliminated, otherwise only workers
1003    that do not need cleanup are removed */
1004 bool pkgAcquire::Queue::Shutdown(bool Final)
1005 {
1006    // Delete all of the workers
1007    pkgAcquire::Worker **Cur = &Workers;
1008    while (*Cur != 0)
1009    {
1010       pkgAcquire::Worker *Jnk = *Cur;
1011       if (Final == true || Jnk->GetConf()->NeedsCleanup == false)
1012       {
1013          *Cur = Jnk->NextQueue;
1014          Owner->Remove(Jnk);
1015          delete Jnk;
1016       }
1017       else
1018          Cur = &(*Cur)->NextQueue;
1019    }
1020
1021    return true;
1022 }
1023                                                                         /*}}}*/
1024 // Queue::FindItem - Find a URI in the item list                        /*{{{*/
1025 // ---------------------------------------------------------------------
1026 /* */
1027 pkgAcquire::Queue::QItem *pkgAcquire::Queue::FindItem(string URI,pkgAcquire::Worker *Owner)
1028 {
1029    for (QItem *I = Items; I != 0; I = I->Next)
1030       if (I->URI == URI && I->Worker == Owner)
1031          return I;
1032    return 0;
1033 }
1034                                                                         /*}}}*/
1035 // Queue::ItemDone - Item has been completed                            /*{{{*/
1036 // ---------------------------------------------------------------------
1037 /* The worker signals this which causes the item to be removed from the
1038    queue. If this is the last queue instance then it is removed from the
1039    main queue too.*/
1040 bool pkgAcquire::Queue::ItemDone(QItem *Itm)
1041 {
1042    PipeDepth--;
1043    for (QItem::owner_iterator O = Itm->Owners.begin(); O != Itm->Owners.end(); ++O)
1044    {
1045       if ((*O)->Status == pkgAcquire::Item::StatFetching)
1046          (*O)->Status = pkgAcquire::Item::StatDone;
1047    }
1048
1049    if (Itm->Owner->QueueCounter <= 1)
1050       Owner->Dequeue(Itm->Owner);
1051    else
1052    {
1053       Dequeue(Itm->Owner);
1054       Owner->Bump();
1055    }
1056
1057    return Cycle();
1058 }
1059                                                                         /*}}}*/
1060 // Queue::Cycle - Queue new items into the method                       /*{{{*/
1061 // ---------------------------------------------------------------------
1062 /* This locates a new idle item and sends it to the worker. If pipelining
1063    is enabled then it keeps the pipe full. */
1064 bool pkgAcquire::Queue::Cycle()
1065 {
1066    if (Items == 0 || Workers == 0)
1067       return true;
1068
1069    if (PipeDepth < 0)
1070       return _error->Error("Pipedepth failure");
1071
1072    // Look for a queable item
1073    QItem *I = Items;
1074    int ActivePriority = 0;
1075    while (PipeDepth < (signed)MaxPipeDepth)
1076    {
1077       for (; I != 0; I = I->Next) {
1078          if (I->Owner->Status == pkgAcquire::Item::StatFetching)
1079             ActivePriority = std::max(ActivePriority, I->GetPriority());
1080          if (I->Owner->Status == pkgAcquire::Item::StatIdle)
1081             break;
1082       }
1083
1084       // Nothing to do, queue is idle.
1085       if (I == 0)
1086          return true;
1087
1088       // This item has a lower priority than stuff in the pipeline, pretend
1089       // the queue is idle
1090       if (I->GetPriority() < ActivePriority)
1091          return true;
1092       I->Worker = Workers;
1093       for (auto const &O: I->Owners)
1094          O->Status = pkgAcquire::Item::StatFetching;
1095       PipeDepth++;
1096       if (Workers->QueueItem(I) == false)
1097          return false;
1098    }
1099
1100    return true;
1101 }
1102                                                                         /*}}}*/
1103 // Queue::Bump - Fetch any pending objects if we are idle               /*{{{*/
1104 // ---------------------------------------------------------------------
1105 /* This is called when an item in multiple queues is dequeued */
1106 void pkgAcquire::Queue::Bump()
1107 {
1108    Cycle();
1109 }
1110                                                                         /*}}}*/
1111 HashStringList pkgAcquire::Queue::QItem::GetExpectedHashes() const      /*{{{*/
1112 {
1113    /* each Item can have multiple owners and each owner might have different
1114       hashes, even if that is unlikely in practice and if so at least some
1115       owners will later fail. There is one situation through which is not a
1116       failure and still needs this handling: Two owners who expect the same
1117       file, but one owner only knows the SHA1 while the other only knows SHA256. */
1118    HashStringList superhsl;
1119    for (pkgAcquire::Queue::QItem::owner_iterator O = Owners.begin(); O != Owners.end(); ++O)
1120    {
1121       HashStringList const hsl = (*O)->GetExpectedHashes();
1122       if (hsl.usable() == false)
1123          continue;
1124       if (superhsl.usable() == false)
1125          superhsl = hsl;
1126       else
1127       {
1128          // we merge both lists - if we find disagreement send no hashes
1129          HashStringList::const_iterator hs = hsl.begin();
1130          for (; hs != hsl.end(); ++hs)
1131             if (superhsl.push_back(*hs) == false)
1132                break;
1133          if (hs != hsl.end())
1134          {
1135             superhsl.clear();
1136             break;
1137          }
1138       }
1139    }
1140    return superhsl;
1141 }
1142                                                                         /*}}}*/
1143 APT_PURE unsigned long long pkgAcquire::Queue::QItem::GetMaximumSize() const    /*{{{*/
1144 {
1145    unsigned long long Maximum = std::numeric_limits<unsigned long long>::max();
1146    for (auto const &O: Owners)
1147    {
1148       if (O->FileSize == 0)
1149          continue;
1150       Maximum = std::min(Maximum, O->FileSize);
1151    }
1152    if (Maximum == std::numeric_limits<unsigned long long>::max())
1153       return 0;
1154    return Maximum;
1155 }
1156                                                                         /*}}}*/
1157 APT_PURE int pkgAcquire::Queue::QItem::GetPriority() const              /*{{{*/
1158 {
1159    int Priority = 0;
1160    for (auto const &O: Owners)
1161       Priority = std::max(Priority, O->Priority());
1162
1163    return Priority;
1164 }
1165                                                                         /*}}}*/
1166 void pkgAcquire::Queue::QItem::SyncDestinationFiles() const             /*{{{*/
1167 {
1168    /* ensure that the first owner has the best partial file of all and
1169       the rest have (potentially dangling) symlinks to it so that
1170       everything (like progress reporting) finds it easily */
1171    std::string superfile = Owner->DestFile;
1172    off_t supersize = 0;
1173    for (pkgAcquire::Queue::QItem::owner_iterator O = Owners.begin(); O != Owners.end(); ++O)
1174    {
1175       if ((*O)->DestFile == superfile)
1176          continue;
1177       struct stat file;
1178       if (lstat((*O)->DestFile.c_str(),&file) == 0)
1179       {
1180          if ((file.st_mode & S_IFREG) == 0)
1181             RemoveFile("SyncDestinationFiles", (*O)->DestFile);
1182          else if (supersize < file.st_size)
1183          {
1184             supersize = file.st_size;
1185             RemoveFile("SyncDestinationFiles", superfile);
1186             rename((*O)->DestFile.c_str(), superfile.c_str());
1187          }
1188          else
1189             RemoveFile("SyncDestinationFiles", (*O)->DestFile);
1190          if (symlink(superfile.c_str(), (*O)->DestFile.c_str()) != 0)
1191          {
1192             ; // not a problem per-se and no real alternative
1193          }
1194       }
1195    }
1196 }
1197                                                                         /*}}}*/
1198 std::string pkgAcquire::Queue::QItem::Custom600Headers() const          /*{{{*/
1199 {
1200    /* The others are relatively easy to merge, but this one?
1201       Lets not merge and see how far we can run with it…
1202       Likely, nobody will ever notice as all the items will
1203       be of the same class and hence generate the same headers. */
1204    return Owner->Custom600Headers();
1205 }
1206                                                                         /*}}}*/
1207
1208 // AcquireStatus::pkgAcquireStatus - Constructor                        /*{{{*/
1209 // ---------------------------------------------------------------------
1210 /* */
1211 pkgAcquireStatus::pkgAcquireStatus() : d(NULL), Percent(-1), Update(true), MorePulses(false)
1212 {
1213    Start();
1214 }
1215                                                                         /*}}}*/
1216 // AcquireStatus::Pulse - Called periodically                           /*{{{*/
1217 // ---------------------------------------------------------------------
1218 /* This computes some internal state variables for the derived classes to
1219    use. It generates the current downloaded bytes and total bytes to download
1220    as well as the current CPS estimate. */
1221 bool pkgAcquireStatus::Pulse(pkgAcquire *Owner)
1222 {
1223    TotalBytes = 0;
1224    CurrentBytes = 0;
1225    TotalItems = 0;
1226    CurrentItems = 0;
1227
1228    // Compute the total number of bytes to fetch
1229    unsigned int Unknown = 0;
1230    unsigned int Count = 0;
1231    bool ExpectAdditionalItems = false;
1232    for (pkgAcquire::ItemCIterator I = Owner->ItemsBegin();
1233         I != Owner->ItemsEnd();
1234         ++I, ++Count)
1235    {
1236       TotalItems++;
1237       if ((*I)->Status == pkgAcquire::Item::StatDone)
1238          ++CurrentItems;
1239
1240       // do we expect to acquire more files than we know of yet?
1241       if ((*I)->ExpectedAdditionalItems > 0)
1242          ExpectAdditionalItems = true;
1243
1244       TotalBytes += (*I)->FileSize;
1245       if ((*I)->Complete == true)
1246          CurrentBytes += (*I)->FileSize;
1247       if ((*I)->FileSize == 0 && (*I)->Complete == false)
1248          ++Unknown;
1249    }
1250
1251    // Compute the current completion
1252    unsigned long long ResumeSize = 0;
1253    for (pkgAcquire::Worker *I = Owner->WorkersBegin(); I != 0;
1254         I = Owner->WorkerStep(I))
1255    {
1256       if (I->CurrentItem != 0 && I->CurrentItem->Owner->Complete == false)
1257       {
1258          CurrentBytes += I->CurrentSize;
1259          ResumeSize += I->ResumePoint;
1260
1261          // Files with unknown size always have 100% completion
1262          if (I->CurrentItem->Owner->FileSize == 0 &&
1263              I->CurrentItem->Owner->Complete == false)
1264             TotalBytes += I->CurrentSize;
1265       }
1266    }
1267
1268    // Normalize the figures and account for unknown size downloads
1269    if (TotalBytes <= 0)
1270       TotalBytes = 1;
1271    if (Unknown == Count)
1272       TotalBytes = Unknown;
1273
1274    // Wha?! Is not supposed to happen.
1275    if (CurrentBytes > TotalBytes)
1276       CurrentBytes = TotalBytes;
1277
1278    // Compute the CPS
1279    struct timeval NewTime;
1280    gettimeofday(&NewTime,0);
1281    if ((NewTime.tv_sec - Time.tv_sec == 6 && NewTime.tv_usec > Time.tv_usec) ||
1282        NewTime.tv_sec - Time.tv_sec > 6)
1283    {
1284       double Delta = NewTime.tv_sec - Time.tv_sec +
1285                      (NewTime.tv_usec - Time.tv_usec)/1000000.0;
1286
1287       // Compute the CPS value
1288       if (Delta < 0.01)
1289          CurrentCPS = 0;
1290       else
1291          CurrentCPS = ((CurrentBytes - ResumeSize) - LastBytes)/Delta;
1292       LastBytes = CurrentBytes - ResumeSize;
1293       ElapsedTime = (unsigned long long)Delta;
1294       Time = NewTime;
1295    }
1296
1297    double const OldPercent = Percent;
1298    // calculate the percentage, if we have too little data assume 1%
1299    if (ExpectAdditionalItems)
1300       Percent = 0;
1301    else
1302       // use both files and bytes because bytes can be unreliable
1303       Percent = (0.8 * (CurrentBytes/float(TotalBytes)*100.0) +
1304                  0.2 * (CurrentItems/float(TotalItems)*100.0));
1305
1306    // debug
1307    if (_config->FindB("Debug::acquire::progress", false) == true)
1308    {
1309       std::clog
1310          << "["
1311          << std::setw(5) << std::setprecision(4) << std::showpoint << Percent
1312          << "]"
1313          << " Bytes: "
1314          << SizeToStr(CurrentBytes) << " / " << SizeToStr(TotalBytes)
1315          << " # Files: "
1316          << CurrentItems << " / " << TotalItems
1317          << std::endl;
1318    }
1319
1320    double const DiffPercent = Percent - OldPercent;
1321    if (DiffPercent < 0.001 && _config->FindB("Acquire::Progress::Diffpercent", false) == true)
1322       return true;
1323
1324    int fd = _config->FindI("APT::Status-Fd",-1);
1325    if(fd > 0)
1326    {
1327       ostringstream status;
1328
1329       char msg[200];
1330       long i = CurrentItems < TotalItems ? CurrentItems + 1 : CurrentItems;
1331       unsigned long long ETA = 0;
1332       if(CurrentCPS > 0)
1333          ETA = (TotalBytes - CurrentBytes) / CurrentCPS;
1334
1335       // only show the ETA if it makes sense
1336       if (ETA > 0 && ETA < 172800 /* two days */ )
1337          snprintf(msg,sizeof(msg), _("Retrieving file %li of %li (%s remaining)"), i, TotalItems, TimeToStr(ETA).c_str());
1338       else
1339          snprintf(msg,sizeof(msg), _("Retrieving file %li of %li"), i, TotalItems);
1340
1341       // build the status str
1342       std::ostringstream str;
1343       str.imbue(std::locale::classic());
1344       str.precision(4);
1345       str << "dlstatus" << ':' << std::fixed << i << ':' << Percent << ':' << msg << '\n';
1346       auto const dlstatus = str.str();
1347       FileFd::Write(fd, dlstatus.data(), dlstatus.size());
1348    }
1349
1350    return true;
1351 }
1352                                                                         /*}}}*/
1353 // AcquireStatus::Start - Called when the download is started           /*{{{*/
1354 // ---------------------------------------------------------------------
1355 /* We just reset the counters */
1356 void pkgAcquireStatus::Start()
1357 {
1358    gettimeofday(&Time,0);
1359    gettimeofday(&StartTime,0);
1360    LastBytes = 0;
1361    CurrentCPS = 0;
1362    CurrentBytes = 0;
1363    TotalBytes = 0;
1364    FetchedBytes = 0;
1365    ElapsedTime = 0;
1366    TotalItems = 0;
1367    CurrentItems = 0;
1368 }
1369                                                                         /*}}}*/
1370 // AcquireStatus::Stop - Finished downloading                           /*{{{*/
1371 // ---------------------------------------------------------------------
1372 /* This accurately computes the elapsed time and the total overall CPS. */
1373 void pkgAcquireStatus::Stop()
1374 {
1375    // Compute the CPS and elapsed time
1376    struct timeval NewTime;
1377    gettimeofday(&NewTime,0);
1378
1379    double Delta = NewTime.tv_sec - StartTime.tv_sec +
1380                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
1381
1382    // Compute the CPS value
1383    if (Delta < 0.01)
1384       CurrentCPS = 0;
1385    else
1386       CurrentCPS = FetchedBytes/Delta;
1387    LastBytes = CurrentBytes;
1388    ElapsedTime = (unsigned long long)Delta;
1389 }
1390                                                                         /*}}}*/
1391 // AcquireStatus::Fetched - Called when a byte set has been fetched     /*{{{*/
1392 // ---------------------------------------------------------------------
1393 /* This is used to get accurate final transfer rate reporting. */
1394 void pkgAcquireStatus::Fetched(unsigned long long Size,unsigned long long Resume)
1395 {
1396    FetchedBytes += Size - Resume;
1397 }
1398                                                                         /*}}}*/
1399
1400 pkgAcquire::UriIterator::UriIterator(pkgAcquire::Queue *Q) : d(NULL), CurQ(Q), CurItem(0)
1401 {
1402    while (CurItem == 0 && CurQ != 0)
1403    {
1404       CurItem = CurQ->Items;
1405       CurQ = CurQ->Next;
1406    }
1407 }
1408
1409 APT_CONST pkgAcquire::UriIterator::~UriIterator() {}
1410 APT_CONST pkgAcquire::MethodConfig::~MethodConfig() {}
1411 APT_CONST pkgAcquireStatus::~pkgAcquireStatus() {}