apt-pkg/acquire.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire.cc,v 1.50 2004/03/17 05:17:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire - File Acquiration
   7
   8    The core element for the schedule system is the concept of a named
   9    queue. Each queue is unique and each queue has a name derived from the
  10    URI. The degree of paralization can be controlled by how the queue
  11    name is derived from the URI.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <config.h>
  17
  18 #include <apt-pkg/acquire.h>
  19 #include <apt-pkg/acquire-item.h>
  20 #include <apt-pkg/acquire-worker.h>
  21 #include <apt-pkg/configuration.h>
  22 #include <apt-pkg/error.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/fileutl.h>
  25
  26 #include <algorithm>
  27 #include <numeric>
  28 #include <string>
  29 #include <vector>
  30 #include <iostream>
  31 #include <sstream>
  32 #include <iomanip>
  33
  34 #include <stdio.h>
  35 #include <stdlib.h>
  36 #include <string.h>
  37 #include <unistd.h>
  38 #include <fcntl.h>
  39 #include <pwd.h>
  40 #include <grp.h>
  41 #include <dirent.h>
  42 #include <sys/time.h>
  43 #include <sys/select.h>
  44 #include <errno.h>
  45 #include <sys/stat.h>
  46
  47 #include <apti18n.h>
  48                                                                         /*}}}*/
  49
  50 using namespace std;
  51
  52 // Acquire::pkgAcquire - Constructor                                    /*{{{*/
  53 // ---------------------------------------------------------------------
  54 /* We grab some runtime state from the configuration space */
  55 pkgAcquire::pkgAcquire() : LockFD(-1), d(NULL), Queues(0), Workers(0), Configs(0), Log(NULL), ToFetch(0),
  56                            Debug(_config->FindB("Debug::pkgAcquire",false)),
  57                            Running(false)
  58 {
  59    Initialize();
  60 }
  61 pkgAcquire::pkgAcquire(pkgAcquireStatus *Progress) : LockFD(-1), d(NULL), Queues(0), Workers(0),
  62                            Configs(0), Log(NULL), ToFetch(0),
  63                            Debug(_config->FindB("Debug::pkgAcquire",false)),
  64                            Running(false)
  65 {
  66    Initialize();
  67    SetLog(Progress);
  68 }
  69 void pkgAcquire::Initialize()
  70 {
  71    string const Mode = _config->Find("Acquire::Queue-Mode","host");
  72    if (strcasecmp(Mode.c_str(),"host") == 0)
  73       QueueMode = QueueHost;
  74    if (strcasecmp(Mode.c_str(),"access") == 0)
  75       QueueMode = QueueAccess;
  76
  77    // chown the auth.conf file as it will be accessed by our methods
  78    std::string const SandboxUser = _config->Find("APT::Sandbox::User");
  79    if (getuid() == 0 && SandboxUser.empty() == false) // if we aren't root, we can't chown, so don't try it
  80    {
  81       struct passwd const * const pw = getpwnam(SandboxUser.c_str());
  82       struct group const * const gr = getgrnam("root");
  83       if (pw != NULL && gr != NULL)
  84       {
  85          std::string const AuthConf = _config->FindFile("Dir::Etc::netrc");
  86          if(AuthConf.empty() == false && RealFileExists(AuthConf) &&
  87                chown(AuthConf.c_str(), pw->pw_uid, gr->gr_gid) != 0)
  88             _error->WarningE("SetupAPTPartialDirectory", "chown to %s:root of file %s failed", SandboxUser.c_str(), AuthConf.c_str());
  89       }
  90    }
  91 }
  92                                                                         /*}}}*/
  93 // Acquire::GetLock - lock directory and prepare for action             /*{{{*/
  94 static bool SetupAPTPartialDirectory(std::string const &grand, std::string const &parent)
  95 {
  96    std::string const partial = parent + "partial";
  97    mode_t const mode = umask(S_IWGRP | S_IWOTH);
  98    bool const creation_fail = (CreateAPTDirectoryIfNeeded(grand, partial) == false &&
  99          CreateAPTDirectoryIfNeeded(parent, partial) == false);
 100    umask(mode);
 101    if (creation_fail == true)
 102       return false;
 103
 104    std::string const SandboxUser = _config->Find("APT::Sandbox::User");
 105    if (getuid() == 0 && SandboxUser.empty() == false) // if we aren't root, we can't chown, so don't try it
 106    {
 107       struct passwd const * const pw = getpwnam(SandboxUser.c_str());
 108       struct group const * const gr = getgrnam("root");
 109       if (pw != NULL && gr != NULL)
 110       {
 111          // chown the partial dir
 112          if(chown(partial.c_str(), pw->pw_uid, gr->gr_gid) != 0)
 113             _error->WarningE("SetupAPTPartialDirectory", "chown to %s:root of directory %s failed", SandboxUser.c_str(), partial.c_str());
 114       }
 115    }
 116    if (chmod(partial.c_str(), 0700) != 0)
 117       _error->WarningE("SetupAPTPartialDirectory", "chmod 0700 of directory %s failed", partial.c_str());
 118
 119    return true;
 120 }
 121 bool pkgAcquire::Setup(pkgAcquireStatus *Progress, string const &Lock)
 122 {
 123    Log = Progress;
 124    if (Lock.empty())
 125    {
 126       string const listDir = _config->FindDir("Dir::State::lists");
 127       if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir) == false)
 128          return _error->Errno("Acquire", _("List directory %spartial is missing."), listDir.c_str());
 129       string const archivesDir = _config->FindDir("Dir::Cache::Archives");
 130       if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir) == false)
 131          return _error->Errno("Acquire", _("Archives directory %spartial is missing."), archivesDir.c_str());
 132       return true;
 133    }
 134    return GetLock(Lock);
 135 }
 136 bool pkgAcquire::GetLock(std::string const &Lock)
 137 {
 138    if (Lock.empty() == true)
 139       return false;
 140
 141    // check for existence and possibly create auxiliary directories
 142    string const listDir = _config->FindDir("Dir::State::lists");
 143    string const archivesDir = _config->FindDir("Dir::Cache::Archives");
 144
 145    if (Lock == listDir)
 146    {
 147       if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir) == false)
 148          return _error->Errno("Acquire", _("List directory %spartial is missing."), listDir.c_str());
 149    }
 150    if (Lock == archivesDir)
 151    {
 152       if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir) == false)
 153          return _error->Errno("Acquire", _("Archives directory %spartial is missing."), archivesDir.c_str());
 154    }
 155
 156    if (_config->FindB("Debug::NoLocking", false) == true)
 157       return true;
 158
 159    // Lock the directory this acquire object will work in
 160    if (LockFD != -1)
 161       close(LockFD);
 162    LockFD = ::GetLock(flCombine(Lock, "lock"));
 163    if (LockFD == -1)
 164       return _error->Error(_("Unable to lock directory %s"), Lock.c_str());
 165
 166    return true;
 167 }
 168                                                                         /*}}}*/
 169 // Acquire::~pkgAcquire - Destructor                                    /*{{{*/
 170 // ---------------------------------------------------------------------
 171 /* Free our memory, clean up the queues (destroy the workers) */
 172 pkgAcquire::~pkgAcquire()
 173 {
 174    Shutdown();
 175
 176    if (LockFD != -1)
 177       close(LockFD);
 178
 179    while (Configs != 0)
 180    {
 181       MethodConfig *Jnk = Configs;
 182       Configs = Configs->Next;
 183       delete Jnk;
 184    }
 185 }
 186                                                                         /*}}}*/
 187 // Acquire::Shutdown - Clean out the acquire object                     /*{{{*/
 188 // ---------------------------------------------------------------------
 189 /* */
 190 void pkgAcquire::Shutdown()
 191 {
 192    while (Items.empty() == false)
 193    {
 194       if (Items[0]->Status == Item::StatFetching)
 195          Items[0]->Status = Item::StatError;
 196       delete Items[0];
 197    }
 198
 199    while (Queues != 0)
 200    {
 201       Queue *Jnk = Queues;
 202       Queues = Queues->Next;
 203       delete Jnk;
 204    }
 205 }
 206                                                                         /*}}}*/
 207 // Acquire::Add - Add a new item                                        /*{{{*/
 208 // ---------------------------------------------------------------------
 209 /* This puts an item on the acquire list. This list is mainly for tracking
 210    item status */
 211 void pkgAcquire::Add(Item *Itm)
 212 {
 213    Items.push_back(Itm);
 214 }
 215                                                                         /*}}}*/
 216 // Acquire::Remove - Remove a item                                      /*{{{*/
 217 // ---------------------------------------------------------------------
 218 /* Remove an item from the acquire list. This is usually not used.. */
 219 void pkgAcquire::Remove(Item *Itm)
 220 {
 221    Dequeue(Itm);
 222
 223    for (ItemIterator I = Items.begin(); I != Items.end();)
 224    {
 225       if (*I == Itm)
 226       {
 227          Items.erase(I);
 228          I = Items.begin();
 229       }
 230       else
 231          ++I;
 232    }
 233 }
 234                                                                         /*}}}*/
 235 // Acquire::Add - Add a worker                                          /*{{{*/
 236 // ---------------------------------------------------------------------
 237 /* A list of workers is kept so that the select loop can direct their FD
 238    usage. */
 239 void pkgAcquire::Add(Worker *Work)
 240 {
 241    Work->NextAcquire = Workers;
 242    Workers = Work;
 243 }
 244                                                                         /*}}}*/
 245 // Acquire::Remove - Remove a worker                                    /*{{{*/
 246 // ---------------------------------------------------------------------
 247 /* A worker has died. This can not be done while the select loop is running
 248    as it would require that RunFds could handling a changing list state and
 249    it can't.. */
 250 void pkgAcquire::Remove(Worker *Work)
 251 {
 252    if (Running == true)
 253       abort();
 254
 255    Worker **I = &Workers;
 256    for (; *I != 0;)
 257    {
 258       if (*I == Work)
 259          *I = (*I)->NextAcquire;
 260       else
 261          I = &(*I)->NextAcquire;
 262    }
 263 }
 264                                                                         /*}}}*/
 265 // Acquire::Enqueue - Queue an URI for fetching                         /*{{{*/
 266 // ---------------------------------------------------------------------
 267 /* This is the entry point for an item. An item calls this function when
 268    it is constructed which creates a queue (based on the current queue
 269    mode) and puts the item in that queue. If the system is running then
 270    the queue might be started. */
 271 void pkgAcquire::Enqueue(ItemDesc &Item)
 272 {
 273    // Determine which queue to put the item in
 274    const MethodConfig *Config;
 275    string Name = QueueName(Item.URI,Config);
 276    if (Name.empty() == true)
 277       return;
 278
 279    // Find the queue structure
 280    Queue *I = Queues;
 281    for (; I != 0 && I->Name != Name; I = I->Next);
 282    if (I == 0)
 283    {
 284       I = new Queue(Name,this);
 285       I->Next = Queues;
 286       Queues = I;
 287
 288       if (Running == true)
 289          I->Startup();
 290    }
 291
 292    // See if this is a local only URI
 293    if (Config->LocalOnly == true && Item.Owner->Complete == false)
 294       Item.Owner->Local = true;
 295    Item.Owner->Status = Item::StatIdle;
 296
 297    // Queue it into the named queue
 298    if(I->Enqueue(Item))
 299       ToFetch++;
 300
 301    // Some trace stuff
 302    if (Debug == true)
 303    {
 304       clog << "Fetching " << Item.URI << endl;
 305       clog << " to " << Item.Owner->DestFile << endl;
 306       clog << " Queue is: " << Name << endl;
 307    }
 308 }
 309                                                                         /*}}}*/
 310 // Acquire::Dequeue - Remove an item from all queues                    /*{{{*/
 311 // ---------------------------------------------------------------------
 312 /* This is called when an item is finished being fetched. It removes it
 313    from all the queues */
 314 void pkgAcquire::Dequeue(Item *Itm)
 315 {
 316    Queue *I = Queues;
 317    bool Res = false;
 318    if (Debug == true)
 319       clog << "Dequeuing " << Itm->DestFile << endl;
 320
 321    for (; I != 0; I = I->Next)
 322    {
 323       if (I->Dequeue(Itm))
 324       {
 325          Res = true;
 326          if (Debug == true)
 327             clog << "Dequeued from " << I->Name << endl;
 328       }
 329    }
 330
 331    if (Res == true)
 332       ToFetch--;
 333 }
 334                                                                         /*}}}*/
 335 // Acquire::QueueName - Return the name of the queue for this URI       /*{{{*/
 336 // ---------------------------------------------------------------------
 337 /* The string returned depends on the configuration settings and the
 338    method parameters. Given something like http://foo.org/bar it can
 339    return http://foo.org or http */
 340 string pkgAcquire::QueueName(string Uri,MethodConfig const *&Config)
 341 {
 342    URI U(Uri);
 343
 344    Config = GetConfig(U.Access);
 345    if (Config == 0)
 346       return string();
 347
 348    /* Single-Instance methods get exactly one queue per URI. This is
 349       also used for the Access queue method  */
 350    if (Config->SingleInstance == true || QueueMode == QueueAccess)
 351       return U.Access;
 352
 353    string AccessSchema = U.Access + ':',
 354         FullQueueName = AccessSchema + U.Host;
 355    unsigned int Instances = 0, SchemaLength = AccessSchema.length();
 356
 357    Queue *I = Queues;
 358    for (; I != 0; I = I->Next) {
 359       // if the queue already exists, re-use it
 360       if (I->Name == FullQueueName)
 361          return FullQueueName;
 362
 363       if (I->Name.compare(0, SchemaLength, AccessSchema) == 0)
 364          Instances++;
 365    }
 366
 367    if (Debug) {
 368       clog << "Found " << Instances << " instances of " << U.Access << endl;
 369    }
 370
 371    if (Instances >= (unsigned int)_config->FindI("Acquire::QueueHost::Limit",10))
 372       return U.Access;
 373
 374    return FullQueueName;
 375 }
 376                                                                         /*}}}*/
 377 // Acquire::GetConfig - Fetch the configuration information             /*{{{*/
 378 // ---------------------------------------------------------------------
 379 /* This locates the configuration structure for an access method. If
 380    a config structure cannot be found a Worker will be created to
 381    retrieve it */
 382 pkgAcquire::MethodConfig *pkgAcquire::GetConfig(string Access)
 383 {
 384    // Search for an existing config
 385    MethodConfig *Conf;
 386    for (Conf = Configs; Conf != 0; Conf = Conf->Next)
 387       if (Conf->Access == Access)
 388          return Conf;
 389
 390    // Create the new config class
 391    Conf = new MethodConfig;
 392    Conf->Access = Access;
 393    Conf->Next = Configs;
 394    Configs = Conf;
 395
 396    // Create the worker to fetch the configuration
 397    Worker Work(Conf);
 398    if (Work.Start() == false)
 399       return 0;
 400
 401    /* if a method uses DownloadLimit, we switch to SingleInstance mode */
 402    if(_config->FindI("Acquire::"+Access+"::Dl-Limit",0) > 0)
 403       Conf->SingleInstance = true;
 404
 405    return Conf;
 406 }
 407                                                                         /*}}}*/
 408 // Acquire::SetFds - Deal with readable FDs                             /*{{{*/
 409 // ---------------------------------------------------------------------
 410 /* Collect FDs that have activity monitors into the fd sets */
 411 void pkgAcquire::SetFds(int &Fd,fd_set *RSet,fd_set *WSet)
 412 {
 413    for (Worker *I = Workers; I != 0; I = I->NextAcquire)
 414    {
 415       if (I->InReady == true && I->InFd >= 0)
 416       {
 417          if (Fd < I->InFd)
 418             Fd = I->InFd;
 419          FD_SET(I->InFd,RSet);
 420       }
 421       if (I->OutReady == true && I->OutFd >= 0)
 422       {
 423          if (Fd < I->OutFd)
 424             Fd = I->OutFd;
 425          FD_SET(I->OutFd,WSet);
 426       }
 427    }
 428 }
 429                                                                         /*}}}*/
 430 // Acquire::RunFds - Deal with active FDs                               /*{{{*/
 431 // ---------------------------------------------------------------------
 432 /* Dispatch active FDs over to the proper workers. It is very important
 433    that a worker never be erased while this is running! The queue class
 434    should never erase a worker except during shutdown processing. */
 435 void pkgAcquire::RunFds(fd_set *RSet,fd_set *WSet)
 436 {
 437    for (Worker *I = Workers; I != 0; I = I->NextAcquire)
 438    {
 439       if (I->InFd >= 0 && FD_ISSET(I->InFd,RSet) != 0)
 440          I->InFdReady();
 441       if (I->OutFd >= 0 && FD_ISSET(I->OutFd,WSet) != 0)
 442          I->OutFdReady();
 443    }
 444 }
 445                                                                         /*}}}*/
 446 // Acquire::Run - Run the fetch sequence                                /*{{{*/
 447 // ---------------------------------------------------------------------
 448 /* This runs the queues. It manages a select loop for all of the
 449    Worker tasks. The workers interact with the queues and items to
 450    manage the actual fetch. */
 451 static void CheckDropPrivsMustBeDisabled(pkgAcquire const &Fetcher)
 452 {
 453    if(getuid() != 0)
 454       return;
 455
 456    std::string SandboxUser = _config->Find("APT::Sandbox::User");
 457    if (SandboxUser.empty())
 458       return;
 459
 460    struct passwd const * const pw = getpwnam(SandboxUser.c_str());
 461    if (pw == NULL)
 462       return;
 463
 464    gid_t const old_euid = geteuid();
 465    gid_t const old_egid = getegid();
 466    if (setegid(pw->pw_gid) != 0)
 467       _error->Errno("setegid", "setegid %u failed", pw->pw_gid);
 468    if (seteuid(pw->pw_uid) != 0)
 469       _error->Errno("seteuid", "seteuid %u failed", pw->pw_uid);
 470
 471    for (pkgAcquire::ItemCIterator I = Fetcher.ItemsBegin();
 472         I != Fetcher.ItemsEnd(); ++I)
 473    {
 474       std::string filename = (*I)->DestFile;
 475       if (filename.empty())
 476          continue;
 477
 478       // no need to drop privileges for a complete file
 479       if ((*I)->Complete == true)
 480          continue;
 481
 482       // we check directory instead of file as the file might or might not
 483       // exist already as a link or not which complicates everything…
 484       std::string dirname = flNotFile(filename);
 485       if (unlikely(dirname.empty()))
 486          continue;
 487       // translate relative to absolute for DirectoryExists
 488       // FIXME: What about ../ and ./../ ?
 489       if (dirname.substr(0,2) == "./")
 490          dirname = SafeGetCWD() + dirname.substr(2);
 491
 492       if (DirectoryExists(dirname))
 493          ;
 494       else
 495          continue; // assume it is created correctly by the acquire system
 496
 497       if (faccessat(-1, dirname.c_str(), R_OK | W_OK | X_OK, AT_EACCESS | AT_SYMLINK_NOFOLLOW) != 0)
 498       {
 499          _error->WarningE("pkgAcquire::Run", _("Can't drop privileges for downloading as file '%s' couldn't be accessed by user '%s'."),
 500                filename.c_str(), SandboxUser.c_str());
 501          _config->Set("APT::Sandbox::User", "");
 502          break;
 503       }
 504    }
 505
 506    if (seteuid(old_euid) != 0)
 507       _error->Errno("seteuid", "seteuid %u failed", old_euid);
 508    if (setegid(old_egid) != 0)
 509       _error->Errno("setegid", "setegid %u failed", old_egid);
 510 }
 511 pkgAcquire::RunResult pkgAcquire::Run(int PulseIntervall)
 512 {
 513    _error->PushToStack();
 514    CheckDropPrivsMustBeDisabled(*this);
 515
 516    Running = true;
 517
 518    for (Queue *I = Queues; I != 0; I = I->Next)
 519       I->Startup();
 520
 521    if (Log != 0)
 522       Log->Start();
 523
 524    bool WasCancelled = false;
 525
 526    // Run till all things have been acquired
 527    struct timeval tv;
 528    tv.tv_sec = 0;
 529    tv.tv_usec = PulseIntervall;
 530    while (ToFetch > 0)
 531    {
 532       fd_set RFds;
 533       fd_set WFds;
 534       int Highest = 0;
 535       FD_ZERO(&RFds);
 536       FD_ZERO(&WFds);
 537       SetFds(Highest,&RFds,&WFds);
 538
 539       int Res;
 540       do
 541       {
 542          Res = select(Highest+1,&RFds,&WFds,0,&tv);
 543       }
 544       while (Res < 0 && errno == EINTR);
 545
 546       if (Res < 0)
 547       {
 548          _error->Errno("select","Select has failed");
 549          break;
 550       }
 551
 552       RunFds(&RFds,&WFds);
 553
 554       // Timeout, notify the log class
 555       if (Res == 0 || (Log != 0 && Log->Update == true))
 556       {
 557          tv.tv_usec = PulseIntervall;
 558          for (Worker *I = Workers; I != 0; I = I->NextAcquire)
 559             I->Pulse();
 560          if (Log != 0 && Log->Pulse(this) == false)
 561          {
 562             WasCancelled = true;
 563             break;
 564          }
 565       }
 566    }
 567
 568    if (Log != 0)
 569       Log->Stop();
 570
 571    // Shut down the acquire bits
 572    Running = false;
 573    for (Queue *I = Queues; I != 0; I = I->Next)
 574       I->Shutdown(false);
 575
 576    // Shut down the items
 577    for (ItemIterator I = Items.begin(); I != Items.end(); ++I)
 578       (*I)->Finished();
 579
 580    bool const newError = _error->PendingError();
 581    _error->MergeWithStack();
 582    if (newError)
 583       return Failed;
 584    if (WasCancelled)
 585       return Cancelled;
 586    return Continue;
 587 }
 588                                                                         /*}}}*/
 589 // Acquire::Bump - Called when an item is dequeued                      /*{{{*/
 590 // ---------------------------------------------------------------------
 591 /* This routine bumps idle queues in hopes that they will be able to fetch
 592    the dequeued item */
 593 void pkgAcquire::Bump()
 594 {
 595    for (Queue *I = Queues; I != 0; I = I->Next)
 596       I->Bump();
 597 }
 598                                                                         /*}}}*/
 599 // Acquire::WorkerStep - Step to the next worker                        /*{{{*/
 600 // ---------------------------------------------------------------------
 601 /* Not inlined to advoid including acquire-worker.h */
 602 pkgAcquire::Worker *pkgAcquire::WorkerStep(Worker *I)
 603 {
 604    return I->NextAcquire;
 605 }
 606                                                                         /*}}}*/
 607 // Acquire::Clean - Cleans a directory                                  /*{{{*/
 608 // ---------------------------------------------------------------------
 609 /* This is a bit simplistic, it looks at every file in the dir and sees
 610    if it is part of the download set. */
 611 bool pkgAcquire::Clean(string Dir)
 612 {
 613    // non-existing directories are by definition clean…
 614    if (DirectoryExists(Dir) == false)
 615       return true;
 616
 617    if(Dir == "/")
 618       return _error->Error(_("Clean of %s is not supported"), Dir.c_str());
 619
 620    DIR *D = opendir(Dir.c_str());
 621    if (D == 0)
 622       return _error->Errno("opendir",_("Unable to read %s"),Dir.c_str());
 623
 624    string StartDir = SafeGetCWD();
 625    if (chdir(Dir.c_str()) != 0)
 626    {
 627       closedir(D);
 628       return _error->Errno("chdir",_("Unable to change to %s"),Dir.c_str());
 629    }
 630
 631    for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D))
 632    {
 633       // Skip some files..
 634       if (strcmp(Dir->d_name,"lock") == 0 ||
 635           strcmp(Dir->d_name,"partial") == 0 ||
 636           strcmp(Dir->d_name,".") == 0 ||
 637           strcmp(Dir->d_name,"..") == 0)
 638          continue;
 639
 640       // Look in the get list
 641       ItemCIterator I = Items.begin();
 642       for (; I != Items.end(); ++I)
 643          if (flNotDir((*I)->DestFile) == Dir->d_name)
 644             break;
 645
 646       // Nothing found, nuke it
 647       if (I == Items.end())
 648          unlink(Dir->d_name);
 649    };
 650
 651    closedir(D);
 652    if (chdir(StartDir.c_str()) != 0)
 653       return _error->Errno("chdir",_("Unable to change to %s"),StartDir.c_str());
 654    return true;
 655 }
 656                                                                         /*}}}*/
 657 // Acquire::TotalNeeded - Number of bytes to fetch                      /*{{{*/
 658 // ---------------------------------------------------------------------
 659 /* This is the total number of bytes needed */
 660 APT_PURE unsigned long long pkgAcquire::TotalNeeded()
 661 {
 662    return std::accumulate(ItemsBegin(), ItemsEnd(), 0,
 663       [](unsigned long long const T, Item const * const I) {
 664          return T + I->FileSize;
 665    });
 666 }
 667                                                                         /*}}}*/
 668 // Acquire::FetchNeeded - Number of bytes needed to get                 /*{{{*/
 669 // ---------------------------------------------------------------------
 670 /* This is the number of bytes that is not local */
 671 APT_PURE unsigned long long pkgAcquire::FetchNeeded()
 672 {
 673    return std::accumulate(ItemsBegin(), ItemsEnd(), 0,
 674       [](unsigned long long const T, Item const * const I) {
 675          if (I->Local == false)
 676             return T + I->FileSize;
 677          else
 678             return T;
 679    });
 680 }
 681                                                                         /*}}}*/
 682 // Acquire::PartialPresent - Number of partial bytes we already have    /*{{{*/
 683 // ---------------------------------------------------------------------
 684 /* This is the number of bytes that is not local */
 685 APT_PURE unsigned long long pkgAcquire::PartialPresent()
 686 {
 687    return std::accumulate(ItemsBegin(), ItemsEnd(), 0,
 688       [](unsigned long long const T, Item const * const I) {
 689          if (I->Local == false)
 690             return T + I->PartialSize;
 691          else
 692             return T;
 693    });
 694 }
 695                                                                         /*}}}*/
 696 // Acquire::UriBegin - Start iterator for the uri list                  /*{{{*/
 697 // ---------------------------------------------------------------------
 698 /* */
 699 pkgAcquire::UriIterator pkgAcquire::UriBegin()
 700 {
 701    return UriIterator(Queues);
 702 }
 703                                                                         /*}}}*/
 704 // Acquire::UriEnd - End iterator for the uri list                      /*{{{*/
 705 // ---------------------------------------------------------------------
 706 /* */
 707 pkgAcquire::UriIterator pkgAcquire::UriEnd()
 708 {
 709    return UriIterator(0);
 710 }
 711                                                                         /*}}}*/
 712 // Acquire::MethodConfig::MethodConfig - Constructor                    /*{{{*/
 713 // ---------------------------------------------------------------------
 714 /* */
 715 pkgAcquire::MethodConfig::MethodConfig() : d(NULL), Next(0), SingleInstance(false),
 716    Pipeline(false), SendConfig(false), LocalOnly(false), NeedsCleanup(false),
 717    Removable(false)
 718 {
 719 }
 720                                                                         /*}}}*/
 721 // Queue::Queue - Constructor                                           /*{{{*/
 722 // ---------------------------------------------------------------------
 723 /* */
 724 pkgAcquire::Queue::Queue(string const &name,pkgAcquire * const owner) : d(NULL), Next(0),
 725    Name(name), Items(0), Workers(0), Owner(owner), PipeDepth(0), MaxPipeDepth(1)
 726 {
 727 }
 728                                                                         /*}}}*/
 729 // Queue::~Queue - Destructor                                           /*{{{*/
 730 // ---------------------------------------------------------------------
 731 /* */
 732 pkgAcquire::Queue::~Queue()
 733 {
 734    Shutdown(true);
 735
 736    while (Items != 0)
 737    {
 738       QItem *Jnk = Items;
 739       Items = Items->Next;
 740       delete Jnk;
 741    }
 742 }
 743                                                                         /*}}}*/
 744 // Queue::Enqueue - Queue an item to the queue                          /*{{{*/
 745 // ---------------------------------------------------------------------
 746 /* */
 747 bool pkgAcquire::Queue::Enqueue(ItemDesc &Item)
 748 {
 749    QItem **I = &Items;
 750    // move to the end of the queue and check for duplicates here
 751    HashStringList const hsl = Item.Owner->GetExpectedHashes();
 752    for (; *I != 0; I = &(*I)->Next)
 753       if (Item.URI == (*I)->URI || hsl == (*I)->Owner->GetExpectedHashes())
 754       {
 755          if (_config->FindB("Debug::pkgAcquire::Worker",false) == true)
 756             std::cerr << " @ Queue: Action combined for " << Item.URI << " and " << (*I)->URI << std::endl;
 757          (*I)->Owners.push_back(Item.Owner);
 758          Item.Owner->Status = (*I)->Owner->Status;
 759          return false;
 760       }
 761
 762    // Create a new item
 763    QItem *Itm = new QItem;
 764    *Itm = Item;
 765    Itm->Next = 0;
 766    *I = Itm;
 767
 768    Item.Owner->QueueCounter++;
 769    if (Items->Next == 0)
 770       Cycle();
 771    return true;
 772 }
 773                                                                         /*}}}*/
 774 // Queue::Dequeue - Remove an item from the queue                       /*{{{*/
 775 // ---------------------------------------------------------------------
 776 /* We return true if we hit something */
 777 bool pkgAcquire::Queue::Dequeue(Item *Owner)
 778 {
 779    if (Owner->Status == pkgAcquire::Item::StatFetching)
 780       return _error->Error("Tried to dequeue a fetching object");
 781
 782    bool Res = false;
 783
 784    QItem **I = &Items;
 785    for (; *I != 0;)
 786    {
 787       if (Owner == (*I)->Owner)
 788       {
 789          QItem *Jnk= *I;
 790          *I = (*I)->Next;
 791          Owner->QueueCounter--;
 792          delete Jnk;
 793          Res = true;
 794       }
 795       else
 796          I = &(*I)->Next;
 797    }
 798
 799    return Res;
 800 }
 801                                                                         /*}}}*/
 802 // Queue::Startup - Start the worker processes                          /*{{{*/
 803 // ---------------------------------------------------------------------
 804 /* It is possible for this to be called with a pre-existing set of
 805    workers. */
 806 bool pkgAcquire::Queue::Startup()
 807 {
 808    if (Workers == 0)
 809    {
 810       URI U(Name);
 811       pkgAcquire::MethodConfig *Cnf = Owner->GetConfig(U.Access);
 812       if (Cnf == 0)
 813          return false;
 814
 815       Workers = new Worker(this,Cnf,Owner->Log);
 816       Owner->Add(Workers);
 817       if (Workers->Start() == false)
 818          return false;
 819
 820       /* When pipelining we commit 10 items. This needs to change when we
 821          added other source retry to have cycle maintain a pipeline depth
 822          on its own. */
 823       if (Cnf->Pipeline == true)
 824          MaxPipeDepth = _config->FindI("Acquire::Max-Pipeline-Depth",10);
 825       else
 826          MaxPipeDepth = 1;
 827    }
 828
 829    return Cycle();
 830 }
 831                                                                         /*}}}*/
 832 // Queue::Shutdown - Shutdown the worker processes                      /*{{{*/
 833 // ---------------------------------------------------------------------
 834 /* If final is true then all workers are eliminated, otherwise only workers
 835    that do not need cleanup are removed */
 836 bool pkgAcquire::Queue::Shutdown(bool Final)
 837 {
 838    // Delete all of the workers
 839    pkgAcquire::Worker **Cur = &Workers;
 840    while (*Cur != 0)
 841    {
 842       pkgAcquire::Worker *Jnk = *Cur;
 843       if (Final == true || Jnk->GetConf()->NeedsCleanup == false)
 844       {
 845          *Cur = Jnk->NextQueue;
 846          Owner->Remove(Jnk);
 847          delete Jnk;
 848       }
 849       else
 850          Cur = &(*Cur)->NextQueue;
 851    }
 852
 853    return true;
 854 }
 855                                                                         /*}}}*/
 856 // Queue::FindItem - Find a URI in the item list                        /*{{{*/
 857 // ---------------------------------------------------------------------
 858 /* */
 859 pkgAcquire::Queue::QItem *pkgAcquire::Queue::FindItem(string URI,pkgAcquire::Worker *Owner)
 860 {
 861    for (QItem *I = Items; I != 0; I = I->Next)
 862       if (I->URI == URI && I->Worker == Owner)
 863          return I;
 864    return 0;
 865 }
 866                                                                         /*}}}*/
 867 // Queue::ItemDone - Item has been completed                            /*{{{*/
 868 // ---------------------------------------------------------------------
 869 /* The worker signals this which causes the item to be removed from the
 870    queue. If this is the last queue instance then it is removed from the
 871    main queue too.*/
 872 bool pkgAcquire::Queue::ItemDone(QItem *Itm)
 873 {
 874    PipeDepth--;
 875    for (QItem::owner_iterator O = Itm->Owners.begin(); O != Itm->Owners.end(); ++O)
 876    {
 877       if ((*O)->Status == pkgAcquire::Item::StatFetching)
 878          (*O)->Status = pkgAcquire::Item::StatDone;
 879    }
 880
 881    if (Itm->Owner->QueueCounter <= 1)
 882       Owner->Dequeue(Itm->Owner);
 883    else
 884    {
 885       Dequeue(Itm->Owner);
 886       Owner->Bump();
 887    }
 888
 889    return Cycle();
 890 }
 891                                                                         /*}}}*/
 892 // Queue::Cycle - Queue new items into the method                       /*{{{*/
 893 // ---------------------------------------------------------------------
 894 /* This locates a new idle item and sends it to the worker. If pipelining
 895    is enabled then it keeps the pipe full. */
 896 bool pkgAcquire::Queue::Cycle()
 897 {
 898    if (Items == 0 || Workers == 0)
 899       return true;
 900
 901    if (PipeDepth < 0)
 902       return _error->Error("Pipedepth failure");
 903
 904    // Look for a queable item
 905    QItem *I = Items;
 906    while (PipeDepth < (signed)MaxPipeDepth)
 907    {
 908       for (; I != 0; I = I->Next)
 909          if (I->Owner->Status == pkgAcquire::Item::StatIdle)
 910             break;
 911
 912       // Nothing to do, queue is idle.
 913       if (I == 0)
 914          return true;
 915
 916       I->Worker = Workers;
 917       for (auto const &O: I->Owners)
 918          O->Status = pkgAcquire::Item::StatFetching;
 919       PipeDepth++;
 920       if (Workers->QueueItem(I) == false)
 921          return false;
 922    }
 923
 924    return true;
 925 }
 926                                                                         /*}}}*/
 927 // Queue::Bump - Fetch any pending objects if we are idle               /*{{{*/
 928 // ---------------------------------------------------------------------
 929 /* This is called when an item in multiple queues is dequeued */
 930 void pkgAcquire::Queue::Bump()
 931 {
 932    Cycle();
 933 }
 934                                                                         /*}}}*/
 935 HashStringList pkgAcquire::Queue::QItem::GetExpectedHashes() const      /*{{{*/
 936 {
 937    /* each Item can have multiple owners and each owner might have different
 938       hashes, even if that is unlikely in practice and if so at least some
 939       owners will later fail. There is one situation through which is not a
 940       failure and still needs this handling: Two owners who expect the same
 941       file, but one owner only knows the SHA1 while the other only knows SHA256. */
 942    HashStringList superhsl;
 943    for (pkgAcquire::Queue::QItem::owner_iterator O = Owners.begin(); O != Owners.end(); ++O)
 944    {
 945       HashStringList const hsl = (*O)->GetExpectedHashes();
 946       if (hsl.usable() == false)
 947          continue;
 948       if (superhsl.usable() == false)
 949          superhsl = hsl;
 950       else
 951       {
 952          // we merge both lists - if we find disagreement send no hashes
 953          HashStringList::const_iterator hs = hsl.begin();
 954          for (; hs != hsl.end(); ++hs)
 955             if (superhsl.push_back(*hs) == false)
 956                break;
 957          if (hs != hsl.end())
 958          {
 959             superhsl.clear();
 960             break;
 961          }
 962       }
 963    }
 964    return superhsl;
 965 }
 966                                                                         /*}}}*/
 967 APT_PURE unsigned long long pkgAcquire::Queue::QItem::GetMaximumSize() const    /*{{{*/
 968 {
 969    unsigned long long Maximum = std::numeric_limits<unsigned long long>::max();
 970    for (auto const &O: Owners)
 971    {
 972       if (O->FileSize == 0)
 973          continue;
 974       Maximum = std::min(Maximum, O->FileSize);
 975    }
 976    if (Maximum == std::numeric_limits<unsigned long long>::max())
 977       return 0;
 978    return Maximum;
 979 }
 980                                                                         /*}}}*/
 981 void pkgAcquire::Queue::QItem::SyncDestinationFiles() const             /*{{{*/
 982 {
 983    /* ensure that the first owner has the best partial file of all and
 984       the rest have (potentially dangling) symlinks to it so that
 985       everything (like progress reporting) finds it easily */
 986    std::string superfile = Owner->DestFile;
 987    off_t supersize = 0;
 988    for (pkgAcquire::Queue::QItem::owner_iterator O = Owners.begin(); O != Owners.end(); ++O)
 989    {
 990       if ((*O)->DestFile == superfile)
 991          continue;
 992       struct stat file;
 993       if (lstat((*O)->DestFile.c_str(),&file) == 0)
 994       {
 995          if ((file.st_mode & S_IFREG) == 0)
 996             unlink((*O)->DestFile.c_str());
 997          else if (supersize < file.st_size)
 998          {
 999             supersize = file.st_size;
1000             unlink(superfile.c_str());
1001             rename((*O)->DestFile.c_str(), superfile.c_str());
1002          }
1003          else
1004             unlink((*O)->DestFile.c_str());
1005          if (symlink(superfile.c_str(), (*O)->DestFile.c_str()) != 0)
1006          {
1007             ; // not a problem per-se and no real alternative
1008          }
1009       }
1010    }
1011 }
1012                                                                         /*}}}*/
1013 std::string pkgAcquire::Queue::QItem::Custom600Headers() const          /*{{{*/
1014 {
1015    /* The others are relatively easy to merge, but this one?
1016       Lets not merge and see how far we can run with it…
1017       Likely, nobody will ever notice as all the items will
1018       be of the same class and hence generate the same headers. */
1019    return Owner->Custom600Headers();
1020 }
1021                                                                         /*}}}*/
1022
1023 // AcquireStatus::pkgAcquireStatus - Constructor                        /*{{{*/
1024 // ---------------------------------------------------------------------
1025 /* */
1026 pkgAcquireStatus::pkgAcquireStatus() : d(NULL), Percent(-1), Update(true), MorePulses(false)
1027 {
1028    Start();
1029 }
1030                                                                         /*}}}*/
1031 // AcquireStatus::Pulse - Called periodically                           /*{{{*/
1032 // ---------------------------------------------------------------------
1033 /* This computes some internal state variables for the derived classes to
1034    use. It generates the current downloaded bytes and total bytes to download
1035    as well as the current CPS estimate. */
1036 bool pkgAcquireStatus::Pulse(pkgAcquire *Owner)
1037 {
1038    TotalBytes = 0;
1039    CurrentBytes = 0;
1040    TotalItems = 0;
1041    CurrentItems = 0;
1042
1043    // Compute the total number of bytes to fetch
1044    unsigned int Unknown = 0;
1045    unsigned int Count = 0;
1046    bool UnfetchedReleaseFiles = false;
1047    for (pkgAcquire::ItemCIterator I = Owner->ItemsBegin();
1048         I != Owner->ItemsEnd();
1049         ++I, ++Count)
1050    {
1051       TotalItems++;
1052       if ((*I)->Status == pkgAcquire::Item::StatDone)
1053          ++CurrentItems;
1054
1055       // Totally ignore local items
1056       if ((*I)->Local == true)
1057          continue;
1058
1059       // see if the method tells us to expect more
1060       TotalItems += (*I)->ExpectedAdditionalItems;
1061
1062       // check if there are unfetched Release files
1063       if ((*I)->Complete == false && (*I)->ExpectedAdditionalItems > 0)
1064          UnfetchedReleaseFiles = true;
1065
1066       TotalBytes += (*I)->FileSize;
1067       if ((*I)->Complete == true)
1068          CurrentBytes += (*I)->FileSize;
1069       if ((*I)->FileSize == 0 && (*I)->Complete == false)
1070          ++Unknown;
1071    }
1072
1073    // Compute the current completion
1074    unsigned long long ResumeSize = 0;
1075    for (pkgAcquire::Worker *I = Owner->WorkersBegin(); I != 0;
1076         I = Owner->WorkerStep(I))
1077    {
1078       if (I->CurrentItem != 0 && I->CurrentItem->Owner->Complete == false)
1079       {
1080          CurrentBytes += I->CurrentSize;
1081          ResumeSize += I->ResumePoint;
1082
1083          // Files with unknown size always have 100% completion
1084          if (I->CurrentItem->Owner->FileSize == 0 &&
1085              I->CurrentItem->Owner->Complete == false)
1086             TotalBytes += I->CurrentSize;
1087       }
1088    }
1089
1090    // Normalize the figures and account for unknown size downloads
1091    if (TotalBytes <= 0)
1092       TotalBytes = 1;
1093    if (Unknown == Count)
1094       TotalBytes = Unknown;
1095
1096    // Wha?! Is not supposed to happen.
1097    if (CurrentBytes > TotalBytes)
1098       CurrentBytes = TotalBytes;
1099
1100    // debug
1101    if (_config->FindB("Debug::acquire::progress", false) == true)
1102       std::clog << " Bytes: "
1103                 << SizeToStr(CurrentBytes) << " / " << SizeToStr(TotalBytes)
1104                 << std::endl;
1105
1106    // Compute the CPS
1107    struct timeval NewTime;
1108    gettimeofday(&NewTime,0);
1109    if ((NewTime.tv_sec - Time.tv_sec == 6 && NewTime.tv_usec > Time.tv_usec) ||
1110        NewTime.tv_sec - Time.tv_sec > 6)
1111    {
1112       double Delta = NewTime.tv_sec - Time.tv_sec +
1113                      (NewTime.tv_usec - Time.tv_usec)/1000000.0;
1114
1115       // Compute the CPS value
1116       if (Delta < 0.01)
1117          CurrentCPS = 0;
1118       else
1119          CurrentCPS = ((CurrentBytes - ResumeSize) - LastBytes)/Delta;
1120       LastBytes = CurrentBytes - ResumeSize;
1121       ElapsedTime = (unsigned long long)Delta;
1122       Time = NewTime;
1123    }
1124
1125    double const OldPercent = Percent;
1126    // calculate the percentage, if we have too little data assume 1%
1127    if (TotalBytes > 0 && UnfetchedReleaseFiles)
1128       Percent = 0;
1129    else
1130       // use both files and bytes because bytes can be unreliable
1131       Percent = (0.8 * (CurrentBytes/float(TotalBytes)*100.0) +
1132                  0.2 * (CurrentItems/float(TotalItems)*100.0));
1133    double const DiffPercent = Percent - OldPercent;
1134    if (DiffPercent < 0.001 && _config->FindB("Acquire::Progress::Diffpercent", false) == true)
1135       return true;
1136
1137    int fd = _config->FindI("APT::Status-Fd",-1);
1138    if(fd > 0)
1139    {
1140       ostringstream status;
1141
1142       char msg[200];
1143       long i = CurrentItems < TotalItems ? CurrentItems + 1 : CurrentItems;
1144       unsigned long long ETA = 0;
1145       if(CurrentCPS > 0)
1146          ETA = (TotalBytes - CurrentBytes) / CurrentCPS;
1147
1148       // only show the ETA if it makes sense
1149       if (ETA > 0 && ETA < 172800 /* two days */ )
1150          snprintf(msg,sizeof(msg), _("Retrieving file %li of %li (%s remaining)"), i, TotalItems, TimeToStr(ETA).c_str());
1151       else
1152          snprintf(msg,sizeof(msg), _("Retrieving file %li of %li"), i, TotalItems);
1153
1154       // build the status str
1155       status << "dlstatus:" << i
1156              << ":"  << std::setprecision(3) << Percent
1157              << ":" << msg
1158              << endl;
1159
1160       std::string const dlstatus = status.str();
1161       FileFd::Write(fd, dlstatus.c_str(), dlstatus.size());
1162    }
1163
1164    return true;
1165 }
1166                                                                         /*}}}*/
1167 // AcquireStatus::Start - Called when the download is started           /*{{{*/
1168 // ---------------------------------------------------------------------
1169 /* We just reset the counters */
1170 void pkgAcquireStatus::Start()
1171 {
1172    gettimeofday(&Time,0);
1173    gettimeofday(&StartTime,0);
1174    LastBytes = 0;
1175    CurrentCPS = 0;
1176    CurrentBytes = 0;
1177    TotalBytes = 0;
1178    FetchedBytes = 0;
1179    ElapsedTime = 0;
1180    TotalItems = 0;
1181    CurrentItems = 0;
1182 }
1183                                                                         /*}}}*/
1184 // AcquireStatus::Stop - Finished downloading                           /*{{{*/
1185 // ---------------------------------------------------------------------
1186 /* This accurately computes the elapsed time and the total overall CPS. */
1187 void pkgAcquireStatus::Stop()
1188 {
1189    // Compute the CPS and elapsed time
1190    struct timeval NewTime;
1191    gettimeofday(&NewTime,0);
1192
1193    double Delta = NewTime.tv_sec - StartTime.tv_sec +
1194                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
1195
1196    // Compute the CPS value
1197    if (Delta < 0.01)
1198       CurrentCPS = 0;
1199    else
1200       CurrentCPS = FetchedBytes/Delta;
1201    LastBytes = CurrentBytes;
1202    ElapsedTime = (unsigned long long)Delta;
1203 }
1204                                                                         /*}}}*/
1205 // AcquireStatus::Fetched - Called when a byte set has been fetched     /*{{{*/
1206 // ---------------------------------------------------------------------
1207 /* This is used to get accurate final transfer rate reporting. */
1208 void pkgAcquireStatus::Fetched(unsigned long long Size,unsigned long long Resume)
1209 {
1210    FetchedBytes += Size - Resume;
1211 }
1212                                                                         /*}}}*/
1213
1214 pkgAcquire::UriIterator::UriIterator(pkgAcquire::Queue *Q) : d(NULL), CurQ(Q), CurItem(0)
1215 {
1216    while (CurItem == 0 && CurQ != 0)
1217    {
1218       CurItem = CurQ->Items;
1219       CurQ = CurQ->Next;
1220    }
1221 }
1222
1223 APT_CONST pkgAcquire::UriIterator::~UriIterator() {}
1224 APT_CONST pkgAcquire::MethodConfig::~MethodConfig() {}
1225 APT_CONST pkgAcquireStatus::~pkgAcquireStatus() {}