apt-pkg/acquire.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire.cc,v 1.50 2004/03/17 05:17:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire - File Acquiration
   7
   8    The core element for the schedule system is the concept of a named
   9    queue. Each queue is unique and each queue has a name derived from the
  10    URI. The degree of paralization can be controlled by how the queue
  11    name is derived from the URI.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <config.h>
  17
  18 #include <apt-pkg/acquire.h>
  19 #include <apt-pkg/acquire-item.h>
  20 #include <apt-pkg/acquire-worker.h>
  21 #include <apt-pkg/configuration.h>
  22 #include <apt-pkg/error.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/fileutl.h>
  25
  26 #include <algorithm>
  27 #include <string>
  28 #include <vector>
  29 #include <iostream>
  30 #include <sstream>
  31 #include <iomanip>
  32
  33 #include <stdio.h>
  34 #include <stdlib.h>
  35 #include <string.h>
  36 #include <unistd.h>
  37 #include <fcntl.h>
  38 #include <pwd.h>
  39 #include <grp.h>
  40 #include <dirent.h>
  41 #include <sys/time.h>
  42 #include <sys/select.h>
  43 #include <errno.h>
  44 #include <sys/stat.h>
  45
  46 #include <apti18n.h>
  47                                                                         /*}}}*/
  48
  49 using namespace std;
  50
  51 // Acquire::pkgAcquire - Constructor                                    /*{{{*/
  52 // ---------------------------------------------------------------------
  53 /* We grab some runtime state from the configuration space */
  54 pkgAcquire::pkgAcquire() : LockFD(-1), d(NULL), Queues(0), Workers(0), Configs(0), Log(NULL), ToFetch(0),
  55                            Debug(_config->FindB("Debug::pkgAcquire",false)),
  56                            Running(false)
  57 {
  58    Initialize();
  59 }
  60 pkgAcquire::pkgAcquire(pkgAcquireStatus *Progress) : LockFD(-1), d(NULL), Queues(0), Workers(0),
  61                            Configs(0), Log(NULL), ToFetch(0),
  62                            Debug(_config->FindB("Debug::pkgAcquire",false)),
  63                            Running(false)
  64 {
  65    Initialize();
  66    SetLog(Progress);
  67 }
  68 void pkgAcquire::Initialize()
  69 {
  70    string const Mode = _config->Find("Acquire::Queue-Mode","host");
  71    if (strcasecmp(Mode.c_str(),"host") == 0)
  72       QueueMode = QueueHost;
  73    if (strcasecmp(Mode.c_str(),"access") == 0)
  74       QueueMode = QueueAccess;
  75
  76    // chown the auth.conf file as it will be accessed by our methods
  77    std::string const SandboxUser = _config->Find("APT::Sandbox::User");
  78    if (getuid() == 0 && SandboxUser.empty() == false) // if we aren't root, we can't chown, so don't try it
  79    {
  80       struct passwd const * const pw = getpwnam(SandboxUser.c_str());
  81       struct group const * const gr = getgrnam("root");
  82       if (pw != NULL && gr != NULL)
  83       {
  84          std::string const AuthConf = _config->FindFile("Dir::Etc::netrc");
  85          if(AuthConf.empty() == false && RealFileExists(AuthConf) &&
  86                chown(AuthConf.c_str(), pw->pw_uid, gr->gr_gid) != 0)
  87             _error->WarningE("SetupAPTPartialDirectory", "chown to %s:root of file %s failed", SandboxUser.c_str(), AuthConf.c_str());
  88       }
  89    }
  90 }
  91                                                                         /*}}}*/
  92 // Acquire::GetLock - lock directory and prepare for action             /*{{{*/
  93 static bool SetupAPTPartialDirectory(std::string const &grand, std::string const &parent)
  94 {
  95    std::string const partial = parent + "partial";
  96    mode_t const mode = umask(S_IWGRP | S_IWOTH);
  97    bool const creation_fail = (CreateAPTDirectoryIfNeeded(grand, partial) == false &&
  98          CreateAPTDirectoryIfNeeded(parent, partial) == false);
  99    umask(mode);
 100    if (creation_fail == true)
 101       return false;
 102
 103    std::string const SandboxUser = _config->Find("APT::Sandbox::User");
 104    if (getuid() == 0 && SandboxUser.empty() == false) // if we aren't root, we can't chown, so don't try it
 105    {
 106       struct passwd const * const pw = getpwnam(SandboxUser.c_str());
 107       struct group const * const gr = getgrnam("root");
 108       if (pw != NULL && gr != NULL)
 109       {
 110          // chown the partial dir
 111          if(chown(partial.c_str(), pw->pw_uid, gr->gr_gid) != 0)
 112             _error->WarningE("SetupAPTPartialDirectory", "chown to %s:root of directory %s failed", SandboxUser.c_str(), partial.c_str());
 113       }
 114    }
 115    if (chmod(partial.c_str(), 0700) != 0)
 116       _error->WarningE("SetupAPTPartialDirectory", "chmod 0700 of directory %s failed", partial.c_str());
 117
 118    return true;
 119 }
 120 bool pkgAcquire::Setup(pkgAcquireStatus *Progress, string const &Lock)
 121 {
 122    Log = Progress;
 123    if (Lock.empty())
 124    {
 125       string const listDir = _config->FindDir("Dir::State::lists");
 126       if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir) == false)
 127          return _error->Errno("Acquire", _("List directory %spartial is missing."), listDir.c_str());
 128       string const archivesDir = _config->FindDir("Dir::Cache::Archives");
 129       if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir) == false)
 130          return _error->Errno("Acquire", _("Archives directory %spartial is missing."), archivesDir.c_str());
 131       return true;
 132    }
 133    return GetLock(Lock);
 134 }
 135 bool pkgAcquire::GetLock(std::string const &Lock)
 136 {
 137    if (Lock.empty() == true)
 138       return false;
 139
 140    // check for existence and possibly create auxiliary directories
 141    string const listDir = _config->FindDir("Dir::State::lists");
 142    string const archivesDir = _config->FindDir("Dir::Cache::Archives");
 143
 144    if (Lock == listDir)
 145    {
 146       if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir) == false)
 147          return _error->Errno("Acquire", _("List directory %spartial is missing."), listDir.c_str());
 148    }
 149    if (Lock == archivesDir)
 150    {
 151       if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir) == false)
 152          return _error->Errno("Acquire", _("Archives directory %spartial is missing."), archivesDir.c_str());
 153    }
 154
 155    if (_config->FindB("Debug::NoLocking", false) == true)
 156       return true;
 157
 158    // Lock the directory this acquire object will work in
 159    if (LockFD != -1)
 160       close(LockFD);
 161    LockFD = ::GetLock(flCombine(Lock, "lock"));
 162    if (LockFD == -1)
 163       return _error->Error(_("Unable to lock directory %s"), Lock.c_str());
 164
 165    return true;
 166 }
 167                                                                         /*}}}*/
 168 // Acquire::~pkgAcquire - Destructor                                    /*{{{*/
 169 // ---------------------------------------------------------------------
 170 /* Free our memory, clean up the queues (destroy the workers) */
 171 pkgAcquire::~pkgAcquire()
 172 {
 173    Shutdown();
 174
 175    if (LockFD != -1)
 176       close(LockFD);
 177
 178    while (Configs != 0)
 179    {
 180       MethodConfig *Jnk = Configs;
 181       Configs = Configs->Next;
 182       delete Jnk;
 183    }
 184 }
 185                                                                         /*}}}*/
 186 // Acquire::Shutdown - Clean out the acquire object                     /*{{{*/
 187 // ---------------------------------------------------------------------
 188 /* */
 189 void pkgAcquire::Shutdown()
 190 {
 191    while (Items.empty() == false)
 192    {
 193       if (Items[0]->Status == Item::StatFetching)
 194          Items[0]->Status = Item::StatError;
 195       delete Items[0];
 196    }
 197
 198    while (Queues != 0)
 199    {
 200       Queue *Jnk = Queues;
 201       Queues = Queues->Next;
 202       delete Jnk;
 203    }
 204 }
 205                                                                         /*}}}*/
 206 // Acquire::Add - Add a new item                                        /*{{{*/
 207 // ---------------------------------------------------------------------
 208 /* This puts an item on the acquire list. This list is mainly for tracking
 209    item status */
 210 void pkgAcquire::Add(Item *Itm)
 211 {
 212    Items.push_back(Itm);
 213 }
 214                                                                         /*}}}*/
 215 // Acquire::Remove - Remove a item                                      /*{{{*/
 216 // ---------------------------------------------------------------------
 217 /* Remove an item from the acquire list. This is usually not used.. */
 218 void pkgAcquire::Remove(Item *Itm)
 219 {
 220    Dequeue(Itm);
 221
 222    for (ItemIterator I = Items.begin(); I != Items.end();)
 223    {
 224       if (*I == Itm)
 225       {
 226          Items.erase(I);
 227          I = Items.begin();
 228       }
 229       else
 230          ++I;
 231    }
 232 }
 233                                                                         /*}}}*/
 234 // Acquire::Add - Add a worker                                          /*{{{*/
 235 // ---------------------------------------------------------------------
 236 /* A list of workers is kept so that the select loop can direct their FD
 237    usage. */
 238 void pkgAcquire::Add(Worker *Work)
 239 {
 240    Work->NextAcquire = Workers;
 241    Workers = Work;
 242 }
 243                                                                         /*}}}*/
 244 // Acquire::Remove - Remove a worker                                    /*{{{*/
 245 // ---------------------------------------------------------------------
 246 /* A worker has died. This can not be done while the select loop is running
 247    as it would require that RunFds could handling a changing list state and
 248    it can't.. */
 249 void pkgAcquire::Remove(Worker *Work)
 250 {
 251    if (Running == true)
 252       abort();
 253
 254    Worker **I = &Workers;
 255    for (; *I != 0;)
 256    {
 257       if (*I == Work)
 258          *I = (*I)->NextAcquire;
 259       else
 260          I = &(*I)->NextAcquire;
 261    }
 262 }
 263                                                                         /*}}}*/
 264 // Acquire::Enqueue - Queue an URI for fetching                         /*{{{*/
 265 // ---------------------------------------------------------------------
 266 /* This is the entry point for an item. An item calls this function when
 267    it is constructed which creates a queue (based on the current queue
 268    mode) and puts the item in that queue. If the system is running then
 269    the queue might be started. */
 270 void pkgAcquire::Enqueue(ItemDesc &Item)
 271 {
 272    // Determine which queue to put the item in
 273    const MethodConfig *Config;
 274    string Name = QueueName(Item.URI,Config);
 275    if (Name.empty() == true)
 276       return;
 277
 278    // Find the queue structure
 279    Queue *I = Queues;
 280    for (; I != 0 && I->Name != Name; I = I->Next);
 281    if (I == 0)
 282    {
 283       I = new Queue(Name,this);
 284       I->Next = Queues;
 285       Queues = I;
 286
 287       if (Running == true)
 288          I->Startup();
 289    }
 290
 291    // See if this is a local only URI
 292    if (Config->LocalOnly == true && Item.Owner->Complete == false)
 293       Item.Owner->Local = true;
 294    Item.Owner->Status = Item::StatIdle;
 295
 296    // Queue it into the named queue
 297    if(I->Enqueue(Item))
 298       ToFetch++;
 299
 300    // Some trace stuff
 301    if (Debug == true)
 302    {
 303       clog << "Fetching " << Item.URI << endl;
 304       clog << " to " << Item.Owner->DestFile << endl;
 305       clog << " Queue is: " << Name << endl;
 306    }
 307 }
 308                                                                         /*}}}*/
 309 // Acquire::Dequeue - Remove an item from all queues                    /*{{{*/
 310 // ---------------------------------------------------------------------
 311 /* This is called when an item is finished being fetched. It removes it
 312    from all the queues */
 313 void pkgAcquire::Dequeue(Item *Itm)
 314 {
 315    Queue *I = Queues;
 316    bool Res = false;
 317    if (Debug == true)
 318       clog << "Dequeuing " << Itm->DestFile << endl;
 319
 320    for (; I != 0; I = I->Next)
 321    {
 322       if (I->Dequeue(Itm))
 323       {
 324          Res = true;
 325          if (Debug == true)
 326             clog << "Dequeued from " << I->Name << endl;
 327       }
 328    }
 329
 330    if (Res == true)
 331       ToFetch--;
 332 }
 333                                                                         /*}}}*/
 334 // Acquire::QueueName - Return the name of the queue for this URI       /*{{{*/
 335 // ---------------------------------------------------------------------
 336 /* The string returned depends on the configuration settings and the
 337    method parameters. Given something like http://foo.org/bar it can
 338    return http://foo.org or http */
 339 string pkgAcquire::QueueName(string Uri,MethodConfig const *&Config)
 340 {
 341    URI U(Uri);
 342
 343    Config = GetConfig(U.Access);
 344    if (Config == 0)
 345       return string();
 346
 347    /* Single-Instance methods get exactly one queue per URI. This is
 348       also used for the Access queue method  */
 349    if (Config->SingleInstance == true || QueueMode == QueueAccess)
 350       return U.Access;
 351
 352    string AccessSchema = U.Access + ':',
 353         FullQueueName = AccessSchema + U.Host;
 354    unsigned int Instances = 0, SchemaLength = AccessSchema.length();
 355
 356    Queue *I = Queues;
 357    for (; I != 0; I = I->Next) {
 358       // if the queue already exists, re-use it
 359       if (I->Name == FullQueueName)
 360          return FullQueueName;
 361
 362       if (I->Name.compare(0, SchemaLength, AccessSchema) == 0)
 363          Instances++;
 364    }
 365
 366    if (Debug) {
 367       clog << "Found " << Instances << " instances of " << U.Access << endl;
 368    }
 369
 370    if (Instances >= (unsigned int)_config->FindI("Acquire::QueueHost::Limit",10))
 371       return U.Access;
 372
 373    return FullQueueName;
 374 }
 375                                                                         /*}}}*/
 376 // Acquire::GetConfig - Fetch the configuration information             /*{{{*/
 377 // ---------------------------------------------------------------------
 378 /* This locates the configuration structure for an access method. If
 379    a config structure cannot be found a Worker will be created to
 380    retrieve it */
 381 pkgAcquire::MethodConfig *pkgAcquire::GetConfig(string Access)
 382 {
 383    // Search for an existing config
 384    MethodConfig *Conf;
 385    for (Conf = Configs; Conf != 0; Conf = Conf->Next)
 386       if (Conf->Access == Access)
 387          return Conf;
 388
 389    // Create the new config class
 390    Conf = new MethodConfig;
 391    Conf->Access = Access;
 392    Conf->Next = Configs;
 393    Configs = Conf;
 394
 395    // Create the worker to fetch the configuration
 396    Worker Work(Conf);
 397    if (Work.Start() == false)
 398       return 0;
 399
 400    /* if a method uses DownloadLimit, we switch to SingleInstance mode */
 401    if(_config->FindI("Acquire::"+Access+"::Dl-Limit",0) > 0)
 402       Conf->SingleInstance = true;
 403
 404    return Conf;
 405 }
 406                                                                         /*}}}*/
 407 // Acquire::SetFds - Deal with readable FDs                             /*{{{*/
 408 // ---------------------------------------------------------------------
 409 /* Collect FDs that have activity monitors into the fd sets */
 410 void pkgAcquire::SetFds(int &Fd,fd_set *RSet,fd_set *WSet)
 411 {
 412    for (Worker *I = Workers; I != 0; I = I->NextAcquire)
 413    {
 414       if (I->InReady == true && I->InFd >= 0)
 415       {
 416          if (Fd < I->InFd)
 417             Fd = I->InFd;
 418          FD_SET(I->InFd,RSet);
 419       }
 420       if (I->OutReady == true && I->OutFd >= 0)
 421       {
 422          if (Fd < I->OutFd)
 423             Fd = I->OutFd;
 424          FD_SET(I->OutFd,WSet);
 425       }
 426    }
 427 }
 428                                                                         /*}}}*/
 429 // Acquire::RunFds - Deal with active FDs                               /*{{{*/
 430 // ---------------------------------------------------------------------
 431 /* Dispatch active FDs over to the proper workers. It is very important
 432    that a worker never be erased while this is running! The queue class
 433    should never erase a worker except during shutdown processing. */
 434 void pkgAcquire::RunFds(fd_set *RSet,fd_set *WSet)
 435 {
 436    for (Worker *I = Workers; I != 0; I = I->NextAcquire)
 437    {
 438       if (I->InFd >= 0 && FD_ISSET(I->InFd,RSet) != 0)
 439          I->InFdReady();
 440       if (I->OutFd >= 0 && FD_ISSET(I->OutFd,WSet) != 0)
 441          I->OutFdReady();
 442    }
 443 }
 444                                                                         /*}}}*/
 445 // Acquire::Run - Run the fetch sequence                                /*{{{*/
 446 // ---------------------------------------------------------------------
 447 /* This runs the queues. It manages a select loop for all of the
 448    Worker tasks. The workers interact with the queues and items to
 449    manage the actual fetch. */
 450 static void CheckDropPrivsMustBeDisabled(pkgAcquire const &Fetcher)
 451 {
 452    if(getuid() != 0)
 453       return;
 454
 455    std::string SandboxUser = _config->Find("APT::Sandbox::User");
 456    if (SandboxUser.empty())
 457       return;
 458
 459    struct passwd const * const pw = getpwnam(SandboxUser.c_str());
 460    if (pw == NULL)
 461       return;
 462
 463    gid_t const old_euid = geteuid();
 464    gid_t const old_egid = getegid();
 465    if (setegid(pw->pw_gid) != 0)
 466       _error->Errno("setegid", "setegid %u failed", pw->pw_gid);
 467    if (seteuid(pw->pw_uid) != 0)
 468       _error->Errno("seteuid", "seteuid %u failed", pw->pw_uid);
 469
 470    bool dropPrivs = true;
 471    for (pkgAcquire::ItemCIterator I = Fetcher.ItemsBegin();
 472         I != Fetcher.ItemsEnd() && dropPrivs == true; ++I)
 473    {
 474       std::string filename = (*I)->DestFile;
 475       if (filename.empty())
 476          continue;
 477
 478       // no need to drop privileges for a complete file
 479       if ((*I)->Complete == true)
 480          continue;
 481
 482       // we check directory instead of file as the file might or might not
 483       // exist already as a link or not which complicates everything…
 484       std::string dirname = flNotFile(filename);
 485       if (unlikely(dirname.empty()))
 486          continue;
 487       // translate relative to absolute for DirectoryExists
 488       // FIXME: What about ../ and ./../ ?
 489       if (dirname.substr(0,2) == "./")
 490          dirname = SafeGetCWD() + dirname.substr(2);
 491
 492       if (DirectoryExists(dirname))
 493          ;
 494       else
 495          continue; // assume it is created correctly by the acquire system
 496
 497       if (faccessat(-1, dirname.c_str(), R_OK | W_OK | X_OK, AT_EACCESS | AT_SYMLINK_NOFOLLOW) != 0)
 498       {
 499          dropPrivs = false;
 500          _error->WarningE("pkgAcquire::Run", _("Can't drop privileges for downloading as file '%s' couldn't be accessed by user '%s'."),
 501                filename.c_str(), SandboxUser.c_str());
 502          _config->Set("APT::Sandbox::User", "");
 503          break;
 504       }
 505    }
 506
 507    if (seteuid(old_euid) != 0)
 508       _error->Errno("seteuid", "seteuid %u failed", old_euid);
 509    if (setegid(old_egid) != 0)
 510       _error->Errno("setegid", "setegid %u failed", old_egid);
 511 }
 512 pkgAcquire::RunResult pkgAcquire::Run(int PulseIntervall)
 513 {
 514    CheckDropPrivsMustBeDisabled(*this);
 515
 516    Running = true;
 517
 518    for (Queue *I = Queues; I != 0; I = I->Next)
 519       I->Startup();
 520
 521    if (Log != 0)
 522       Log->Start();
 523
 524    bool WasCancelled = false;
 525
 526    // Run till all things have been acquired
 527    struct timeval tv;
 528    tv.tv_sec = 0;
 529    tv.tv_usec = PulseIntervall;
 530    while (ToFetch > 0)
 531    {
 532       fd_set RFds;
 533       fd_set WFds;
 534       int Highest = 0;
 535       FD_ZERO(&RFds);
 536       FD_ZERO(&WFds);
 537       SetFds(Highest,&RFds,&WFds);
 538
 539       int Res;
 540       do
 541       {
 542          Res = select(Highest+1,&RFds,&WFds,0,&tv);
 543       }
 544       while (Res < 0 && errno == EINTR);
 545
 546       if (Res < 0)
 547       {
 548          _error->Errno("select","Select has failed");
 549          break;
 550       }
 551
 552       RunFds(&RFds,&WFds);
 553       if (_error->PendingError() == true)
 554          break;
 555
 556       // Timeout, notify the log class
 557       if (Res == 0 || (Log != 0 && Log->Update == true))
 558       {
 559          tv.tv_usec = PulseIntervall;
 560          for (Worker *I = Workers; I != 0; I = I->NextAcquire)
 561             I->Pulse();
 562          if (Log != 0 && Log->Pulse(this) == false)
 563          {
 564             WasCancelled = true;
 565             break;
 566          }
 567       }
 568    }
 569
 570    if (Log != 0)
 571       Log->Stop();
 572
 573    // Shut down the acquire bits
 574    Running = false;
 575    for (Queue *I = Queues; I != 0; I = I->Next)
 576       I->Shutdown(false);
 577
 578    // Shut down the items
 579    for (ItemIterator I = Items.begin(); I != Items.end(); ++I)
 580       (*I)->Finished();
 581
 582    if (_error->PendingError())
 583       return Failed;
 584    if (WasCancelled)
 585       return Cancelled;
 586    return Continue;
 587 }
 588                                                                         /*}}}*/
 589 // Acquire::Bump - Called when an item is dequeued                      /*{{{*/
 590 // ---------------------------------------------------------------------
 591 /* This routine bumps idle queues in hopes that they will be able to fetch
 592    the dequeued item */
 593 void pkgAcquire::Bump()
 594 {
 595    for (Queue *I = Queues; I != 0; I = I->Next)
 596       I->Bump();
 597 }
 598                                                                         /*}}}*/
 599 // Acquire::WorkerStep - Step to the next worker                        /*{{{*/
 600 // ---------------------------------------------------------------------
 601 /* Not inlined to advoid including acquire-worker.h */
 602 pkgAcquire::Worker *pkgAcquire::WorkerStep(Worker *I)
 603 {
 604    return I->NextAcquire;
 605 }
 606                                                                         /*}}}*/
 607 // Acquire::Clean - Cleans a directory                                  /*{{{*/
 608 // ---------------------------------------------------------------------
 609 /* This is a bit simplistic, it looks at every file in the dir and sees
 610    if it is part of the download set. */
 611 bool pkgAcquire::Clean(string Dir)
 612 {
 613    // non-existing directories are by definition clean…
 614    if (DirectoryExists(Dir) == false)
 615       return true;
 616
 617    if(Dir == "/")
 618       return _error->Error(_("Clean of %s is not supported"), Dir.c_str());
 619
 620    DIR *D = opendir(Dir.c_str());
 621    if (D == 0)
 622       return _error->Errno("opendir",_("Unable to read %s"),Dir.c_str());
 623
 624    string StartDir = SafeGetCWD();
 625    if (chdir(Dir.c_str()) != 0)
 626    {
 627       closedir(D);
 628       return _error->Errno("chdir",_("Unable to change to %s"),Dir.c_str());
 629    }
 630
 631    for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D))
 632    {
 633       // Skip some files..
 634       if (strcmp(Dir->d_name,"lock") == 0 ||
 635           strcmp(Dir->d_name,"partial") == 0 ||
 636           strcmp(Dir->d_name,".") == 0 ||
 637           strcmp(Dir->d_name,"..") == 0)
 638          continue;
 639
 640       // Look in the get list
 641       ItemCIterator I = Items.begin();
 642       for (; I != Items.end(); ++I)
 643          if (flNotDir((*I)->DestFile) == Dir->d_name)
 644             break;
 645
 646       // Nothing found, nuke it
 647       if (I == Items.end())
 648          unlink(Dir->d_name);
 649    };
 650
 651    closedir(D);
 652    if (chdir(StartDir.c_str()) != 0)
 653       return _error->Errno("chdir",_("Unable to change to %s"),StartDir.c_str());
 654    return true;
 655 }
 656                                                                         /*}}}*/
 657 // Acquire::TotalNeeded - Number of bytes to fetch                      /*{{{*/
 658 // ---------------------------------------------------------------------
 659 /* This is the total number of bytes needed */
 660 APT_PURE unsigned long long pkgAcquire::TotalNeeded()
 661 {
 662    unsigned long long Total = 0;
 663    for (ItemCIterator I = ItemsBegin(); I != ItemsEnd(); ++I)
 664       Total += (*I)->FileSize;
 665    return Total;
 666 }
 667                                                                         /*}}}*/
 668 // Acquire::FetchNeeded - Number of bytes needed to get                 /*{{{*/
 669 // ---------------------------------------------------------------------
 670 /* This is the number of bytes that is not local */
 671 APT_PURE unsigned long long pkgAcquire::FetchNeeded()
 672 {
 673    unsigned long long Total = 0;
 674    for (ItemCIterator I = ItemsBegin(); I != ItemsEnd(); ++I)
 675       if ((*I)->Local == false)
 676          Total += (*I)->FileSize;
 677    return Total;
 678 }
 679                                                                         /*}}}*/
 680 // Acquire::PartialPresent - Number of partial bytes we already have    /*{{{*/
 681 // ---------------------------------------------------------------------
 682 /* This is the number of bytes that is not local */
 683 APT_PURE unsigned long long pkgAcquire::PartialPresent()
 684 {
 685   unsigned long long Total = 0;
 686    for (ItemCIterator I = ItemsBegin(); I != ItemsEnd(); ++I)
 687       if ((*I)->Local == false)
 688          Total += (*I)->PartialSize;
 689    return Total;
 690 }
 691                                                                         /*}}}*/
 692 // Acquire::UriBegin - Start iterator for the uri list                  /*{{{*/
 693 // ---------------------------------------------------------------------
 694 /* */
 695 pkgAcquire::UriIterator pkgAcquire::UriBegin()
 696 {
 697    return UriIterator(Queues);
 698 }
 699                                                                         /*}}}*/
 700 // Acquire::UriEnd - End iterator for the uri list                      /*{{{*/
 701 // ---------------------------------------------------------------------
 702 /* */
 703 pkgAcquire::UriIterator pkgAcquire::UriEnd()
 704 {
 705    return UriIterator(0);
 706 }
 707                                                                         /*}}}*/
 708 // Acquire::MethodConfig::MethodConfig - Constructor                    /*{{{*/
 709 // ---------------------------------------------------------------------
 710 /* */
 711 pkgAcquire::MethodConfig::MethodConfig() : d(NULL), Next(0), SingleInstance(false),
 712    Pipeline(false), SendConfig(false), LocalOnly(false), NeedsCleanup(false),
 713    Removable(false)
 714 {
 715 }
 716                                                                         /*}}}*/
 717 // Queue::Queue - Constructor                                           /*{{{*/
 718 // ---------------------------------------------------------------------
 719 /* */
 720 pkgAcquire::Queue::Queue(string const &name,pkgAcquire * const owner) : d(NULL), Next(0),
 721    Name(name), Items(0), Workers(0), Owner(owner), PipeDepth(0), MaxPipeDepth(1)
 722 {
 723 }
 724                                                                         /*}}}*/
 725 // Queue::~Queue - Destructor                                           /*{{{*/
 726 // ---------------------------------------------------------------------
 727 /* */
 728 pkgAcquire::Queue::~Queue()
 729 {
 730    Shutdown(true);
 731
 732    while (Items != 0)
 733    {
 734       QItem *Jnk = Items;
 735       Items = Items->Next;
 736       delete Jnk;
 737    }
 738 }
 739                                                                         /*}}}*/
 740 // Queue::Enqueue - Queue an item to the queue                          /*{{{*/
 741 // ---------------------------------------------------------------------
 742 /* */
 743 bool pkgAcquire::Queue::Enqueue(ItemDesc &Item)
 744 {
 745    QItem **I = &Items;
 746    // move to the end of the queue and check for duplicates here
 747    HashStringList const hsl = Item.Owner->GetExpectedHashes();
 748    for (; *I != 0; I = &(*I)->Next)
 749       if (Item.URI == (*I)->URI || hsl == (*I)->Owner->GetExpectedHashes())
 750       {
 751          if (_config->FindB("Debug::pkgAcquire::Worker",false) == true)
 752             std::cerr << " @ Queue: Action combined for " << Item.URI << " and " << (*I)->URI << std::endl;
 753          (*I)->Owners.push_back(Item.Owner);
 754          Item.Owner->Status = (*I)->Owner->Status;
 755          return false;
 756       }
 757
 758    // Create a new item
 759    QItem *Itm = new QItem;
 760    *Itm = Item;
 761    Itm->Next = 0;
 762    *I = Itm;
 763
 764    Item.Owner->QueueCounter++;
 765    if (Items->Next == 0)
 766       Cycle();
 767    return true;
 768 }
 769                                                                         /*}}}*/
 770 // Queue::Dequeue - Remove an item from the queue                       /*{{{*/
 771 // ---------------------------------------------------------------------
 772 /* We return true if we hit something */
 773 bool pkgAcquire::Queue::Dequeue(Item *Owner)
 774 {
 775    if (Owner->Status == pkgAcquire::Item::StatFetching)
 776       return _error->Error("Tried to dequeue a fetching object");
 777
 778    bool Res = false;
 779
 780    QItem **I = &Items;
 781    for (; *I != 0;)
 782    {
 783       if (Owner == (*I)->Owner)
 784       {
 785          QItem *Jnk= *I;
 786          *I = (*I)->Next;
 787          Owner->QueueCounter--;
 788          delete Jnk;
 789          Res = true;
 790       }
 791       else
 792          I = &(*I)->Next;
 793    }
 794
 795    return Res;
 796 }
 797                                                                         /*}}}*/
 798 // Queue::Startup - Start the worker processes                          /*{{{*/
 799 // ---------------------------------------------------------------------
 800 /* It is possible for this to be called with a pre-existing set of
 801    workers. */
 802 bool pkgAcquire::Queue::Startup()
 803 {
 804    if (Workers == 0)
 805    {
 806       URI U(Name);
 807       pkgAcquire::MethodConfig *Cnf = Owner->GetConfig(U.Access);
 808       if (Cnf == 0)
 809          return false;
 810
 811       Workers = new Worker(this,Cnf,Owner->Log);
 812       Owner->Add(Workers);
 813       if (Workers->Start() == false)
 814          return false;
 815
 816       /* When pipelining we commit 10 items. This needs to change when we
 817          added other source retry to have cycle maintain a pipeline depth
 818          on its own. */
 819       if (Cnf->Pipeline == true)
 820          MaxPipeDepth = _config->FindI("Acquire::Max-Pipeline-Depth",10);
 821       else
 822          MaxPipeDepth = 1;
 823    }
 824
 825    return Cycle();
 826 }
 827                                                                         /*}}}*/
 828 // Queue::Shutdown - Shutdown the worker processes                      /*{{{*/
 829 // ---------------------------------------------------------------------
 830 /* If final is true then all workers are eliminated, otherwise only workers
 831    that do not need cleanup are removed */
 832 bool pkgAcquire::Queue::Shutdown(bool Final)
 833 {
 834    // Delete all of the workers
 835    pkgAcquire::Worker **Cur = &Workers;
 836    while (*Cur != 0)
 837    {
 838       pkgAcquire::Worker *Jnk = *Cur;
 839       if (Final == true || Jnk->GetConf()->NeedsCleanup == false)
 840       {
 841          *Cur = Jnk->NextQueue;
 842          Owner->Remove(Jnk);
 843          delete Jnk;
 844       }
 845       else
 846          Cur = &(*Cur)->NextQueue;
 847    }
 848
 849    return true;
 850 }
 851                                                                         /*}}}*/
 852 // Queue::FindItem - Find a URI in the item list                        /*{{{*/
 853 // ---------------------------------------------------------------------
 854 /* */
 855 pkgAcquire::Queue::QItem *pkgAcquire::Queue::FindItem(string URI,pkgAcquire::Worker *Owner)
 856 {
 857    for (QItem *I = Items; I != 0; I = I->Next)
 858       if (I->URI == URI && I->Worker == Owner)
 859          return I;
 860    return 0;
 861 }
 862                                                                         /*}}}*/
 863 // Queue::ItemDone - Item has been completed                            /*{{{*/
 864 // ---------------------------------------------------------------------
 865 /* The worker signals this which causes the item to be removed from the
 866    queue. If this is the last queue instance then it is removed from the
 867    main queue too.*/
 868 bool pkgAcquire::Queue::ItemDone(QItem *Itm)
 869 {
 870    PipeDepth--;
 871    for (QItem::owner_iterator O = Itm->Owners.begin(); O != Itm->Owners.end(); ++O)
 872    {
 873       if ((*O)->Status == pkgAcquire::Item::StatFetching)
 874          (*O)->Status = pkgAcquire::Item::StatDone;
 875    }
 876
 877    if (Itm->Owner->QueueCounter <= 1)
 878       Owner->Dequeue(Itm->Owner);
 879    else
 880    {
 881       Dequeue(Itm->Owner);
 882       Owner->Bump();
 883    }
 884
 885    return Cycle();
 886 }
 887                                                                         /*}}}*/
 888 // Queue::Cycle - Queue new items into the method                       /*{{{*/
 889 // ---------------------------------------------------------------------
 890 /* This locates a new idle item and sends it to the worker. If pipelining
 891    is enabled then it keeps the pipe full. */
 892 bool pkgAcquire::Queue::Cycle()
 893 {
 894    if (Items == 0 || Workers == 0)
 895       return true;
 896
 897    if (PipeDepth < 0)
 898       return _error->Error("Pipedepth failure");
 899
 900    // Look for a queable item
 901    QItem *I = Items;
 902    while (PipeDepth < (signed)MaxPipeDepth)
 903    {
 904       for (; I != 0; I = I->Next)
 905          if (I->Owner->Status == pkgAcquire::Item::StatIdle)
 906             break;
 907
 908       // Nothing to do, queue is idle.
 909       if (I == 0)
 910          return true;
 911
 912       I->Worker = Workers;
 913       for (QItem::owner_iterator O = I->Owners.begin(); O != I->Owners.end(); ++O)
 914          (*O)->Status = pkgAcquire::Item::StatFetching;
 915       PipeDepth++;
 916       if (Workers->QueueItem(I) == false)
 917          return false;
 918    }
 919
 920    return true;
 921 }
 922                                                                         /*}}}*/
 923 // Queue::Bump - Fetch any pending objects if we are idle               /*{{{*/
 924 // ---------------------------------------------------------------------
 925 /* This is called when an item in multiple queues is dequeued */
 926 void pkgAcquire::Queue::Bump()
 927 {
 928    Cycle();
 929 }
 930                                                                         /*}}}*/
 931 HashStringList pkgAcquire::Queue::QItem::GetExpectedHashes() const      /*{{{*/
 932 {
 933    /* each Item can have multiple owners and each owner might have different
 934       hashes, even if that is unlikely in practice and if so at least some
 935       owners will later fail. There is one situation through which is not a
 936       failure and still needs this handling: Two owners who expect the same
 937       file, but one owner only knows the SHA1 while the other only knows SHA256. */
 938    HashStringList superhsl;
 939    for (pkgAcquire::Queue::QItem::owner_iterator O = Owners.begin(); O != Owners.end(); ++O)
 940    {
 941       HashStringList const hsl = (*O)->GetExpectedHashes();
 942       if (hsl.usable() == false)
 943          continue;
 944       if (superhsl.usable() == false)
 945          superhsl = hsl;
 946       else
 947       {
 948          // we merge both lists - if we find disagreement send no hashes
 949          HashStringList::const_iterator hs = hsl.begin();
 950          for (; hs != hsl.end(); ++hs)
 951             if (superhsl.push_back(*hs) == false)
 952                break;
 953          if (hs != hsl.end())
 954          {
 955             superhsl.clear();
 956             break;
 957          }
 958       }
 959    }
 960    return superhsl;
 961 }
 962                                                                         /*}}}*/
 963 APT_PURE unsigned long long pkgAcquire::Queue::QItem::GetMaximumSize() const    /*{{{*/
 964 {
 965    unsigned long long Maximum = std::numeric_limits<unsigned long long>::max();
 966    for (pkgAcquire::Queue::QItem::owner_iterator O = Owners.begin(); O != Owners.end(); ++O)
 967    {
 968       if ((*O)->FileSize == 0)
 969          continue;
 970       Maximum = std::min(Maximum, (*O)->FileSize);
 971    }
 972    if (Maximum == std::numeric_limits<unsigned long long>::max())
 973       return 0;
 974    return Maximum;
 975 }
 976                                                                         /*}}}*/
 977 void pkgAcquire::Queue::QItem::SyncDestinationFiles() const             /*{{{*/
 978 {
 979    /* ensure that the first owner has the best partial file of all and
 980       the rest have (potentially dangling) symlinks to it so that
 981       everything (like progress reporting) finds it easily */
 982    std::string superfile = Owner->DestFile;
 983    off_t supersize = 0;
 984    for (pkgAcquire::Queue::QItem::owner_iterator O = Owners.begin(); O != Owners.end(); ++O)
 985    {
 986       if ((*O)->DestFile == superfile)
 987          continue;
 988       struct stat file;
 989       if (lstat((*O)->DestFile.c_str(),&file) == 0)
 990       {
 991          if ((file.st_mode & S_IFREG) == 0)
 992             unlink((*O)->DestFile.c_str());
 993          else if (supersize < file.st_size)
 994          {
 995             supersize = file.st_size;
 996             unlink(superfile.c_str());
 997             rename((*O)->DestFile.c_str(), superfile.c_str());
 998          }
 999          else
1000             unlink((*O)->DestFile.c_str());
1001          if (symlink(superfile.c_str(), (*O)->DestFile.c_str()) != 0)
1002          {
1003             ; // not a problem per-se and no real alternative
1004          }
1005       }
1006    }
1007 }
1008                                                                         /*}}}*/
1009 std::string pkgAcquire::Queue::QItem::Custom600Headers() const          /*{{{*/
1010 {
1011    /* The others are relatively easy to merge, but this one?
1012       Lets not merge and see how far we can run with it…
1013       Likely, nobody will ever notice as all the items will
1014       be of the same class and hence generate the same headers. */
1015    return Owner->Custom600Headers();
1016 }
1017                                                                         /*}}}*/
1018
1019 // AcquireStatus::pkgAcquireStatus - Constructor                        /*{{{*/
1020 // ---------------------------------------------------------------------
1021 /* */
1022 pkgAcquireStatus::pkgAcquireStatus() : d(NULL), Percent(-1), Update(true), MorePulses(false)
1023 {
1024    Start();
1025 }
1026                                                                         /*}}}*/
1027 // AcquireStatus::Pulse - Called periodically                           /*{{{*/
1028 // ---------------------------------------------------------------------
1029 /* This computes some internal state variables for the derived classes to
1030    use. It generates the current downloaded bytes and total bytes to download
1031    as well as the current CPS estimate. */
1032 bool pkgAcquireStatus::Pulse(pkgAcquire *Owner)
1033 {
1034    TotalBytes = 0;
1035    CurrentBytes = 0;
1036    TotalItems = 0;
1037    CurrentItems = 0;
1038
1039    // Compute the total number of bytes to fetch
1040    unsigned int Unknown = 0;
1041    unsigned int Count = 0;
1042    bool UnfetchedReleaseFiles = false;
1043    for (pkgAcquire::ItemCIterator I = Owner->ItemsBegin();
1044         I != Owner->ItemsEnd();
1045         ++I, ++Count)
1046    {
1047       TotalItems++;
1048       if ((*I)->Status == pkgAcquire::Item::StatDone)
1049          ++CurrentItems;
1050
1051       // Totally ignore local items
1052       if ((*I)->Local == true)
1053          continue;
1054
1055       // see if the method tells us to expect more
1056       TotalItems += (*I)->ExpectedAdditionalItems;
1057
1058       // check if there are unfetched Release files
1059       if ((*I)->Complete == false && (*I)->ExpectedAdditionalItems > 0)
1060          UnfetchedReleaseFiles = true;
1061
1062       TotalBytes += (*I)->FileSize;
1063       if ((*I)->Complete == true)
1064          CurrentBytes += (*I)->FileSize;
1065       if ((*I)->FileSize == 0 && (*I)->Complete == false)
1066          ++Unknown;
1067    }
1068
1069    // Compute the current completion
1070    unsigned long long ResumeSize = 0;
1071    for (pkgAcquire::Worker *I = Owner->WorkersBegin(); I != 0;
1072         I = Owner->WorkerStep(I))
1073    {
1074       if (I->CurrentItem != 0 && I->CurrentItem->Owner->Complete == false)
1075       {
1076          CurrentBytes += I->CurrentSize;
1077          ResumeSize += I->ResumePoint;
1078
1079          // Files with unknown size always have 100% completion
1080          if (I->CurrentItem->Owner->FileSize == 0 &&
1081              I->CurrentItem->Owner->Complete == false)
1082             TotalBytes += I->CurrentSize;
1083       }
1084    }
1085
1086    // Normalize the figures and account for unknown size downloads
1087    if (TotalBytes <= 0)
1088       TotalBytes = 1;
1089    if (Unknown == Count)
1090       TotalBytes = Unknown;
1091
1092    // Wha?! Is not supposed to happen.
1093    if (CurrentBytes > TotalBytes)
1094       CurrentBytes = TotalBytes;
1095
1096    // debug
1097    if (_config->FindB("Debug::acquire::progress", false) == true)
1098       std::clog << " Bytes: "
1099                 << SizeToStr(CurrentBytes) << " / " << SizeToStr(TotalBytes)
1100                 << std::endl;
1101
1102    // Compute the CPS
1103    struct timeval NewTime;
1104    gettimeofday(&NewTime,0);
1105    if ((NewTime.tv_sec - Time.tv_sec == 6 && NewTime.tv_usec > Time.tv_usec) ||
1106        NewTime.tv_sec - Time.tv_sec > 6)
1107    {
1108       double Delta = NewTime.tv_sec - Time.tv_sec +
1109                      (NewTime.tv_usec - Time.tv_usec)/1000000.0;
1110
1111       // Compute the CPS value
1112       if (Delta < 0.01)
1113          CurrentCPS = 0;
1114       else
1115          CurrentCPS = ((CurrentBytes - ResumeSize) - LastBytes)/Delta;
1116       LastBytes = CurrentBytes - ResumeSize;
1117       ElapsedTime = (unsigned long long)Delta;
1118       Time = NewTime;
1119    }
1120
1121    double const OldPercent = Percent;
1122    // calculate the percentage, if we have too little data assume 1%
1123    if (TotalBytes > 0 && UnfetchedReleaseFiles)
1124       Percent = 0;
1125    else
1126       // use both files and bytes because bytes can be unreliable
1127       Percent = (0.8 * (CurrentBytes/float(TotalBytes)*100.0) +
1128                  0.2 * (CurrentItems/float(TotalItems)*100.0));
1129    double const DiffPercent = Percent - OldPercent;
1130    if (DiffPercent < 0.001 && _config->FindB("Acquire::Progress::Diffpercent", false) == true)
1131       return true;
1132
1133    int fd = _config->FindI("APT::Status-Fd",-1);
1134    if(fd > 0)
1135    {
1136       ostringstream status;
1137
1138       char msg[200];
1139       long i = CurrentItems < TotalItems ? CurrentItems + 1 : CurrentItems;
1140       unsigned long long ETA = 0;
1141       if(CurrentCPS > 0)
1142          ETA = (TotalBytes - CurrentBytes) / CurrentCPS;
1143
1144       // only show the ETA if it makes sense
1145       if (ETA > 0 && ETA < 172800 /* two days */ )
1146          snprintf(msg,sizeof(msg), _("Retrieving file %li of %li (%s remaining)"), i, TotalItems, TimeToStr(ETA).c_str());
1147       else
1148          snprintf(msg,sizeof(msg), _("Retrieving file %li of %li"), i, TotalItems);
1149
1150       // build the status str
1151       status << "dlstatus:" << i
1152              << ":"  << std::setprecision(3) << Percent
1153              << ":" << msg
1154              << endl;
1155
1156       std::string const dlstatus = status.str();
1157       FileFd::Write(fd, dlstatus.c_str(), dlstatus.size());
1158    }
1159
1160    return true;
1161 }
1162                                                                         /*}}}*/
1163 // AcquireStatus::Start - Called when the download is started           /*{{{*/
1164 // ---------------------------------------------------------------------
1165 /* We just reset the counters */
1166 void pkgAcquireStatus::Start()
1167 {
1168    gettimeofday(&Time,0);
1169    gettimeofday(&StartTime,0);
1170    LastBytes = 0;
1171    CurrentCPS = 0;
1172    CurrentBytes = 0;
1173    TotalBytes = 0;
1174    FetchedBytes = 0;
1175    ElapsedTime = 0;
1176    TotalItems = 0;
1177    CurrentItems = 0;
1178 }
1179                                                                         /*}}}*/
1180 // AcquireStatus::Stop - Finished downloading                           /*{{{*/
1181 // ---------------------------------------------------------------------
1182 /* This accurately computes the elapsed time and the total overall CPS. */
1183 void pkgAcquireStatus::Stop()
1184 {
1185    // Compute the CPS and elapsed time
1186    struct timeval NewTime;
1187    gettimeofday(&NewTime,0);
1188
1189    double Delta = NewTime.tv_sec - StartTime.tv_sec +
1190                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
1191
1192    // Compute the CPS value
1193    if (Delta < 0.01)
1194       CurrentCPS = 0;
1195    else
1196       CurrentCPS = FetchedBytes/Delta;
1197    LastBytes = CurrentBytes;
1198    ElapsedTime = (unsigned long long)Delta;
1199 }
1200                                                                         /*}}}*/
1201 // AcquireStatus::Fetched - Called when a byte set has been fetched     /*{{{*/
1202 // ---------------------------------------------------------------------
1203 /* This is used to get accurate final transfer rate reporting. */
1204 void pkgAcquireStatus::Fetched(unsigned long long Size,unsigned long long Resume)
1205 {
1206    FetchedBytes += Size - Resume;
1207 }
1208                                                                         /*}}}*/
1209
1210 pkgAcquire::UriIterator::UriIterator(pkgAcquire::Queue *Q) : d(NULL), CurQ(Q), CurItem(0)
1211 {
1212    while (CurItem == 0 && CurQ != 0)
1213    {
1214       CurItem = CurQ->Items;
1215       CurQ = CurQ->Next;
1216    }
1217 }
1218
1219 APT_CONST pkgAcquire::UriIterator::~UriIterator() {}
1220 APT_CONST pkgAcquire::MethodConfig::~MethodConfig() {}
1221 APT_CONST pkgAcquireStatus::~pkgAcquireStatus() {}