apt-pkg/acquire.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire.cc,v 1.7 1998/11/01 05:27:34 jgg Exp $
   4 /* ######################################################################
   5
   6    Acquire - File Acquiration
   7
   8    The core element for the schedual system is the concept of a named
   9    queue. Each queue is unique and each queue has a name derived from the
  10    URI. The degree of paralization can be controled by how the queue
  11    name is derived from the URI.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #ifdef __GNUG__
  17 #pragma implementation "apt-pkg/acquire.h"
  18 #endif
  19 #include <apt-pkg/acquire.h>
  20 #include <apt-pkg/acquire-item.h>
  21 #include <apt-pkg/acquire-worker.h>
  22 #include <apt-pkg/configuration.h>
  23 #include <apt-pkg/error.h>
  24 #include <strutl.h>
  25                                                                         /*}}}*/
  26
  27 // Acquire::pkgAcquire - Constructor                                    /*{{{*/
  28 // ---------------------------------------------------------------------
  29 /* We grab some runtime state from the configuration space */
  30 pkgAcquire::pkgAcquire()
  31 {
  32    Queues = 0;
  33    Configs = 0;
  34    Workers = 0;
  35    ToFetch = 0;
  36    Running = false;
  37
  38    string Mode = _config->Find("Acquire::Queue-Mode","host");
  39    if (strcasecmp(Mode.c_str(),"host") == 0)
  40       QueueMode = QueueHost;
  41    if (strcasecmp(Mode.c_str(),"access") == 0)
  42       QueueMode = QueueAccess;
  43
  44    Debug = _config->FindB("Debug::pkgAcquire",false);
  45 }
  46                                                                         /*}}}*/
  47 // Acquire::~pkgAcquire - Destructor                                    /*{{{*/
  48 // ---------------------------------------------------------------------
  49 /* Free our memory, clean up the queues (destroy the workers) */
  50 pkgAcquire::~pkgAcquire()
  51 {
  52    while (Items.size() != 0)
  53       delete Items[0];
  54
  55    while (Configs != 0)
  56    {
  57       MethodConfig *Jnk = Configs;
  58       Configs = Configs->Next;
  59       delete Jnk;
  60    }
  61
  62    while (Queues != 0)
  63    {
  64       Queue *Jnk = Queues;
  65       Queues = Queues->Next;
  66       delete Jnk;
  67    }
  68 }
  69                                                                         /*}}}*/
  70 // Acquire::Add - Add a new item                                        /*{{{*/
  71 // ---------------------------------------------------------------------
  72 /* This puts an item on the acquire list. This list is mainly for tracking
  73    item status */
  74 void pkgAcquire::Add(Item *Itm)
  75 {
  76    Items.push_back(Itm);
  77 }
  78                                                                         /*}}}*/
  79 // Acquire::Remove - Remove a item                                      /*{{{*/
  80 // ---------------------------------------------------------------------
  81 /* Remove an item from the acquire list. This is usually not used.. */
  82 void pkgAcquire::Remove(Item *Itm)
  83 {
  84    for (vector<Item *>::iterator I = Items.begin(); I < Items.end(); I++)
  85    {
  86       if (*I == Itm)
  87          Items.erase(I);
  88    }
  89 }
  90                                                                         /*}}}*/
  91 // Acquire::Add - Add a worker                                          /*{{{*/
  92 // ---------------------------------------------------------------------
  93 /* A list of workers is kept so that the select loop can direct their FD
  94    usage. */
  95 void pkgAcquire::Add(Worker *Work)
  96 {
  97    Work->NextAcquire = Workers;
  98    Workers = Work;
  99 }
 100                                                                         /*}}}*/
 101 // Acquire::Remove - Remove a worker                                    /*{{{*/
 102 // ---------------------------------------------------------------------
 103 /* A worker has died. This can not be done while the select loop is running
 104    as it would require that RunFds could handling a changing list state and
 105    it cant.. */
 106 void pkgAcquire::Remove(Worker *Work)
 107 {
 108    if (Running == true)
 109       abort();
 110
 111    Worker **I = &Workers;
 112    for (; *I != 0;)
 113    {
 114       if (*I == Work)
 115          *I = (*I)->NextAcquire;
 116       else
 117          I = &(*I)->NextAcquire;
 118    }
 119 }
 120                                                                         /*}}}*/
 121 // Acquire::Enqueue - Queue an URI for fetching                         /*{{{*/
 122 // ---------------------------------------------------------------------
 123 /* This is the entry point for an item. An item calls this function when
 124    it is construction which creates a queue (based on the current queue
 125    mode) and puts the item in that queue. If the system is running then
 126    the queue might be started. */
 127 void pkgAcquire::Enqueue(Item *Itm,string URI,string Description)
 128 {
 129    // Determine which queue to put the item in
 130    string Name = QueueName(URI);
 131    if (Name.empty() == true)
 132       return;
 133
 134    // Find the queue structure
 135    Queue *I = Queues;
 136    for (; I != 0 && I->Name != Name; I = I->Next);
 137    if (I == 0)
 138    {
 139       I = new Queue(Name,this);
 140       I->Next = Queues;
 141       Queues = I;
 142
 143       if (Running == true)
 144          I->Startup();
 145    }
 146
 147    // Queue it into the named queue
 148    I->Enqueue(Itm,URI,Description);
 149    ToFetch++;
 150
 151    // Some trace stuff
 152    if (Debug == true)
 153    {
 154       clog << "Fetching " << URI << endl;
 155       clog << " to " << Itm->DestFile << endl;
 156       clog << " Queue is: " << QueueName(URI) << endl;
 157    }
 158 }
 159                                                                         /*}}}*/
 160 // Acquire::Dequeue - Remove an item from all queues                    /*{{{*/
 161 // ---------------------------------------------------------------------
 162 /* This is called when an item is finished being fetched. It removes it
 163    from all the queues */
 164 void pkgAcquire::Dequeue(Item *Itm)
 165 {
 166    Queue *I = Queues;
 167    for (; I != 0; I = I->Next)
 168       I->Dequeue(Itm);
 169
 170    if (Debug == true)
 171       clog << "Dequeuing " << Itm->DestFile << endl;
 172    ToFetch--;
 173 }
 174                                                                         /*}}}*/
 175 // Acquire::QueueName - Return the name of the queue for this URI       /*{{{*/
 176 // ---------------------------------------------------------------------
 177 /* The string returned depends on the configuration settings and the
 178    method parameters. Given something like http://foo.org/bar it can
 179    return http://foo.org or http */
 180 string pkgAcquire::QueueName(string Uri)
 181 {
 182    URI U(Uri);
 183
 184    const MethodConfig *Config = GetConfig(U.Access);
 185    if (Config == 0)
 186       return string();
 187
 188    /* Single-Instance methods get exactly one queue per URI. This is
 189       also used for the Access queue method  */
 190    if (Config->SingleInstance == true || QueueMode == QueueAccess)
 191       return U.Access;
 192
 193    return U.Access + ':' + U.Host;
 194 }
 195                                                                         /*}}}*/
 196 // Acquire::GetConfig - Fetch the configuration information             /*{{{*/
 197 // ---------------------------------------------------------------------
 198 /* This locates the configuration structure for an access method. If
 199    a config structure cannot be found a Worker will be created to
 200    retrieve it */
 201 pkgAcquire::MethodConfig *pkgAcquire::GetConfig(string Access)
 202 {
 203    // Search for an existing config
 204    MethodConfig *Conf;
 205    for (Conf = Configs; Conf != 0; Conf = Conf->Next)
 206       if (Conf->Access == Access)
 207          return Conf;
 208
 209    // Create the new config class
 210    Conf = new MethodConfig;
 211    Conf->Access = Access;
 212    Conf->Next = Configs;
 213    Configs = Conf;
 214
 215    // Create the worker to fetch the configuration
 216    Worker Work(Conf);
 217    if (Work.Start() == false)
 218       return 0;
 219
 220    return Conf;
 221 }
 222                                                                         /*}}}*/
 223 // Acquire::SetFds - Deal with readable FDs                             /*{{{*/
 224 // ---------------------------------------------------------------------
 225 /* Collect FDs that have activity monitors into the fd sets */
 226 void pkgAcquire::SetFds(int &Fd,fd_set *RSet,fd_set *WSet)
 227 {
 228    for (Worker *I = Workers; I != 0; I = I->NextAcquire)
 229    {
 230       if (I->InReady == true && I->InFd >= 0)
 231       {
 232          if (Fd < I->InFd)
 233             Fd = I->InFd;
 234          FD_SET(I->InFd,RSet);
 235       }
 236       if (I->OutReady == true && I->OutFd >= 0)
 237       {
 238          if (Fd < I->OutFd)
 239             Fd = I->OutFd;
 240          FD_SET(I->OutFd,WSet);
 241       }
 242    }
 243 }
 244                                                                         /*}}}*/
 245 // Acquire::RunFds - Deal with active FDs                               /*{{{*/
 246 // ---------------------------------------------------------------------
 247 /* Dispatch active FDs over to the proper workers. It is very important
 248    that a worker never be erased while this is running! The queue class
 249    should never erase a worker except during shutdown processing. */
 250 void pkgAcquire::RunFds(fd_set *RSet,fd_set *WSet)
 251 {
 252    for (Worker *I = Workers; I != 0; I = I->NextAcquire)
 253    {
 254       if (I->InFd >= 0 && FD_ISSET(I->InFd,RSet) != 0)
 255          I->InFdReady();
 256       if (I->OutFd >= 0 && FD_ISSET(I->OutFd,WSet) != 0)
 257          I->OutFdReady();
 258    }
 259 }
 260                                                                         /*}}}*/
 261 // Acquire::Run - Run the fetch sequence                                /*{{{*/
 262 // ---------------------------------------------------------------------
 263 /* This runs the queues. It manages a select loop for all of the
 264    Worker tasks. The workers interact with the queues and items to
 265    manage the actual fetch. */
 266 bool pkgAcquire::Run()
 267 {
 268    Running = true;
 269
 270    for (Queue *I = Queues; I != 0; I = I->Next)
 271       I->Startup();
 272
 273    // Run till all things have been acquired
 274    while (ToFetch > 0)
 275    {
 276       fd_set RFds;
 277       fd_set WFds;
 278       int Highest = 0;
 279       FD_ZERO(&RFds);
 280       FD_ZERO(&WFds);
 281       SetFds(Highest,&RFds,&WFds);
 282
 283       if (select(Highest+1,&RFds,&WFds,0,0) <= 0)
 284       {
 285          Running = false;
 286          return _error->Errno("select","Select has failed");
 287       }
 288
 289       RunFds(&RFds,&WFds);
 290       if (_error->PendingError() == true)
 291          break;
 292    }
 293
 294    // Shut down the acquire bits
 295    Running = false;
 296    for (Queue *I = Queues; I != 0; I = I->Next)
 297       I->Shutdown();
 298
 299    return _error->PendingError();
 300 }
 301                                                                         /*}}}*/
 302 // Acquire::Bump - Called when an item is dequeued                      /*{{{*/
 303 // ---------------------------------------------------------------------
 304 /* This routine bumps idle queues in hopes that they will be able to fetch
 305    the dequeued item */
 306 void pkgAcquire::Bump()
 307 {
 308    for (Queue *I = Queues; I != 0; I = I->Next)
 309       I->Bump();
 310 }
 311                                                                         /*}}}*/
 312
 313 // Acquire::MethodConfig::MethodConfig - Constructor                    /*{{{*/
 314 // ---------------------------------------------------------------------
 315 /* */
 316 pkgAcquire::MethodConfig::MethodConfig()
 317 {
 318    SingleInstance = false;
 319    PreScan = false;
 320    Pipeline = false;
 321    SendConfig = false;
 322    Next = 0;
 323 }
 324                                                                         /*}}}*/
 325
 326 // Queue::Queue - Constructor                                           /*{{{*/
 327 // ---------------------------------------------------------------------
 328 /* */
 329 pkgAcquire::Queue::Queue(string Name,pkgAcquire *Owner) : Name(Name),
 330             Owner(Owner)
 331 {
 332    Items = 0;
 333    Next = 0;
 334    Workers = 0;
 335 }
 336                                                                         /*}}}*/
 337 // Queue::~Queue - Destructor                                           /*{{{*/
 338 // ---------------------------------------------------------------------
 339 /* */
 340 pkgAcquire::Queue::~Queue()
 341 {
 342    Shutdown();
 343
 344    while (Items != 0)
 345    {
 346       QItem *Jnk = Items;
 347       Items = Items->Next;
 348       delete Jnk;
 349    }
 350 }
 351                                                                         /*}}}*/
 352 // Queue::Enqueue - Queue an item to the queue                          /*{{{*/
 353 // ---------------------------------------------------------------------
 354 /* */
 355 void pkgAcquire::Queue::Enqueue(Item *Owner,string URI,string Description)
 356 {
 357    // Create a new item
 358    QItem *I = new QItem;
 359    I->Next = Items;
 360    Items = I;
 361
 362    // Fill it in
 363    Items->Owner = Owner;
 364    Items->URI = URI;
 365    Items->Description = Description;
 366    Owner->QueueCounter++;
 367
 368    if (Items->Next == 0)
 369       Cycle();
 370 }
 371                                                                         /*}}}*/
 372 // Queue::Dequeue - Remove an item from the queue                       /*{{{*/
 373 // ---------------------------------------------------------------------
 374 /* */
 375 void pkgAcquire::Queue::Dequeue(Item *Owner)
 376 {
 377    QItem **I = &Items;
 378    for (; *I != 0;)
 379    {
 380       if ((*I)->Owner == Owner)
 381       {
 382          QItem *Jnk= *I;
 383          *I = (*I)->Next;
 384          Owner->QueueCounter--;
 385          delete Jnk;
 386       }
 387       else
 388          I = &(*I)->Next;
 389    }
 390 }
 391                                                                         /*}}}*/
 392 // Queue::Startup - Start the worker processes                          /*{{{*/
 393 // ---------------------------------------------------------------------
 394 /* */
 395 bool pkgAcquire::Queue::Startup()
 396 {
 397    Shutdown();
 398
 399    URI U(Name);
 400    pkgAcquire::MethodConfig *Cnf = Owner->GetConfig(U.Access);
 401    if (Cnf == 0)
 402       return false;
 403
 404    Workers = new Worker(this,Cnf);
 405    Owner->Add(Workers);
 406    if (Workers->Start() == false)
 407       return false;
 408
 409    return Cycle();
 410 }
 411                                                                         /*}}}*/
 412 // Queue::Shutdown - Shutdown the worker processes                      /*{{{*/
 413 // ---------------------------------------------------------------------
 414 /* */
 415 bool pkgAcquire::Queue::Shutdown()
 416 {
 417    // Delete all of the workers
 418    while (Workers != 0)
 419    {
 420       pkgAcquire::Worker *Jnk = Workers;
 421       Workers = Workers->NextQueue;
 422       Owner->Remove(Jnk);
 423       delete Jnk;
 424    }
 425
 426    return true;
 427 }
 428                                                                         /*}}}*/
 429 // Queue::Finditem - Find a URI in the item list                        /*{{{*/
 430 // ---------------------------------------------------------------------
 431 /* */
 432 pkgAcquire::Queue::QItem *pkgAcquire::Queue::FindItem(string URI,pkgAcquire::Worker *Owner)
 433 {
 434    for (QItem *I = Items; I != 0; I = I->Next)
 435       if (I->URI == URI && I->Worker == Owner)
 436          return I;
 437    return 0;
 438 }
 439                                                                         /*}}}*/
 440 // Queue::ItemDone - Item has been completed                            /*{{{*/
 441 // ---------------------------------------------------------------------
 442 /* The worker signals this which causes the item to be removed from the
 443    queue. If this is the last queue instance then it is removed from the
 444    main queue too.*/
 445 bool pkgAcquire::Queue::ItemDone(QItem *Itm)
 446 {
 447    if (Itm->Owner->QueueCounter <= 1)
 448       Owner->Dequeue(Itm->Owner);
 449    else
 450    {
 451       Dequeue(Itm->Owner);
 452       Owner->Bump();
 453    }
 454
 455    return Cycle();
 456 }
 457                                                                         /*}}}*/
 458 // Queue::Cycle - Queue new items into the method                       /*{{{*/
 459 // ---------------------------------------------------------------------
 460 /* This locates a new idle item and sends it to the worker */
 461 bool pkgAcquire::Queue::Cycle()
 462 {
 463    if (Items == 0 || Workers == 0)
 464       return true;
 465
 466    // Look for a queable item
 467    QItem *I = Items;
 468    for (; I != 0; I = I->Next)
 469       if (I->Owner->Status == pkgAcquire::Item::StatIdle)
 470          break;
 471
 472    // Nothing to do, queue is idle.
 473    if (I == 0)
 474       return true;
 475
 476    I->Worker = Workers;
 477    I->Owner->Status = pkgAcquire::Item::StatFetching;
 478    return Workers->QueueItem(I);
 479 }
 480                                                                         /*}}}*/
 481 // Queue::Bump - Fetch any pending objects if we are idle               /*{{{*/
 482 // ---------------------------------------------------------------------
 483 /* */
 484 void pkgAcquire::Queue::Bump()
 485 {
 486 }
 487                                                                         /*}}}*/