]> git.saurik.com Git - apt.git/blob - apt-pkg/acquire.cc
Wow
[apt.git] / apt-pkg / acquire.cc
1 // -*- mode: cpp; mode: fold -*-
2 // Description /*{{{*/
3 // $Id: acquire.cc,v 1.15 1998/11/13 07:08:54 jgg Exp $
4 /* ######################################################################
5
6 Acquire - File Acquiration
7
8 The core element for the schedual system is the concept of a named
9 queue. Each queue is unique and each queue has a name derived from the
10 URI. The degree of paralization can be controled by how the queue
11 name is derived from the URI.
12
13 ##################################################################### */
14 /*}}}*/
15 // Include Files /*{{{*/
16 #ifdef __GNUG__
17 #pragma implementation "apt-pkg/acquire.h"
18 #endif
19 #include <apt-pkg/acquire.h>
20 #include <apt-pkg/acquire-item.h>
21 #include <apt-pkg/acquire-worker.h>
22 #include <apt-pkg/configuration.h>
23 #include <apt-pkg/error.h>
24 #include <strutl.h>
25
26 #include <dirent.h>
27 #include <sys/time.h>
28 /*}}}*/
29
30 // Acquire::pkgAcquire - Constructor /*{{{*/
31 // ---------------------------------------------------------------------
32 /* We grab some runtime state from the configuration space */
33 pkgAcquire::pkgAcquire(pkgAcquireStatus *Log) : Log(Log)
34 {
35 Queues = 0;
36 Configs = 0;
37 Workers = 0;
38 ToFetch = 0;
39 Running = false;
40
41 string Mode = _config->Find("Acquire::Queue-Mode","host");
42 if (strcasecmp(Mode.c_str(),"host") == 0)
43 QueueMode = QueueHost;
44 if (strcasecmp(Mode.c_str(),"access") == 0)
45 QueueMode = QueueAccess;
46
47 Debug = _config->FindB("Debug::pkgAcquire",false);
48 }
49 /*}}}*/
50 // Acquire::~pkgAcquire - Destructor /*{{{*/
51 // ---------------------------------------------------------------------
52 /* Free our memory, clean up the queues (destroy the workers) */
53 pkgAcquire::~pkgAcquire()
54 {
55 while (Items.size() != 0)
56 delete Items[0];
57
58 while (Configs != 0)
59 {
60 MethodConfig *Jnk = Configs;
61 Configs = Configs->Next;
62 delete Jnk;
63 }
64
65 while (Queues != 0)
66 {
67 Queue *Jnk = Queues;
68 Queues = Queues->Next;
69 delete Jnk;
70 }
71 }
72 /*}}}*/
73 // Acquire::Add - Add a new item /*{{{*/
74 // ---------------------------------------------------------------------
75 /* This puts an item on the acquire list. This list is mainly for tracking
76 item status */
77 void pkgAcquire::Add(Item *Itm)
78 {
79 Items.push_back(Itm);
80 }
81 /*}}}*/
82 // Acquire::Remove - Remove a item /*{{{*/
83 // ---------------------------------------------------------------------
84 /* Remove an item from the acquire list. This is usually not used.. */
85 void pkgAcquire::Remove(Item *Itm)
86 {
87 for (vector<Item *>::iterator I = Items.begin(); I < Items.end(); I++)
88 {
89 if (*I == Itm)
90 Items.erase(I);
91 }
92 }
93 /*}}}*/
94 // Acquire::Add - Add a worker /*{{{*/
95 // ---------------------------------------------------------------------
96 /* A list of workers is kept so that the select loop can direct their FD
97 usage. */
98 void pkgAcquire::Add(Worker *Work)
99 {
100 Work->NextAcquire = Workers;
101 Workers = Work;
102 }
103 /*}}}*/
104 // Acquire::Remove - Remove a worker /*{{{*/
105 // ---------------------------------------------------------------------
106 /* A worker has died. This can not be done while the select loop is running
107 as it would require that RunFds could handling a changing list state and
108 it cant.. */
109 void pkgAcquire::Remove(Worker *Work)
110 {
111 if (Running == true)
112 abort();
113
114 Worker **I = &Workers;
115 for (; *I != 0;)
116 {
117 if (*I == Work)
118 *I = (*I)->NextAcquire;
119 else
120 I = &(*I)->NextAcquire;
121 }
122 }
123 /*}}}*/
124 // Acquire::Enqueue - Queue an URI for fetching /*{{{*/
125 // ---------------------------------------------------------------------
126 /* This is the entry point for an item. An item calls this function when
127 it is construction which creates a queue (based on the current queue
128 mode) and puts the item in that queue. If the system is running then
129 the queue might be started. */
130 void pkgAcquire::Enqueue(ItemDesc &Item)
131 {
132 // Determine which queue to put the item in
133 string Name = QueueName(Item.URI);
134 if (Name.empty() == true)
135 return;
136
137 // Find the queue structure
138 Queue *I = Queues;
139 for (; I != 0 && I->Name != Name; I = I->Next);
140 if (I == 0)
141 {
142 I = new Queue(Name,this);
143 I->Next = Queues;
144 Queues = I;
145
146 if (Running == true)
147 I->Startup();
148 }
149
150 Item.Owner->Status = Item::StatIdle;
151
152 // Queue it into the named queue
153 I->Enqueue(Item);
154 ToFetch++;
155
156 // Some trace stuff
157 if (Debug == true)
158 {
159 clog << "Fetching " << Item.URI << endl;
160 clog << " to " << Item.Owner->DestFile << endl;
161 clog << " Queue is: " << QueueName(Item.URI) << endl;
162 }
163 }
164 /*}}}*/
165 // Acquire::Dequeue - Remove an item from all queues /*{{{*/
166 // ---------------------------------------------------------------------
167 /* This is called when an item is finished being fetched. It removes it
168 from all the queues */
169 void pkgAcquire::Dequeue(Item *Itm)
170 {
171 Queue *I = Queues;
172 bool Res = false;
173 for (; I != 0; I = I->Next)
174 Res |= I->Dequeue(Itm);
175
176 if (Debug == true)
177 clog << "Dequeuing " << Itm->DestFile << endl;
178 if (Res == true)
179 ToFetch--;
180 }
181 /*}}}*/
182 // Acquire::QueueName - Return the name of the queue for this URI /*{{{*/
183 // ---------------------------------------------------------------------
184 /* The string returned depends on the configuration settings and the
185 method parameters. Given something like http://foo.org/bar it can
186 return http://foo.org or http */
187 string pkgAcquire::QueueName(string Uri)
188 {
189 URI U(Uri);
190
191 const MethodConfig *Config = GetConfig(U.Access);
192 if (Config == 0)
193 return string();
194
195 /* Single-Instance methods get exactly one queue per URI. This is
196 also used for the Access queue method */
197 if (Config->SingleInstance == true || QueueMode == QueueAccess)
198 return U.Access;
199
200 return U.Access + ':' + U.Host;
201 }
202 /*}}}*/
203 // Acquire::GetConfig - Fetch the configuration information /*{{{*/
204 // ---------------------------------------------------------------------
205 /* This locates the configuration structure for an access method. If
206 a config structure cannot be found a Worker will be created to
207 retrieve it */
208 pkgAcquire::MethodConfig *pkgAcquire::GetConfig(string Access)
209 {
210 // Search for an existing config
211 MethodConfig *Conf;
212 for (Conf = Configs; Conf != 0; Conf = Conf->Next)
213 if (Conf->Access == Access)
214 return Conf;
215
216 // Create the new config class
217 Conf = new MethodConfig;
218 Conf->Access = Access;
219 Conf->Next = Configs;
220 Configs = Conf;
221
222 // Create the worker to fetch the configuration
223 Worker Work(Conf);
224 if (Work.Start() == false)
225 return 0;
226
227 return Conf;
228 }
229 /*}}}*/
230 // Acquire::SetFds - Deal with readable FDs /*{{{*/
231 // ---------------------------------------------------------------------
232 /* Collect FDs that have activity monitors into the fd sets */
233 void pkgAcquire::SetFds(int &Fd,fd_set *RSet,fd_set *WSet)
234 {
235 for (Worker *I = Workers; I != 0; I = I->NextAcquire)
236 {
237 if (I->InReady == true && I->InFd >= 0)
238 {
239 if (Fd < I->InFd)
240 Fd = I->InFd;
241 FD_SET(I->InFd,RSet);
242 }
243 if (I->OutReady == true && I->OutFd >= 0)
244 {
245 if (Fd < I->OutFd)
246 Fd = I->OutFd;
247 FD_SET(I->OutFd,WSet);
248 }
249 }
250 }
251 /*}}}*/
252 // Acquire::RunFds - Deal with active FDs /*{{{*/
253 // ---------------------------------------------------------------------
254 /* Dispatch active FDs over to the proper workers. It is very important
255 that a worker never be erased while this is running! The queue class
256 should never erase a worker except during shutdown processing. */
257 void pkgAcquire::RunFds(fd_set *RSet,fd_set *WSet)
258 {
259 for (Worker *I = Workers; I != 0; I = I->NextAcquire)
260 {
261 if (I->InFd >= 0 && FD_ISSET(I->InFd,RSet) != 0)
262 I->InFdReady();
263 if (I->OutFd >= 0 && FD_ISSET(I->OutFd,WSet) != 0)
264 I->OutFdReady();
265 }
266 }
267 /*}}}*/
268 // Acquire::Run - Run the fetch sequence /*{{{*/
269 // ---------------------------------------------------------------------
270 /* This runs the queues. It manages a select loop for all of the
271 Worker tasks. The workers interact with the queues and items to
272 manage the actual fetch. */
273 bool pkgAcquire::Run()
274 {
275 Running = true;
276
277 for (Queue *I = Queues; I != 0; I = I->Next)
278 I->Startup();
279
280 if (Log != 0)
281 Log->Start();
282
283 // Run till all things have been acquired
284 struct timeval tv;
285 tv.tv_sec = 0;
286 tv.tv_usec = 500000;
287 while (ToFetch > 0)
288 {
289 fd_set RFds;
290 fd_set WFds;
291 int Highest = 0;
292 FD_ZERO(&RFds);
293 FD_ZERO(&WFds);
294 SetFds(Highest,&RFds,&WFds);
295
296 int Res = select(Highest+1,&RFds,&WFds,0,&tv);
297 if (Res < 0)
298 {
299 _error->Errno("select","Select has failed");
300 break;
301 }
302
303 RunFds(&RFds,&WFds);
304 if (_error->PendingError() == true)
305 break;
306
307 // Timeout, notify the log class
308 if (Res == 0 || (Log != 0 && Log->Update == true))
309 {
310 tv.tv_usec = 500000;
311 for (Worker *I = Workers; I != 0; I = I->NextAcquire)
312 I->Pulse();
313 if (Log != 0)
314 Log->Pulse(this);
315 }
316 }
317
318 if (Log != 0)
319 Log->Stop();
320
321 // Shut down the acquire bits
322 Running = false;
323 for (Queue *I = Queues; I != 0; I = I->Next)
324 I->Shutdown();
325
326 return !_error->PendingError();
327 }
328 /*}}}*/
329 // Acquire::Bump - Called when an item is dequeued /*{{{*/
330 // ---------------------------------------------------------------------
331 /* This routine bumps idle queues in hopes that they will be able to fetch
332 the dequeued item */
333 void pkgAcquire::Bump()
334 {
335 for (Queue *I = Queues; I != 0; I = I->Next)
336 I->Bump();
337 }
338 /*}}}*/
339 // Acquire::WorkerStep - Step to the next worker /*{{{*/
340 // ---------------------------------------------------------------------
341 /* Not inlined to advoid including acquire-worker.h */
342 pkgAcquire::Worker *pkgAcquire::WorkerStep(Worker *I)
343 {
344 return I->NextAcquire;
345 };
346 /*}}}*/
347 // Acquire::Clean - Cleans a directory /*{{{*/
348 // ---------------------------------------------------------------------
349 /* This is a bit simplistic, it looks at every file in the dir and sees
350 if it is part of the download set. */
351 bool pkgAcquire::Clean(string Dir)
352 {
353 DIR *D = opendir(Dir.c_str());
354 if (D == 0)
355 return _error->Errno("opendir","Unable to read %s",Dir.c_str());
356
357 string StartDir = SafeGetCWD();
358 if (chdir(Dir.c_str()) != 0)
359 {
360 closedir(D);
361 return _error->Errno("chdir","Unable to change to ",Dir.c_str());
362 }
363
364 for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D))
365 {
366 // Skip some files..
367 if (strcmp(Dir->d_name,"lock") == 0 ||
368 strcmp(Dir->d_name,"partial") == 0 ||
369 strcmp(Dir->d_name,".") == 0 ||
370 strcmp(Dir->d_name,"..") == 0)
371 continue;
372
373 // Look in the get list
374 vector<Item *>::iterator I = Items.begin();
375 for (; I != Items.end(); I++)
376 if (flNotDir((*I)->DestFile) == Dir->d_name)
377 break;
378
379 // Nothing found, nuke it
380 if (I == Items.end())
381 unlink(Dir->d_name);
382 };
383
384 chdir(StartDir.c_str());
385 closedir(D);
386 return true;
387 }
388 /*}}}*/
389 // Acquire::MethodConfig::MethodConfig - Constructor /*{{{*/
390 // ---------------------------------------------------------------------
391 /* */
392 pkgAcquire::MethodConfig::MethodConfig()
393 {
394 SingleInstance = false;
395 PreScan = false;
396 Pipeline = false;
397 SendConfig = false;
398 Next = 0;
399 }
400 /*}}}*/
401 // Acquire::TotalNeeded - Number of bytes to fetch /*{{{*/
402 // ---------------------------------------------------------------------
403 /* This is the total number of bytes needed */
404 unsigned long pkgAcquire::TotalNeeded()
405 {
406 unsigned long Total = 0;
407 for (pkgAcquire::Item **I = ItemsBegin(); I != ItemsEnd(); I++)
408 Total += (*I)->FileSize;
409 return Total;
410 }
411 /*}}}*/
412 // Acquire::FetchNeeded - Number of bytes needed to get /*{{{*/
413 // ---------------------------------------------------------------------
414 /* This is the number of bytes that is not local */
415 unsigned long pkgAcquire::FetchNeeded()
416 {
417 unsigned long Total = 0;
418 for (pkgAcquire::Item **I = ItemsBegin(); I != ItemsEnd(); I++)
419 if ((*I)->Local == false)
420 Total += (*I)->FileSize;
421 return Total;
422 }
423 /*}}}*/
424
425 // Queue::Queue - Constructor /*{{{*/
426 // ---------------------------------------------------------------------
427 /* */
428 pkgAcquire::Queue::Queue(string Name,pkgAcquire *Owner) : Name(Name),
429 Owner(Owner)
430 {
431 Items = 0;
432 Next = 0;
433 Workers = 0;
434 }
435 /*}}}*/
436 // Queue::~Queue - Destructor /*{{{*/
437 // ---------------------------------------------------------------------
438 /* */
439 pkgAcquire::Queue::~Queue()
440 {
441 Shutdown();
442
443 while (Items != 0)
444 {
445 QItem *Jnk = Items;
446 Items = Items->Next;
447 delete Jnk;
448 }
449 }
450 /*}}}*/
451 // Queue::Enqueue - Queue an item to the queue /*{{{*/
452 // ---------------------------------------------------------------------
453 /* */
454 void pkgAcquire::Queue::Enqueue(ItemDesc &Item)
455 {
456 // Create a new item
457 QItem *I = new QItem;
458 I->Next = Items;
459 Items = I;
460 *I = Item;
461
462 Item.Owner->QueueCounter++;
463 if (Items->Next == 0)
464 Cycle();
465 }
466 /*}}}*/
467 // Queue::Dequeue - Remove an item from the queue /*{{{*/
468 // ---------------------------------------------------------------------
469 /* We return true if we hit something*/
470 bool pkgAcquire::Queue::Dequeue(Item *Owner)
471 {
472 bool Res = false;
473
474 QItem **I = &Items;
475 for (; *I != 0;)
476 {
477 if ((*I)->Owner == Owner)
478 {
479 QItem *Jnk= *I;
480 *I = (*I)->Next;
481 Owner->QueueCounter--;
482 delete Jnk;
483 Res = true;
484 }
485 else
486 I = &(*I)->Next;
487 }
488
489 return Res;
490 }
491 /*}}}*/
492 // Queue::Startup - Start the worker processes /*{{{*/
493 // ---------------------------------------------------------------------
494 /* */
495 bool pkgAcquire::Queue::Startup()
496 {
497 Shutdown();
498
499 URI U(Name);
500 pkgAcquire::MethodConfig *Cnf = Owner->GetConfig(U.Access);
501 if (Cnf == 0)
502 return false;
503
504 Workers = new Worker(this,Cnf,Owner->Log);
505 Owner->Add(Workers);
506 if (Workers->Start() == false)
507 return false;
508
509 return Cycle();
510 }
511 /*}}}*/
512 // Queue::Shutdown - Shutdown the worker processes /*{{{*/
513 // ---------------------------------------------------------------------
514 /* */
515 bool pkgAcquire::Queue::Shutdown()
516 {
517 // Delete all of the workers
518 while (Workers != 0)
519 {
520 pkgAcquire::Worker *Jnk = Workers;
521 Workers = Workers->NextQueue;
522 Owner->Remove(Jnk);
523 delete Jnk;
524 }
525
526 return true;
527 }
528 /*}}}*/
529 // Queue::Finditem - Find a URI in the item list /*{{{*/
530 // ---------------------------------------------------------------------
531 /* */
532 pkgAcquire::Queue::QItem *pkgAcquire::Queue::FindItem(string URI,pkgAcquire::Worker *Owner)
533 {
534 for (QItem *I = Items; I != 0; I = I->Next)
535 if (I->URI == URI && I->Worker == Owner)
536 return I;
537 return 0;
538 }
539 /*}}}*/
540 // Queue::ItemDone - Item has been completed /*{{{*/
541 // ---------------------------------------------------------------------
542 /* The worker signals this which causes the item to be removed from the
543 queue. If this is the last queue instance then it is removed from the
544 main queue too.*/
545 bool pkgAcquire::Queue::ItemDone(QItem *Itm)
546 {
547 if (Itm->Owner->QueueCounter <= 1)
548 Owner->Dequeue(Itm->Owner);
549 else
550 {
551 Dequeue(Itm->Owner);
552 Owner->Bump();
553 }
554
555 return Cycle();
556 }
557 /*}}}*/
558 // Queue::Cycle - Queue new items into the method /*{{{*/
559 // ---------------------------------------------------------------------
560 /* This locates a new idle item and sends it to the worker */
561 bool pkgAcquire::Queue::Cycle()
562 {
563 if (Items == 0 || Workers == 0)
564 return true;
565
566 // Look for a queable item
567 QItem *I = Items;
568 for (; I != 0; I = I->Next)
569 if (I->Owner->Status == pkgAcquire::Item::StatIdle)
570 break;
571
572 // Nothing to do, queue is idle.
573 if (I == 0)
574 return true;
575
576 I->Worker = Workers;
577 I->Owner->Status = pkgAcquire::Item::StatFetching;
578 return Workers->QueueItem(I);
579 }
580 /*}}}*/
581 // Queue::Bump - Fetch any pending objects if we are idle /*{{{*/
582 // ---------------------------------------------------------------------
583 /* */
584 void pkgAcquire::Queue::Bump()
585 {
586 }
587 /*}}}*/
588
589 // AcquireStatus::pkgAcquireStatus - Constructor /*{{{*/
590 // ---------------------------------------------------------------------
591 /* */
592 pkgAcquireStatus::pkgAcquireStatus()
593 {
594 Start();
595 }
596 /*}}}*/
597 // AcquireStatus::Pulse - Called periodically /*{{{*/
598 // ---------------------------------------------------------------------
599 /* This computes some internal state variables for the derived classes to
600 use. It generates the current downloaded bytes and total bytes to download
601 as well as the current CPS estimate. */
602 void pkgAcquireStatus::Pulse(pkgAcquire *Owner)
603 {
604 TotalBytes = 0;
605 CurrentBytes = 0;
606
607 // Compute the total number of bytes to fetch
608 unsigned int Unknown = 0;
609 unsigned int Count = 0;
610 for (pkgAcquire::Item **I = Owner->ItemsBegin(); I != Owner->ItemsEnd();
611 I++, Count++)
612 {
613 // Totally ignore local items
614 if ((*I)->Local == true)
615 continue;
616
617 TotalBytes += (*I)->FileSize;
618 if ((*I)->Complete == true)
619 CurrentBytes += (*I)->FileSize;
620 if ((*I)->FileSize == 0 && (*I)->Complete == false)
621 Unknown++;
622 }
623
624 // Compute the current completion
625 for (pkgAcquire::Worker *I = Owner->WorkersBegin(); I != 0;
626 I = Owner->WorkerStep(I))
627 if (I->CurrentItem != 0 && I->CurrentItem->Owner->Complete == false)
628 CurrentBytes += I->CurrentSize;
629
630 // Normalize the figures and account for unknown size downloads
631 if (TotalBytes <= 0)
632 TotalBytes = 1;
633 if (Unknown == Count)
634 TotalBytes = Unknown;
635 else
636 TotalBytes += TotalBytes/(Count - Unknown)*Unknown;
637
638 // Compute the CPS
639 struct timeval NewTime;
640 gettimeofday(&NewTime,0);
641 if (NewTime.tv_sec - Time.tv_sec == 6 && NewTime.tv_usec > Time.tv_usec ||
642 NewTime.tv_sec - Time.tv_sec > 6)
643 {
644 // Compute the delta time with full accuracy
645 long usdiff = NewTime.tv_usec - Time.tv_usec;
646 long sdiff = NewTime.tv_sec - Time.tv_sec;
647
648 // Borrow
649 if (usdiff < 0)
650 {
651 usdiff += 1000000;
652 sdiff--;
653 }
654
655 // Compute the CPS value
656 CurrentCPS = (CurrentBytes - LastBytes)/(sdiff + usdiff/1000000.0);
657 LastBytes = CurrentBytes;
658 ElapsedTime = NewTime.tv_sec - StartTime.tv_sec;
659 Time = NewTime;
660 }
661 }
662 /*}}}*/
663 // AcquireStatus::Start - Called when the download is started /*{{{*/
664 // ---------------------------------------------------------------------
665 /* We just reset the counters */
666 void pkgAcquireStatus::Start()
667 {
668 gettimeofday(&Time,0);
669 gettimeofday(&StartTime,0);
670 LastBytes = 0;
671 CurrentCPS = 0;
672 CurrentBytes = 0;
673 TotalBytes = 0;
674 FetchedBytes = 0;
675 ElapsedTime = 0;
676 }
677 /*}}}*/
678 // AcquireStatus::Stop - Finished downloading /*{{{*/
679 // ---------------------------------------------------------------------
680 /* This accurately computes the elapsed time and the total overall CPS. */
681 void pkgAcquireStatus::Stop()
682 {
683 // Compute the CPS and elapsed time
684 struct timeval NewTime;
685 gettimeofday(&NewTime,0);
686
687 // Compute the delta time with full accuracy
688 long usdiff = NewTime.tv_usec - StartTime.tv_usec;
689 long sdiff = NewTime.tv_sec - StartTime.tv_sec;
690
691 // Borrow
692 if (usdiff < 0)
693 {
694 usdiff += 1000000;
695 sdiff--;
696 }
697
698 // Compute the CPS value
699 CurrentCPS = FetchedBytes/(sdiff + usdiff/1000000.0);
700 LastBytes = CurrentBytes;
701 ElapsedTime = sdiff;
702 }
703 /*}}}*/
704 // AcquireStatus::Fetched - Called when a byte set has been fetched /*{{{*/
705 // ---------------------------------------------------------------------
706 /* This is used to get accurate final transfer rate reporting. */
707 void pkgAcquireStatus::Fetched(unsigned long Size,unsigned long Resume)
708 {
709 FetchedBytes += Size - Resume;
710 }
711 /*}}}*/