]> git.saurik.com Git - apt.git/blob - apt-pkg/acquire.cc
Fixed problem with not being able to open files
[apt.git] / apt-pkg / acquire.cc
1 // -*- mode: cpp; mode: fold -*-
2 // Description /*{{{*/
3 // $Id: acquire.cc,v 1.16 1998/11/14 01:39:45 jgg Exp $
4 /* ######################################################################
5
6 Acquire - File Acquiration
7
8 The core element for the schedual system is the concept of a named
9 queue. Each queue is unique and each queue has a name derived from the
10 URI. The degree of paralization can be controled by how the queue
11 name is derived from the URI.
12
13 ##################################################################### */
14 /*}}}*/
15 // Include Files /*{{{*/
16 #ifdef __GNUG__
17 #pragma implementation "apt-pkg/acquire.h"
18 #endif
19 #include <apt-pkg/acquire.h>
20 #include <apt-pkg/acquire-item.h>
21 #include <apt-pkg/acquire-worker.h>
22 #include <apt-pkg/configuration.h>
23 #include <apt-pkg/error.h>
24 #include <strutl.h>
25
26 #include <dirent.h>
27 #include <sys/time.h>
28 /*}}}*/
29
30 // Acquire::pkgAcquire - Constructor /*{{{*/
31 // ---------------------------------------------------------------------
32 /* We grab some runtime state from the configuration space */
33 pkgAcquire::pkgAcquire(pkgAcquireStatus *Log) : Log(Log)
34 {
35 Queues = 0;
36 Configs = 0;
37 Workers = 0;
38 ToFetch = 0;
39 Running = false;
40
41 string Mode = _config->Find("Acquire::Queue-Mode","host");
42 if (strcasecmp(Mode.c_str(),"host") == 0)
43 QueueMode = QueueHost;
44 if (strcasecmp(Mode.c_str(),"access") == 0)
45 QueueMode = QueueAccess;
46
47 Debug = _config->FindB("Debug::pkgAcquire",false);
48 }
49 /*}}}*/
50 // Acquire::~pkgAcquire - Destructor /*{{{*/
51 // ---------------------------------------------------------------------
52 /* Free our memory, clean up the queues (destroy the workers) */
53 pkgAcquire::~pkgAcquire()
54 {
55 while (Items.size() != 0)
56 delete Items[0];
57
58 while (Configs != 0)
59 {
60 MethodConfig *Jnk = Configs;
61 Configs = Configs->Next;
62 delete Jnk;
63 }
64
65 while (Queues != 0)
66 {
67 Queue *Jnk = Queues;
68 Queues = Queues->Next;
69 delete Jnk;
70 }
71 }
72 /*}}}*/
73 // Acquire::Add - Add a new item /*{{{*/
74 // ---------------------------------------------------------------------
75 /* This puts an item on the acquire list. This list is mainly for tracking
76 item status */
77 void pkgAcquire::Add(Item *Itm)
78 {
79 Items.push_back(Itm);
80 }
81 /*}}}*/
82 // Acquire::Remove - Remove a item /*{{{*/
83 // ---------------------------------------------------------------------
84 /* Remove an item from the acquire list. This is usually not used.. */
85 void pkgAcquire::Remove(Item *Itm)
86 {
87 for (vector<Item *>::iterator I = Items.begin(); I < Items.end(); I++)
88 {
89 if (*I == Itm)
90 Items.erase(I);
91 }
92 }
93 /*}}}*/
94 // Acquire::Add - Add a worker /*{{{*/
95 // ---------------------------------------------------------------------
96 /* A list of workers is kept so that the select loop can direct their FD
97 usage. */
98 void pkgAcquire::Add(Worker *Work)
99 {
100 Work->NextAcquire = Workers;
101 Workers = Work;
102 }
103 /*}}}*/
104 // Acquire::Remove - Remove a worker /*{{{*/
105 // ---------------------------------------------------------------------
106 /* A worker has died. This can not be done while the select loop is running
107 as it would require that RunFds could handling a changing list state and
108 it cant.. */
109 void pkgAcquire::Remove(Worker *Work)
110 {
111 if (Running == true)
112 abort();
113
114 Worker **I = &Workers;
115 for (; *I != 0;)
116 {
117 if (*I == Work)
118 *I = (*I)->NextAcquire;
119 else
120 I = &(*I)->NextAcquire;
121 }
122 }
123 /*}}}*/
124 // Acquire::Enqueue - Queue an URI for fetching /*{{{*/
125 // ---------------------------------------------------------------------
126 /* This is the entry point for an item. An item calls this function when
127 it is construction which creates a queue (based on the current queue
128 mode) and puts the item in that queue. If the system is running then
129 the queue might be started. */
130 void pkgAcquire::Enqueue(ItemDesc &Item)
131 {
132 // Determine which queue to put the item in
133 const MethodConfig *Config;
134 string Name = QueueName(Item.URI,Config);
135 if (Name.empty() == true)
136 return;
137
138 // Find the queue structure
139 Queue *I = Queues;
140 for (; I != 0 && I->Name != Name; I = I->Next);
141 if (I == 0)
142 {
143 I = new Queue(Name,this);
144 I->Next = Queues;
145 Queues = I;
146
147 if (Running == true)
148 I->Startup();
149 }
150
151 // See if this is a local only URI
152 if (Config->LocalOnly == true && Item.Owner->Complete == false)
153 Item.Owner->Local = true;
154 Item.Owner->Status = Item::StatIdle;
155
156 // Queue it into the named queue
157 I->Enqueue(Item);
158 ToFetch++;
159
160 // Some trace stuff
161 if (Debug == true)
162 {
163 clog << "Fetching " << Item.URI << endl;
164 clog << " to " << Item.Owner->DestFile << endl;
165 clog << " Queue is: " << Name << endl;
166 }
167 }
168 /*}}}*/
169 // Acquire::Dequeue - Remove an item from all queues /*{{{*/
170 // ---------------------------------------------------------------------
171 /* This is called when an item is finished being fetched. It removes it
172 from all the queues */
173 void pkgAcquire::Dequeue(Item *Itm)
174 {
175 Queue *I = Queues;
176 bool Res = false;
177 for (; I != 0; I = I->Next)
178 Res |= I->Dequeue(Itm);
179
180 if (Debug == true)
181 clog << "Dequeuing " << Itm->DestFile << endl;
182 if (Res == true)
183 ToFetch--;
184 }
185 /*}}}*/
186 // Acquire::QueueName - Return the name of the queue for this URI /*{{{*/
187 // ---------------------------------------------------------------------
188 /* The string returned depends on the configuration settings and the
189 method parameters. Given something like http://foo.org/bar it can
190 return http://foo.org or http */
191 string pkgAcquire::QueueName(string Uri,MethodConfig const *&Config)
192 {
193 URI U(Uri);
194
195 Config = GetConfig(U.Access);
196 if (Config == 0)
197 return string();
198
199 /* Single-Instance methods get exactly one queue per URI. This is
200 also used for the Access queue method */
201 if (Config->SingleInstance == true || QueueMode == QueueAccess)
202 return U.Access;
203
204 return U.Access + ':' + U.Host;
205 }
206 /*}}}*/
207 // Acquire::GetConfig - Fetch the configuration information /*{{{*/
208 // ---------------------------------------------------------------------
209 /* This locates the configuration structure for an access method. If
210 a config structure cannot be found a Worker will be created to
211 retrieve it */
212 pkgAcquire::MethodConfig *pkgAcquire::GetConfig(string Access)
213 {
214 // Search for an existing config
215 MethodConfig *Conf;
216 for (Conf = Configs; Conf != 0; Conf = Conf->Next)
217 if (Conf->Access == Access)
218 return Conf;
219
220 // Create the new config class
221 Conf = new MethodConfig;
222 Conf->Access = Access;
223 Conf->Next = Configs;
224 Configs = Conf;
225
226 // Create the worker to fetch the configuration
227 Worker Work(Conf);
228 if (Work.Start() == false)
229 return 0;
230
231 return Conf;
232 }
233 /*}}}*/
234 // Acquire::SetFds - Deal with readable FDs /*{{{*/
235 // ---------------------------------------------------------------------
236 /* Collect FDs that have activity monitors into the fd sets */
237 void pkgAcquire::SetFds(int &Fd,fd_set *RSet,fd_set *WSet)
238 {
239 for (Worker *I = Workers; I != 0; I = I->NextAcquire)
240 {
241 if (I->InReady == true && I->InFd >= 0)
242 {
243 if (Fd < I->InFd)
244 Fd = I->InFd;
245 FD_SET(I->InFd,RSet);
246 }
247 if (I->OutReady == true && I->OutFd >= 0)
248 {
249 if (Fd < I->OutFd)
250 Fd = I->OutFd;
251 FD_SET(I->OutFd,WSet);
252 }
253 }
254 }
255 /*}}}*/
256 // Acquire::RunFds - Deal with active FDs /*{{{*/
257 // ---------------------------------------------------------------------
258 /* Dispatch active FDs over to the proper workers. It is very important
259 that a worker never be erased while this is running! The queue class
260 should never erase a worker except during shutdown processing. */
261 void pkgAcquire::RunFds(fd_set *RSet,fd_set *WSet)
262 {
263 for (Worker *I = Workers; I != 0; I = I->NextAcquire)
264 {
265 if (I->InFd >= 0 && FD_ISSET(I->InFd,RSet) != 0)
266 I->InFdReady();
267 if (I->OutFd >= 0 && FD_ISSET(I->OutFd,WSet) != 0)
268 I->OutFdReady();
269 }
270 }
271 /*}}}*/
272 // Acquire::Run - Run the fetch sequence /*{{{*/
273 // ---------------------------------------------------------------------
274 /* This runs the queues. It manages a select loop for all of the
275 Worker tasks. The workers interact with the queues and items to
276 manage the actual fetch. */
277 bool pkgAcquire::Run()
278 {
279 Running = true;
280
281 for (Queue *I = Queues; I != 0; I = I->Next)
282 I->Startup();
283
284 if (Log != 0)
285 Log->Start();
286
287 // Run till all things have been acquired
288 struct timeval tv;
289 tv.tv_sec = 0;
290 tv.tv_usec = 500000;
291 while (ToFetch > 0)
292 {
293 fd_set RFds;
294 fd_set WFds;
295 int Highest = 0;
296 FD_ZERO(&RFds);
297 FD_ZERO(&WFds);
298 SetFds(Highest,&RFds,&WFds);
299
300 int Res = select(Highest+1,&RFds,&WFds,0,&tv);
301 if (Res < 0)
302 {
303 _error->Errno("select","Select has failed");
304 break;
305 }
306
307 RunFds(&RFds,&WFds);
308 if (_error->PendingError() == true)
309 break;
310
311 // Timeout, notify the log class
312 if (Res == 0 || (Log != 0 && Log->Update == true))
313 {
314 tv.tv_usec = 500000;
315 for (Worker *I = Workers; I != 0; I = I->NextAcquire)
316 I->Pulse();
317 if (Log != 0)
318 Log->Pulse(this);
319 }
320 }
321
322 if (Log != 0)
323 Log->Stop();
324
325 // Shut down the acquire bits
326 Running = false;
327 for (Queue *I = Queues; I != 0; I = I->Next)
328 I->Shutdown();
329
330 return !_error->PendingError();
331 }
332 /*}}}*/
333 // Acquire::Bump - Called when an item is dequeued /*{{{*/
334 // ---------------------------------------------------------------------
335 /* This routine bumps idle queues in hopes that they will be able to fetch
336 the dequeued item */
337 void pkgAcquire::Bump()
338 {
339 for (Queue *I = Queues; I != 0; I = I->Next)
340 I->Bump();
341 }
342 /*}}}*/
343 // Acquire::WorkerStep - Step to the next worker /*{{{*/
344 // ---------------------------------------------------------------------
345 /* Not inlined to advoid including acquire-worker.h */
346 pkgAcquire::Worker *pkgAcquire::WorkerStep(Worker *I)
347 {
348 return I->NextAcquire;
349 };
350 /*}}}*/
351 // Acquire::Clean - Cleans a directory /*{{{*/
352 // ---------------------------------------------------------------------
353 /* This is a bit simplistic, it looks at every file in the dir and sees
354 if it is part of the download set. */
355 bool pkgAcquire::Clean(string Dir)
356 {
357 DIR *D = opendir(Dir.c_str());
358 if (D == 0)
359 return _error->Errno("opendir","Unable to read %s",Dir.c_str());
360
361 string StartDir = SafeGetCWD();
362 if (chdir(Dir.c_str()) != 0)
363 {
364 closedir(D);
365 return _error->Errno("chdir","Unable to change to ",Dir.c_str());
366 }
367
368 for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D))
369 {
370 // Skip some files..
371 if (strcmp(Dir->d_name,"lock") == 0 ||
372 strcmp(Dir->d_name,"partial") == 0 ||
373 strcmp(Dir->d_name,".") == 0 ||
374 strcmp(Dir->d_name,"..") == 0)
375 continue;
376
377 // Look in the get list
378 vector<Item *>::iterator I = Items.begin();
379 for (; I != Items.end(); I++)
380 if (flNotDir((*I)->DestFile) == Dir->d_name)
381 break;
382
383 // Nothing found, nuke it
384 if (I == Items.end())
385 unlink(Dir->d_name);
386 };
387
388 chdir(StartDir.c_str());
389 closedir(D);
390 return true;
391 }
392 /*}}}*/
393 // Acquire::TotalNeeded - Number of bytes to fetch /*{{{*/
394 // ---------------------------------------------------------------------
395 /* This is the total number of bytes needed */
396 unsigned long pkgAcquire::TotalNeeded()
397 {
398 unsigned long Total = 0;
399 for (pkgAcquire::Item **I = ItemsBegin(); I != ItemsEnd(); I++)
400 Total += (*I)->FileSize;
401 return Total;
402 }
403 /*}}}*/
404 // Acquire::FetchNeeded - Number of bytes needed to get /*{{{*/
405 // ---------------------------------------------------------------------
406 /* This is the number of bytes that is not local */
407 unsigned long pkgAcquire::FetchNeeded()
408 {
409 unsigned long Total = 0;
410 for (pkgAcquire::Item **I = ItemsBegin(); I != ItemsEnd(); I++)
411 if ((*I)->Local == false)
412 Total += (*I)->FileSize;
413 return Total;
414 }
415 /*}}}*/
416
417 // Acquire::MethodConfig::MethodConfig - Constructor /*{{{*/
418 // ---------------------------------------------------------------------
419 /* */
420 pkgAcquire::MethodConfig::MethodConfig()
421 {
422 SingleInstance = false;
423 PreScan = false;
424 Pipeline = false;
425 SendConfig = false;
426 LocalOnly = false;
427 Next = 0;
428 }
429 /*}}}*/
430
431 // Queue::Queue - Constructor /*{{{*/
432 // ---------------------------------------------------------------------
433 /* */
434 pkgAcquire::Queue::Queue(string Name,pkgAcquire *Owner) : Name(Name),
435 Owner(Owner)
436 {
437 Items = 0;
438 Next = 0;
439 Workers = 0;
440 }
441 /*}}}*/
442 // Queue::~Queue - Destructor /*{{{*/
443 // ---------------------------------------------------------------------
444 /* */
445 pkgAcquire::Queue::~Queue()
446 {
447 Shutdown();
448
449 while (Items != 0)
450 {
451 QItem *Jnk = Items;
452 Items = Items->Next;
453 delete Jnk;
454 }
455 }
456 /*}}}*/
457 // Queue::Enqueue - Queue an item to the queue /*{{{*/
458 // ---------------------------------------------------------------------
459 /* */
460 void pkgAcquire::Queue::Enqueue(ItemDesc &Item)
461 {
462 // Create a new item
463 QItem *I = new QItem;
464 I->Next = Items;
465 Items = I;
466 *I = Item;
467
468 Item.Owner->QueueCounter++;
469 if (Items->Next == 0)
470 Cycle();
471 }
472 /*}}}*/
473 // Queue::Dequeue - Remove an item from the queue /*{{{*/
474 // ---------------------------------------------------------------------
475 /* We return true if we hit something*/
476 bool pkgAcquire::Queue::Dequeue(Item *Owner)
477 {
478 bool Res = false;
479
480 QItem **I = &Items;
481 for (; *I != 0;)
482 {
483 if ((*I)->Owner == Owner)
484 {
485 QItem *Jnk= *I;
486 *I = (*I)->Next;
487 Owner->QueueCounter--;
488 delete Jnk;
489 Res = true;
490 }
491 else
492 I = &(*I)->Next;
493 }
494
495 return Res;
496 }
497 /*}}}*/
498 // Queue::Startup - Start the worker processes /*{{{*/
499 // ---------------------------------------------------------------------
500 /* */
501 bool pkgAcquire::Queue::Startup()
502 {
503 Shutdown();
504
505 URI U(Name);
506 pkgAcquire::MethodConfig *Cnf = Owner->GetConfig(U.Access);
507 if (Cnf == 0)
508 return false;
509
510 Workers = new Worker(this,Cnf,Owner->Log);
511 Owner->Add(Workers);
512 if (Workers->Start() == false)
513 return false;
514
515 return Cycle();
516 }
517 /*}}}*/
518 // Queue::Shutdown - Shutdown the worker processes /*{{{*/
519 // ---------------------------------------------------------------------
520 /* */
521 bool pkgAcquire::Queue::Shutdown()
522 {
523 // Delete all of the workers
524 while (Workers != 0)
525 {
526 pkgAcquire::Worker *Jnk = Workers;
527 Workers = Workers->NextQueue;
528 Owner->Remove(Jnk);
529 delete Jnk;
530 }
531
532 return true;
533 }
534 /*}}}*/
535 // Queue::Finditem - Find a URI in the item list /*{{{*/
536 // ---------------------------------------------------------------------
537 /* */
538 pkgAcquire::Queue::QItem *pkgAcquire::Queue::FindItem(string URI,pkgAcquire::Worker *Owner)
539 {
540 for (QItem *I = Items; I != 0; I = I->Next)
541 if (I->URI == URI && I->Worker == Owner)
542 return I;
543 return 0;
544 }
545 /*}}}*/
546 // Queue::ItemDone - Item has been completed /*{{{*/
547 // ---------------------------------------------------------------------
548 /* The worker signals this which causes the item to be removed from the
549 queue. If this is the last queue instance then it is removed from the
550 main queue too.*/
551 bool pkgAcquire::Queue::ItemDone(QItem *Itm)
552 {
553 if (Itm->Owner->QueueCounter <= 1)
554 Owner->Dequeue(Itm->Owner);
555 else
556 {
557 Dequeue(Itm->Owner);
558 Owner->Bump();
559 }
560
561 return Cycle();
562 }
563 /*}}}*/
564 // Queue::Cycle - Queue new items into the method /*{{{*/
565 // ---------------------------------------------------------------------
566 /* This locates a new idle item and sends it to the worker */
567 bool pkgAcquire::Queue::Cycle()
568 {
569 if (Items == 0 || Workers == 0)
570 return true;
571
572 // Look for a queable item
573 QItem *I = Items;
574 for (; I != 0; I = I->Next)
575 if (I->Owner->Status == pkgAcquire::Item::StatIdle)
576 break;
577
578 // Nothing to do, queue is idle.
579 if (I == 0)
580 return true;
581
582 I->Worker = Workers;
583 I->Owner->Status = pkgAcquire::Item::StatFetching;
584 return Workers->QueueItem(I);
585 }
586 /*}}}*/
587 // Queue::Bump - Fetch any pending objects if we are idle /*{{{*/
588 // ---------------------------------------------------------------------
589 /* */
590 void pkgAcquire::Queue::Bump()
591 {
592 }
593 /*}}}*/
594
595 // AcquireStatus::pkgAcquireStatus - Constructor /*{{{*/
596 // ---------------------------------------------------------------------
597 /* */
598 pkgAcquireStatus::pkgAcquireStatus()
599 {
600 Start();
601 }
602 /*}}}*/
603 // AcquireStatus::Pulse - Called periodically /*{{{*/
604 // ---------------------------------------------------------------------
605 /* This computes some internal state variables for the derived classes to
606 use. It generates the current downloaded bytes and total bytes to download
607 as well as the current CPS estimate. */
608 void pkgAcquireStatus::Pulse(pkgAcquire *Owner)
609 {
610 TotalBytes = 0;
611 CurrentBytes = 0;
612
613 // Compute the total number of bytes to fetch
614 unsigned int Unknown = 0;
615 unsigned int Count = 0;
616 for (pkgAcquire::Item **I = Owner->ItemsBegin(); I != Owner->ItemsEnd();
617 I++, Count++)
618 {
619 // Totally ignore local items
620 if ((*I)->Local == true)
621 continue;
622
623 TotalBytes += (*I)->FileSize;
624 if ((*I)->Complete == true)
625 CurrentBytes += (*I)->FileSize;
626 if ((*I)->FileSize == 0 && (*I)->Complete == false)
627 Unknown++;
628 }
629
630 // Compute the current completion
631 for (pkgAcquire::Worker *I = Owner->WorkersBegin(); I != 0;
632 I = Owner->WorkerStep(I))
633 if (I->CurrentItem != 0 && I->CurrentItem->Owner->Complete == false)
634 CurrentBytes += I->CurrentSize;
635
636 // Normalize the figures and account for unknown size downloads
637 if (TotalBytes <= 0)
638 TotalBytes = 1;
639 if (Unknown == Count)
640 TotalBytes = Unknown;
641 else
642 TotalBytes += TotalBytes/(Count - Unknown)*Unknown;
643
644 // Compute the CPS
645 struct timeval NewTime;
646 gettimeofday(&NewTime,0);
647 if (NewTime.tv_sec - Time.tv_sec == 6 && NewTime.tv_usec > Time.tv_usec ||
648 NewTime.tv_sec - Time.tv_sec > 6)
649 {
650 // Compute the delta time with full accuracy
651 long usdiff = NewTime.tv_usec - Time.tv_usec;
652 long sdiff = NewTime.tv_sec - Time.tv_sec;
653
654 // Borrow
655 if (usdiff < 0)
656 {
657 usdiff += 1000000;
658 sdiff--;
659 }
660
661 // Compute the CPS value
662 if (sdiff == 0 && usdiff == 0)
663 CurrentCPS = 0;
664 else
665 CurrentCPS = (CurrentBytes - LastBytes)/(sdiff + usdiff/1000000.0);
666 LastBytes = CurrentBytes;
667 ElapsedTime = NewTime.tv_sec - StartTime.tv_sec;
668 Time = NewTime;
669 }
670 }
671 /*}}}*/
672 // AcquireStatus::Start - Called when the download is started /*{{{*/
673 // ---------------------------------------------------------------------
674 /* We just reset the counters */
675 void pkgAcquireStatus::Start()
676 {
677 gettimeofday(&Time,0);
678 gettimeofday(&StartTime,0);
679 LastBytes = 0;
680 CurrentCPS = 0;
681 CurrentBytes = 0;
682 TotalBytes = 0;
683 FetchedBytes = 0;
684 ElapsedTime = 0;
685 }
686 /*}}}*/
687 // AcquireStatus::Stop - Finished downloading /*{{{*/
688 // ---------------------------------------------------------------------
689 /* This accurately computes the elapsed time and the total overall CPS. */
690 void pkgAcquireStatus::Stop()
691 {
692 // Compute the CPS and elapsed time
693 struct timeval NewTime;
694 gettimeofday(&NewTime,0);
695
696 // Compute the delta time with full accuracy
697 long usdiff = NewTime.tv_usec - StartTime.tv_usec;
698 long sdiff = NewTime.tv_sec - StartTime.tv_sec;
699
700 // Borrow
701 if (usdiff < 0)
702 {
703 usdiff += 1000000;
704 sdiff--;
705 }
706
707 // Compute the CPS value
708 if (sdiff == 0 && usdiff == 0)
709 CurrentCPS = 0;
710 else
711 CurrentCPS = FetchedBytes/(sdiff + usdiff/1000000.0);
712 LastBytes = CurrentBytes;
713 ElapsedTime = sdiff;
714 }
715 /*}}}*/
716 // AcquireStatus::Fetched - Called when a byte set has been fetched /*{{{*/
717 // ---------------------------------------------------------------------
718 /* This is used to get accurate final transfer rate reporting. */
719 void pkgAcquireStatus::Fetched(unsigned long Size,unsigned long Resume)
720 {
721 FetchedBytes += Size - Resume;
722 }
723 /*}}}*/