]>
git.saurik.com Git - apt.git/blob - apt-pkg/acquire.cc
1 // -*- mode: cpp; mode: fold -*-
3 // $Id: acquire.cc,v 1.15 1998/11/13 07:08:54 jgg Exp $
4 /* ######################################################################
6 Acquire - File Acquiration
8 The core element for the schedual system is the concept of a named
9 queue. Each queue is unique and each queue has a name derived from the
10 URI. The degree of paralization can be controled by how the queue
11 name is derived from the URI.
13 ##################################################################### */
15 // Include Files /*{{{*/
17 #pragma implementation "apt-pkg/acquire.h"
19 #include <apt-pkg/acquire.h>
20 #include <apt-pkg/acquire-item.h>
21 #include <apt-pkg/acquire-worker.h>
22 #include <apt-pkg/configuration.h>
23 #include <apt-pkg/error.h>
30 // Acquire::pkgAcquire - Constructor /*{{{*/
31 // ---------------------------------------------------------------------
32 /* We grab some runtime state from the configuration space */
33 pkgAcquire::pkgAcquire(pkgAcquireStatus
*Log
) : Log(Log
)
41 string Mode
= _config
->Find("Acquire::Queue-Mode","host");
42 if (strcasecmp(Mode
.c_str(),"host") == 0)
43 QueueMode
= QueueHost
;
44 if (strcasecmp(Mode
.c_str(),"access") == 0)
45 QueueMode
= QueueAccess
;
47 Debug
= _config
->FindB("Debug::pkgAcquire",false);
50 // Acquire::~pkgAcquire - Destructor /*{{{*/
51 // ---------------------------------------------------------------------
52 /* Free our memory, clean up the queues (destroy the workers) */
53 pkgAcquire::~pkgAcquire()
55 while (Items
.size() != 0)
60 MethodConfig
*Jnk
= Configs
;
61 Configs
= Configs
->Next
;
68 Queues
= Queues
->Next
;
73 // Acquire::Add - Add a new item /*{{{*/
74 // ---------------------------------------------------------------------
75 /* This puts an item on the acquire list. This list is mainly for tracking
77 void pkgAcquire::Add(Item
*Itm
)
82 // Acquire::Remove - Remove a item /*{{{*/
83 // ---------------------------------------------------------------------
84 /* Remove an item from the acquire list. This is usually not used.. */
85 void pkgAcquire::Remove(Item
*Itm
)
87 for (vector
<Item
*>::iterator I
= Items
.begin(); I
< Items
.end(); I
++)
94 // Acquire::Add - Add a worker /*{{{*/
95 // ---------------------------------------------------------------------
96 /* A list of workers is kept so that the select loop can direct their FD
98 void pkgAcquire::Add(Worker
*Work
)
100 Work
->NextAcquire
= Workers
;
104 // Acquire::Remove - Remove a worker /*{{{*/
105 // ---------------------------------------------------------------------
106 /* A worker has died. This can not be done while the select loop is running
107 as it would require that RunFds could handling a changing list state and
109 void pkgAcquire::Remove(Worker
*Work
)
114 Worker
**I
= &Workers
;
118 *I
= (*I
)->NextAcquire
;
120 I
= &(*I
)->NextAcquire
;
124 // Acquire::Enqueue - Queue an URI for fetching /*{{{*/
125 // ---------------------------------------------------------------------
126 /* This is the entry point for an item. An item calls this function when
127 it is construction which creates a queue (based on the current queue
128 mode) and puts the item in that queue. If the system is running then
129 the queue might be started. */
130 void pkgAcquire::Enqueue(ItemDesc
&Item
)
132 // Determine which queue to put the item in
133 string Name
= QueueName(Item
.URI
);
134 if (Name
.empty() == true)
137 // Find the queue structure
139 for (; I
!= 0 && I
->Name
!= Name
; I
= I
->Next
);
142 I
= new Queue(Name
,this);
150 Item
.Owner
->Status
= Item::StatIdle
;
152 // Queue it into the named queue
159 clog
<< "Fetching " << Item
.URI
<< endl
;
160 clog
<< " to " << Item
.Owner
->DestFile
<< endl
;
161 clog
<< " Queue is: " << QueueName(Item
.URI
) << endl
;
165 // Acquire::Dequeue - Remove an item from all queues /*{{{*/
166 // ---------------------------------------------------------------------
167 /* This is called when an item is finished being fetched. It removes it
168 from all the queues */
169 void pkgAcquire::Dequeue(Item
*Itm
)
173 for (; I
!= 0; I
= I
->Next
)
174 Res
|= I
->Dequeue(Itm
);
177 clog
<< "Dequeuing " << Itm
->DestFile
<< endl
;
182 // Acquire::QueueName - Return the name of the queue for this URI /*{{{*/
183 // ---------------------------------------------------------------------
184 /* The string returned depends on the configuration settings and the
185 method parameters. Given something like http://foo.org/bar it can
186 return http://foo.org or http */
187 string
pkgAcquire::QueueName(string Uri
)
191 const MethodConfig
*Config
= GetConfig(U
.Access
);
195 /* Single-Instance methods get exactly one queue per URI. This is
196 also used for the Access queue method */
197 if (Config
->SingleInstance
== true || QueueMode
== QueueAccess
)
200 return U
.Access
+ ':' + U
.Host
;
203 // Acquire::GetConfig - Fetch the configuration information /*{{{*/
204 // ---------------------------------------------------------------------
205 /* This locates the configuration structure for an access method. If
206 a config structure cannot be found a Worker will be created to
208 pkgAcquire::MethodConfig
*pkgAcquire::GetConfig(string Access
)
210 // Search for an existing config
212 for (Conf
= Configs
; Conf
!= 0; Conf
= Conf
->Next
)
213 if (Conf
->Access
== Access
)
216 // Create the new config class
217 Conf
= new MethodConfig
;
218 Conf
->Access
= Access
;
219 Conf
->Next
= Configs
;
222 // Create the worker to fetch the configuration
224 if (Work
.Start() == false)
230 // Acquire::SetFds - Deal with readable FDs /*{{{*/
231 // ---------------------------------------------------------------------
232 /* Collect FDs that have activity monitors into the fd sets */
233 void pkgAcquire::SetFds(int &Fd
,fd_set
*RSet
,fd_set
*WSet
)
235 for (Worker
*I
= Workers
; I
!= 0; I
= I
->NextAcquire
)
237 if (I
->InReady
== true && I
->InFd
>= 0)
241 FD_SET(I
->InFd
,RSet
);
243 if (I
->OutReady
== true && I
->OutFd
>= 0)
247 FD_SET(I
->OutFd
,WSet
);
252 // Acquire::RunFds - Deal with active FDs /*{{{*/
253 // ---------------------------------------------------------------------
254 /* Dispatch active FDs over to the proper workers. It is very important
255 that a worker never be erased while this is running! The queue class
256 should never erase a worker except during shutdown processing. */
257 void pkgAcquire::RunFds(fd_set
*RSet
,fd_set
*WSet
)
259 for (Worker
*I
= Workers
; I
!= 0; I
= I
->NextAcquire
)
261 if (I
->InFd
>= 0 && FD_ISSET(I
->InFd
,RSet
) != 0)
263 if (I
->OutFd
>= 0 && FD_ISSET(I
->OutFd
,WSet
) != 0)
268 // Acquire::Run - Run the fetch sequence /*{{{*/
269 // ---------------------------------------------------------------------
270 /* This runs the queues. It manages a select loop for all of the
271 Worker tasks. The workers interact with the queues and items to
272 manage the actual fetch. */
273 bool pkgAcquire::Run()
277 for (Queue
*I
= Queues
; I
!= 0; I
= I
->Next
)
283 // Run till all things have been acquired
294 SetFds(Highest
,&RFds
,&WFds
);
296 int Res
= select(Highest
+1,&RFds
,&WFds
,0,&tv
);
299 _error
->Errno("select","Select has failed");
304 if (_error
->PendingError() == true)
307 // Timeout, notify the log class
308 if (Res
== 0 || (Log
!= 0 && Log
->Update
== true))
311 for (Worker
*I
= Workers
; I
!= 0; I
= I
->NextAcquire
)
321 // Shut down the acquire bits
323 for (Queue
*I
= Queues
; I
!= 0; I
= I
->Next
)
326 return !_error
->PendingError();
329 // Acquire::Bump - Called when an item is dequeued /*{{{*/
330 // ---------------------------------------------------------------------
331 /* This routine bumps idle queues in hopes that they will be able to fetch
333 void pkgAcquire::Bump()
335 for (Queue
*I
= Queues
; I
!= 0; I
= I
->Next
)
339 // Acquire::WorkerStep - Step to the next worker /*{{{*/
340 // ---------------------------------------------------------------------
341 /* Not inlined to advoid including acquire-worker.h */
342 pkgAcquire::Worker
*pkgAcquire::WorkerStep(Worker
*I
)
344 return I
->NextAcquire
;
347 // Acquire::Clean - Cleans a directory /*{{{*/
348 // ---------------------------------------------------------------------
349 /* This is a bit simplistic, it looks at every file in the dir and sees
350 if it is part of the download set. */
351 bool pkgAcquire::Clean(string Dir
)
353 DIR *D
= opendir(Dir
.c_str());
355 return _error
->Errno("opendir","Unable to read %s",Dir
.c_str());
357 string StartDir
= SafeGetCWD();
358 if (chdir(Dir
.c_str()) != 0)
361 return _error
->Errno("chdir","Unable to change to ",Dir
.c_str());
364 for (struct dirent
*Dir
= readdir(D
); Dir
!= 0; Dir
= readdir(D
))
367 if (strcmp(Dir
->d_name
,"lock") == 0 ||
368 strcmp(Dir
->d_name
,"partial") == 0 ||
369 strcmp(Dir
->d_name
,".") == 0 ||
370 strcmp(Dir
->d_name
,"..") == 0)
373 // Look in the get list
374 vector
<Item
*>::iterator I
= Items
.begin();
375 for (; I
!= Items
.end(); I
++)
376 if (flNotDir((*I
)->DestFile
) == Dir
->d_name
)
379 // Nothing found, nuke it
380 if (I
== Items
.end())
384 chdir(StartDir
.c_str());
389 // Acquire::MethodConfig::MethodConfig - Constructor /*{{{*/
390 // ---------------------------------------------------------------------
392 pkgAcquire::MethodConfig::MethodConfig()
394 SingleInstance
= false;
401 // Acquire::TotalNeeded - Number of bytes to fetch /*{{{*/
402 // ---------------------------------------------------------------------
403 /* This is the total number of bytes needed */
404 unsigned long pkgAcquire::TotalNeeded()
406 unsigned long Total
= 0;
407 for (pkgAcquire::Item
**I
= ItemsBegin(); I
!= ItemsEnd(); I
++)
408 Total
+= (*I
)->FileSize
;
412 // Acquire::FetchNeeded - Number of bytes needed to get /*{{{*/
413 // ---------------------------------------------------------------------
414 /* This is the number of bytes that is not local */
415 unsigned long pkgAcquire::FetchNeeded()
417 unsigned long Total
= 0;
418 for (pkgAcquire::Item
**I
= ItemsBegin(); I
!= ItemsEnd(); I
++)
419 if ((*I
)->Local
== false)
420 Total
+= (*I
)->FileSize
;
425 // Queue::Queue - Constructor /*{{{*/
426 // ---------------------------------------------------------------------
428 pkgAcquire::Queue::Queue(string Name
,pkgAcquire
*Owner
) : Name(Name
),
436 // Queue::~Queue - Destructor /*{{{*/
437 // ---------------------------------------------------------------------
439 pkgAcquire::Queue::~Queue()
451 // Queue::Enqueue - Queue an item to the queue /*{{{*/
452 // ---------------------------------------------------------------------
454 void pkgAcquire::Queue::Enqueue(ItemDesc
&Item
)
457 QItem
*I
= new QItem
;
462 Item
.Owner
->QueueCounter
++;
463 if (Items
->Next
== 0)
467 // Queue::Dequeue - Remove an item from the queue /*{{{*/
468 // ---------------------------------------------------------------------
469 /* We return true if we hit something*/
470 bool pkgAcquire::Queue::Dequeue(Item
*Owner
)
477 if ((*I
)->Owner
== Owner
)
481 Owner
->QueueCounter
--;
492 // Queue::Startup - Start the worker processes /*{{{*/
493 // ---------------------------------------------------------------------
495 bool pkgAcquire::Queue::Startup()
500 pkgAcquire::MethodConfig
*Cnf
= Owner
->GetConfig(U
.Access
);
504 Workers
= new Worker(this,Cnf
,Owner
->Log
);
506 if (Workers
->Start() == false)
512 // Queue::Shutdown - Shutdown the worker processes /*{{{*/
513 // ---------------------------------------------------------------------
515 bool pkgAcquire::Queue::Shutdown()
517 // Delete all of the workers
520 pkgAcquire::Worker
*Jnk
= Workers
;
521 Workers
= Workers
->NextQueue
;
529 // Queue::Finditem - Find a URI in the item list /*{{{*/
530 // ---------------------------------------------------------------------
532 pkgAcquire::Queue::QItem
*pkgAcquire::Queue::FindItem(string URI
,pkgAcquire::Worker
*Owner
)
534 for (QItem
*I
= Items
; I
!= 0; I
= I
->Next
)
535 if (I
->URI
== URI
&& I
->Worker
== Owner
)
540 // Queue::ItemDone - Item has been completed /*{{{*/
541 // ---------------------------------------------------------------------
542 /* The worker signals this which causes the item to be removed from the
543 queue. If this is the last queue instance then it is removed from the
545 bool pkgAcquire::Queue::ItemDone(QItem
*Itm
)
547 if (Itm
->Owner
->QueueCounter
<= 1)
548 Owner
->Dequeue(Itm
->Owner
);
558 // Queue::Cycle - Queue new items into the method /*{{{*/
559 // ---------------------------------------------------------------------
560 /* This locates a new idle item and sends it to the worker */
561 bool pkgAcquire::Queue::Cycle()
563 if (Items
== 0 || Workers
== 0)
566 // Look for a queable item
568 for (; I
!= 0; I
= I
->Next
)
569 if (I
->Owner
->Status
== pkgAcquire::Item::StatIdle
)
572 // Nothing to do, queue is idle.
577 I
->Owner
->Status
= pkgAcquire::Item::StatFetching
;
578 return Workers
->QueueItem(I
);
581 // Queue::Bump - Fetch any pending objects if we are idle /*{{{*/
582 // ---------------------------------------------------------------------
584 void pkgAcquire::Queue::Bump()
589 // AcquireStatus::pkgAcquireStatus - Constructor /*{{{*/
590 // ---------------------------------------------------------------------
592 pkgAcquireStatus::pkgAcquireStatus()
597 // AcquireStatus::Pulse - Called periodically /*{{{*/
598 // ---------------------------------------------------------------------
599 /* This computes some internal state variables for the derived classes to
600 use. It generates the current downloaded bytes and total bytes to download
601 as well as the current CPS estimate. */
602 void pkgAcquireStatus::Pulse(pkgAcquire
*Owner
)
607 // Compute the total number of bytes to fetch
608 unsigned int Unknown
= 0;
609 unsigned int Count
= 0;
610 for (pkgAcquire::Item
**I
= Owner
->ItemsBegin(); I
!= Owner
->ItemsEnd();
613 // Totally ignore local items
614 if ((*I
)->Local
== true)
617 TotalBytes
+= (*I
)->FileSize
;
618 if ((*I
)->Complete
== true)
619 CurrentBytes
+= (*I
)->FileSize
;
620 if ((*I
)->FileSize
== 0 && (*I
)->Complete
== false)
624 // Compute the current completion
625 for (pkgAcquire::Worker
*I
= Owner
->WorkersBegin(); I
!= 0;
626 I
= Owner
->WorkerStep(I
))
627 if (I
->CurrentItem
!= 0 && I
->CurrentItem
->Owner
->Complete
== false)
628 CurrentBytes
+= I
->CurrentSize
;
630 // Normalize the figures and account for unknown size downloads
633 if (Unknown
== Count
)
634 TotalBytes
= Unknown
;
636 TotalBytes
+= TotalBytes
/(Count
- Unknown
)*Unknown
;
639 struct timeval NewTime
;
640 gettimeofday(&NewTime
,0);
641 if (NewTime
.tv_sec
- Time
.tv_sec
== 6 && NewTime
.tv_usec
> Time
.tv_usec
||
642 NewTime
.tv_sec
- Time
.tv_sec
> 6)
644 // Compute the delta time with full accuracy
645 long usdiff
= NewTime
.tv_usec
- Time
.tv_usec
;
646 long sdiff
= NewTime
.tv_sec
- Time
.tv_sec
;
655 // Compute the CPS value
656 CurrentCPS
= (CurrentBytes
- LastBytes
)/(sdiff
+ usdiff
/1000000.0);
657 LastBytes
= CurrentBytes
;
658 ElapsedTime
= NewTime
.tv_sec
- StartTime
.tv_sec
;
663 // AcquireStatus::Start - Called when the download is started /*{{{*/
664 // ---------------------------------------------------------------------
665 /* We just reset the counters */
666 void pkgAcquireStatus::Start()
668 gettimeofday(&Time
,0);
669 gettimeofday(&StartTime
,0);
678 // AcquireStatus::Stop - Finished downloading /*{{{*/
679 // ---------------------------------------------------------------------
680 /* This accurately computes the elapsed time and the total overall CPS. */
681 void pkgAcquireStatus::Stop()
683 // Compute the CPS and elapsed time
684 struct timeval NewTime
;
685 gettimeofday(&NewTime
,0);
687 // Compute the delta time with full accuracy
688 long usdiff
= NewTime
.tv_usec
- StartTime
.tv_usec
;
689 long sdiff
= NewTime
.tv_sec
- StartTime
.tv_sec
;
698 // Compute the CPS value
699 CurrentCPS
= FetchedBytes
/(sdiff
+ usdiff
/1000000.0);
700 LastBytes
= CurrentBytes
;
704 // AcquireStatus::Fetched - Called when a byte set has been fetched /*{{{*/
705 // ---------------------------------------------------------------------
706 /* This is used to get accurate final transfer rate reporting. */
707 void pkgAcquireStatus::Fetched(unsigned long Size
,unsigned long Resume
)
709 FetchedBytes
+= Size
- Resume
;