]>
git.saurik.com Git - apt.git/blob - apt-pkg/acquire.cc
1 // -*- mode: cpp; mode: fold -*-
3 // $Id: acquire.cc,v 1.14 1998/11/12 04:10:54 jgg Exp $
4 /* ######################################################################
6 Acquire - File Acquiration
8 The core element for the schedual system is the concept of a named
9 queue. Each queue is unique and each queue has a name derived from the
10 URI. The degree of paralization can be controled by how the queue
11 name is derived from the URI.
13 ##################################################################### */
15 // Include Files /*{{{*/
17 #pragma implementation "apt-pkg/acquire.h"
19 #include <apt-pkg/acquire.h>
20 #include <apt-pkg/acquire-item.h>
21 #include <apt-pkg/acquire-worker.h>
22 #include <apt-pkg/configuration.h>
23 #include <apt-pkg/error.h>
30 // Acquire::pkgAcquire - Constructor /*{{{*/
31 // ---------------------------------------------------------------------
32 /* We grab some runtime state from the configuration space */
33 pkgAcquire::pkgAcquire(pkgAcquireStatus
*Log
) : Log(Log
)
41 string Mode
= _config
->Find("Acquire::Queue-Mode","host");
42 if (strcasecmp(Mode
.c_str(),"host") == 0)
43 QueueMode
= QueueHost
;
44 if (strcasecmp(Mode
.c_str(),"access") == 0)
45 QueueMode
= QueueAccess
;
47 Debug
= _config
->FindB("Debug::pkgAcquire",false);
50 // Acquire::~pkgAcquire - Destructor /*{{{*/
51 // ---------------------------------------------------------------------
52 /* Free our memory, clean up the queues (destroy the workers) */
53 pkgAcquire::~pkgAcquire()
55 while (Items
.size() != 0)
60 MethodConfig
*Jnk
= Configs
;
61 Configs
= Configs
->Next
;
68 Queues
= Queues
->Next
;
73 // Acquire::Add - Add a new item /*{{{*/
74 // ---------------------------------------------------------------------
75 /* This puts an item on the acquire list. This list is mainly for tracking
77 void pkgAcquire::Add(Item
*Itm
)
82 // Acquire::Remove - Remove a item /*{{{*/
83 // ---------------------------------------------------------------------
84 /* Remove an item from the acquire list. This is usually not used.. */
85 void pkgAcquire::Remove(Item
*Itm
)
87 for (vector
<Item
*>::iterator I
= Items
.begin(); I
< Items
.end(); I
++)
94 // Acquire::Add - Add a worker /*{{{*/
95 // ---------------------------------------------------------------------
96 /* A list of workers is kept so that the select loop can direct their FD
98 void pkgAcquire::Add(Worker
*Work
)
100 Work
->NextAcquire
= Workers
;
104 // Acquire::Remove - Remove a worker /*{{{*/
105 // ---------------------------------------------------------------------
106 /* A worker has died. This can not be done while the select loop is running
107 as it would require that RunFds could handling a changing list state and
109 void pkgAcquire::Remove(Worker
*Work
)
114 Worker
**I
= &Workers
;
118 *I
= (*I
)->NextAcquire
;
120 I
= &(*I
)->NextAcquire
;
124 // Acquire::Enqueue - Queue an URI for fetching /*{{{*/
125 // ---------------------------------------------------------------------
126 /* This is the entry point for an item. An item calls this function when
127 it is construction which creates a queue (based on the current queue
128 mode) and puts the item in that queue. If the system is running then
129 the queue might be started. */
130 void pkgAcquire::Enqueue(ItemDesc
&Item
)
132 // Determine which queue to put the item in
133 string Name
= QueueName(Item
.URI
);
134 if (Name
.empty() == true)
137 // Find the queue structure
139 for (; I
!= 0 && I
->Name
!= Name
; I
= I
->Next
);
142 I
= new Queue(Name
,this);
150 Item
.Owner
->Status
= Item::StatIdle
;
152 // Queue it into the named queue
159 clog
<< "Fetching " << Item
.URI
<< endl
;
160 clog
<< " to " << Item
.Owner
->DestFile
<< endl
;
161 clog
<< " Queue is: " << QueueName(Item
.URI
) << endl
;
165 // Acquire::Dequeue - Remove an item from all queues /*{{{*/
166 // ---------------------------------------------------------------------
167 /* This is called when an item is finished being fetched. It removes it
168 from all the queues */
169 void pkgAcquire::Dequeue(Item
*Itm
)
173 for (; I
!= 0; I
= I
->Next
)
174 Res
|= I
->Dequeue(Itm
);
177 clog
<< "Dequeuing " << Itm
->DestFile
<< endl
;
182 // Acquire::QueueName - Return the name of the queue for this URI /*{{{*/
183 // ---------------------------------------------------------------------
184 /* The string returned depends on the configuration settings and the
185 method parameters. Given something like http://foo.org/bar it can
186 return http://foo.org or http */
187 string
pkgAcquire::QueueName(string Uri
)
191 const MethodConfig
*Config
= GetConfig(U
.Access
);
195 /* Single-Instance methods get exactly one queue per URI. This is
196 also used for the Access queue method */
197 if (Config
->SingleInstance
== true || QueueMode
== QueueAccess
)
200 return U
.Access
+ ':' + U
.Host
;
203 // Acquire::GetConfig - Fetch the configuration information /*{{{*/
204 // ---------------------------------------------------------------------
205 /* This locates the configuration structure for an access method. If
206 a config structure cannot be found a Worker will be created to
208 pkgAcquire::MethodConfig
*pkgAcquire::GetConfig(string Access
)
210 // Search for an existing config
212 for (Conf
= Configs
; Conf
!= 0; Conf
= Conf
->Next
)
213 if (Conf
->Access
== Access
)
216 // Create the new config class
217 Conf
= new MethodConfig
;
218 Conf
->Access
= Access
;
219 Conf
->Next
= Configs
;
222 // Create the worker to fetch the configuration
224 if (Work
.Start() == false)
230 // Acquire::SetFds - Deal with readable FDs /*{{{*/
231 // ---------------------------------------------------------------------
232 /* Collect FDs that have activity monitors into the fd sets */
233 void pkgAcquire::SetFds(int &Fd
,fd_set
*RSet
,fd_set
*WSet
)
235 for (Worker
*I
= Workers
; I
!= 0; I
= I
->NextAcquire
)
237 if (I
->InReady
== true && I
->InFd
>= 0)
241 FD_SET(I
->InFd
,RSet
);
243 if (I
->OutReady
== true && I
->OutFd
>= 0)
247 FD_SET(I
->OutFd
,WSet
);
252 // Acquire::RunFds - Deal with active FDs /*{{{*/
253 // ---------------------------------------------------------------------
254 /* Dispatch active FDs over to the proper workers. It is very important
255 that a worker never be erased while this is running! The queue class
256 should never erase a worker except during shutdown processing. */
257 void pkgAcquire::RunFds(fd_set
*RSet
,fd_set
*WSet
)
259 for (Worker
*I
= Workers
; I
!= 0; I
= I
->NextAcquire
)
261 if (I
->InFd
>= 0 && FD_ISSET(I
->InFd
,RSet
) != 0)
263 if (I
->OutFd
>= 0 && FD_ISSET(I
->OutFd
,WSet
) != 0)
268 // Acquire::Run - Run the fetch sequence /*{{{*/
269 // ---------------------------------------------------------------------
270 /* This runs the queues. It manages a select loop for all of the
271 Worker tasks. The workers interact with the queues and items to
272 manage the actual fetch. */
273 bool pkgAcquire::Run()
277 for (Queue
*I
= Queues
; I
!= 0; I
= I
->Next
)
283 // Run till all things have been acquired
294 SetFds(Highest
,&RFds
,&WFds
);
296 int Res
= select(Highest
+1,&RFds
,&WFds
,0,&tv
);
299 _error
->Errno("select","Select has failed");
304 if (_error
->PendingError() == true)
307 // Timeout, notify the log class
308 if (Res
== 0 || (Log
!= 0 && Log
->Update
== true))
311 for (Worker
*I
= Workers
; I
!= 0; I
= I
->NextAcquire
)
321 // Shut down the acquire bits
323 for (Queue
*I
= Queues
; I
!= 0; I
= I
->Next
)
326 return !_error
->PendingError();
329 // Acquire::Bump - Called when an item is dequeued /*{{{*/
330 // ---------------------------------------------------------------------
331 /* This routine bumps idle queues in hopes that they will be able to fetch
333 void pkgAcquire::Bump()
335 for (Queue
*I
= Queues
; I
!= 0; I
= I
->Next
)
339 // Acquire::WorkerStep - Step to the next worker /*{{{*/
340 // ---------------------------------------------------------------------
341 /* Not inlined to advoid including acquire-worker.h */
342 pkgAcquire::Worker
*pkgAcquire::WorkerStep(Worker
*I
)
344 return I
->NextAcquire
;
347 // pkgAcquire::Clean - Cleans a directory /*{{{*/
348 // ---------------------------------------------------------------------
349 /* This is a bit simplistic, it looks at every file in the dir and sees
350 if it is part of the download set. */
351 bool pkgAcquire::Clean(string Dir
)
353 DIR *D
= opendir(Dir
.c_str());
355 return _error
->Errno("opendir","Unable to read %s",Dir
.c_str());
357 string StartDir
= SafeGetCWD();
358 if (chdir(Dir
.c_str()) != 0)
361 return _error
->Errno("chdir","Unable to change to ",Dir
.c_str());
364 for (struct dirent
*Dir
= readdir(D
); Dir
!= 0; Dir
= readdir(D
))
367 if (strcmp(Dir
->d_name
,"lock") == 0 ||
368 strcmp(Dir
->d_name
,"partial") == 0 ||
369 strcmp(Dir
->d_name
,".") == 0 ||
370 strcmp(Dir
->d_name
,"..") == 0)
373 // Look in the get list
374 vector
<Item
*>::iterator I
= Items
.begin();
375 for (; I
!= Items
.end(); I
++)
376 if (flNotDir((*I
)->DestFile
) == Dir
->d_name
)
379 // Nothing found, nuke it
380 if (I
== Items
.end())
384 chdir(StartDir
.c_str());
389 // Acquire::MethodConfig::MethodConfig - Constructor /*{{{*/
390 // ---------------------------------------------------------------------
392 pkgAcquire::MethodConfig::MethodConfig()
394 SingleInstance
= false;
402 // Queue::Queue - Constructor /*{{{*/
403 // ---------------------------------------------------------------------
405 pkgAcquire::Queue::Queue(string Name
,pkgAcquire
*Owner
) : Name(Name
),
413 // Queue::~Queue - Destructor /*{{{*/
414 // ---------------------------------------------------------------------
416 pkgAcquire::Queue::~Queue()
428 // Queue::Enqueue - Queue an item to the queue /*{{{*/
429 // ---------------------------------------------------------------------
431 void pkgAcquire::Queue::Enqueue(ItemDesc
&Item
)
434 QItem
*I
= new QItem
;
439 Item
.Owner
->QueueCounter
++;
440 if (Items
->Next
== 0)
444 // Queue::Dequeue - Remove an item from the queue /*{{{*/
445 // ---------------------------------------------------------------------
446 /* We return true if we hit something*/
447 bool pkgAcquire::Queue::Dequeue(Item
*Owner
)
454 if ((*I
)->Owner
== Owner
)
458 Owner
->QueueCounter
--;
469 // Queue::Startup - Start the worker processes /*{{{*/
470 // ---------------------------------------------------------------------
472 bool pkgAcquire::Queue::Startup()
477 pkgAcquire::MethodConfig
*Cnf
= Owner
->GetConfig(U
.Access
);
481 Workers
= new Worker(this,Cnf
,Owner
->Log
);
483 if (Workers
->Start() == false)
489 // Queue::Shutdown - Shutdown the worker processes /*{{{*/
490 // ---------------------------------------------------------------------
492 bool pkgAcquire::Queue::Shutdown()
494 // Delete all of the workers
497 pkgAcquire::Worker
*Jnk
= Workers
;
498 Workers
= Workers
->NextQueue
;
506 // Queue::Finditem - Find a URI in the item list /*{{{*/
507 // ---------------------------------------------------------------------
509 pkgAcquire::Queue::QItem
*pkgAcquire::Queue::FindItem(string URI
,pkgAcquire::Worker
*Owner
)
511 for (QItem
*I
= Items
; I
!= 0; I
= I
->Next
)
512 if (I
->URI
== URI
&& I
->Worker
== Owner
)
517 // Queue::ItemDone - Item has been completed /*{{{*/
518 // ---------------------------------------------------------------------
519 /* The worker signals this which causes the item to be removed from the
520 queue. If this is the last queue instance then it is removed from the
522 bool pkgAcquire::Queue::ItemDone(QItem
*Itm
)
524 if (Itm
->Owner
->QueueCounter
<= 1)
525 Owner
->Dequeue(Itm
->Owner
);
535 // Queue::Cycle - Queue new items into the method /*{{{*/
536 // ---------------------------------------------------------------------
537 /* This locates a new idle item and sends it to the worker */
538 bool pkgAcquire::Queue::Cycle()
540 if (Items
== 0 || Workers
== 0)
543 // Look for a queable item
545 for (; I
!= 0; I
= I
->Next
)
546 if (I
->Owner
->Status
== pkgAcquire::Item::StatIdle
)
549 // Nothing to do, queue is idle.
554 I
->Owner
->Status
= pkgAcquire::Item::StatFetching
;
555 return Workers
->QueueItem(I
);
558 // Queue::Bump - Fetch any pending objects if we are idle /*{{{*/
559 // ---------------------------------------------------------------------
561 void pkgAcquire::Queue::Bump()
566 // AcquireStatus::pkgAcquireStatus - Constructor /*{{{*/
567 // ---------------------------------------------------------------------
569 pkgAcquireStatus::pkgAcquireStatus()
574 // AcquireStatus::Pulse - Called periodically /*{{{*/
575 // ---------------------------------------------------------------------
576 /* This computes some internal state variables for the derived classes to
577 use. It generates the current downloaded bytes and total bytes to download
578 as well as the current CPS estimate. */
579 void pkgAcquireStatus::Pulse(pkgAcquire
*Owner
)
584 // Compute the total number of bytes to fetch
585 unsigned int Unknown
= 0;
586 unsigned int Count
= 0;
587 for (pkgAcquire::Item
**I
= Owner
->ItemsBegin(); I
!= Owner
->ItemsEnd();
590 TotalBytes
+= (*I
)->FileSize
;
591 if ((*I
)->Complete
== true)
592 CurrentBytes
+= (*I
)->FileSize
;
593 if ((*I
)->FileSize
== 0 && (*I
)->Complete
== false)
597 // Compute the current completion
598 for (pkgAcquire::Worker
*I
= Owner
->WorkersBegin(); I
!= 0;
599 I
= Owner
->WorkerStep(I
))
600 if (I
->CurrentItem
!= 0 && I
->CurrentItem
->Owner
->Complete
== false)
601 CurrentBytes
+= I
->CurrentSize
;
603 // Normalize the figures and account for unknown size downloads
606 if (Unknown
== Count
)
607 TotalBytes
= Unknown
;
609 TotalBytes
+= TotalBytes
/(Count
- Unknown
)*Unknown
;
612 struct timeval NewTime
;
613 gettimeofday(&NewTime
,0);
614 if (NewTime
.tv_sec
- Time
.tv_sec
== 6 && NewTime
.tv_usec
> Time
.tv_usec
||
615 NewTime
.tv_sec
- Time
.tv_sec
> 6)
617 // Compute the delta time with full accuracy
618 long usdiff
= NewTime
.tv_usec
- Time
.tv_usec
;
619 long sdiff
= NewTime
.tv_sec
- Time
.tv_sec
;
628 // Compute the CPS value
629 CurrentCPS
= (CurrentBytes
- LastBytes
)/(sdiff
+ usdiff
/1000000.0);
630 LastBytes
= CurrentBytes
;
631 ElapsedTime
= NewTime
.tv_sec
- StartTime
.tv_sec
;
636 // AcquireStatus::Start - Called when the download is started /*{{{*/
637 // ---------------------------------------------------------------------
638 /* We just reset the counters */
639 void pkgAcquireStatus::Start()
641 gettimeofday(&Time
,0);
642 gettimeofday(&StartTime
,0);
651 // pkgAcquireStatus::Stop - Finished downloading /*{{{*/
652 // ---------------------------------------------------------------------
653 /* This accurately computes the elapsed time and the total overall CPS. */
654 void pkgAcquireStatus::Stop()
656 // Compute the CPS and elapsed time
657 struct timeval NewTime
;
658 gettimeofday(&NewTime
,0);
660 // Compute the delta time with full accuracy
661 long usdiff
= NewTime
.tv_usec
- StartTime
.tv_usec
;
662 long sdiff
= NewTime
.tv_sec
- StartTime
.tv_sec
;
671 // Compute the CPS value
672 CurrentCPS
= FetchedBytes
/(sdiff
+ usdiff
/1000000.0);
673 LastBytes
= CurrentBytes
;
677 // AcquireStatus::Fetched - Called when a byte set has been fetched /*{{{*/
678 // ---------------------------------------------------------------------
679 /* This is used to get accurate final transfer rate reporting. */
680 void pkgAcquireStatus::Fetched(unsigned long Size
,unsigned long Resume
)
682 FetchedBytes
+= Size
- Resume
;