]>
git.saurik.com Git - apt.git/blob - apt-pkg/acquire.cc
1 // -*- mode: cpp; mode: fold -*-
3 // $Id: acquire.cc,v 1.18 1998/11/23 07:32:19 jgg Exp $
4 /* ######################################################################
6 Acquire - File Acquiration
8 The core element for the schedual system is the concept of a named
9 queue. Each queue is unique and each queue has a name derived from the
10 URI. The degree of paralization can be controled by how the queue
11 name is derived from the URI.
13 ##################################################################### */
15 // Include Files /*{{{*/
17 #pragma implementation "apt-pkg/acquire.h"
19 #include <apt-pkg/acquire.h>
20 #include <apt-pkg/acquire-item.h>
21 #include <apt-pkg/acquire-worker.h>
22 #include <apt-pkg/configuration.h>
23 #include <apt-pkg/error.h>
30 // Acquire::pkgAcquire - Constructor /*{{{*/
31 // ---------------------------------------------------------------------
32 /* We grab some runtime state from the configuration space */
33 pkgAcquire::pkgAcquire(pkgAcquireStatus
*Log
) : Log(Log
)
41 string Mode
= _config
->Find("Acquire::Queue-Mode","host");
42 if (strcasecmp(Mode
.c_str(),"host") == 0)
43 QueueMode
= QueueHost
;
44 if (strcasecmp(Mode
.c_str(),"access") == 0)
45 QueueMode
= QueueAccess
;
47 Debug
= _config
->FindB("Debug::pkgAcquire",false);
50 // Acquire::~pkgAcquire - Destructor /*{{{*/
51 // ---------------------------------------------------------------------
52 /* Free our memory, clean up the queues (destroy the workers) */
53 pkgAcquire::~pkgAcquire()
55 while (Items
.size() != 0)
60 MethodConfig
*Jnk
= Configs
;
61 Configs
= Configs
->Next
;
68 Queues
= Queues
->Next
;
73 // Acquire::Add - Add a new item /*{{{*/
74 // ---------------------------------------------------------------------
75 /* This puts an item on the acquire list. This list is mainly for tracking
77 void pkgAcquire::Add(Item
*Itm
)
82 // Acquire::Remove - Remove a item /*{{{*/
83 // ---------------------------------------------------------------------
84 /* Remove an item from the acquire list. This is usually not used.. */
85 void pkgAcquire::Remove(Item
*Itm
)
87 for (vector
<Item
*>::iterator I
= Items
.begin(); I
< Items
.end(); I
++)
94 // Acquire::Add - Add a worker /*{{{*/
95 // ---------------------------------------------------------------------
96 /* A list of workers is kept so that the select loop can direct their FD
98 void pkgAcquire::Add(Worker
*Work
)
100 Work
->NextAcquire
= Workers
;
104 // Acquire::Remove - Remove a worker /*{{{*/
105 // ---------------------------------------------------------------------
106 /* A worker has died. This can not be done while the select loop is running
107 as it would require that RunFds could handling a changing list state and
109 void pkgAcquire::Remove(Worker
*Work
)
114 Worker
**I
= &Workers
;
118 *I
= (*I
)->NextAcquire
;
120 I
= &(*I
)->NextAcquire
;
124 // Acquire::Enqueue - Queue an URI for fetching /*{{{*/
125 // ---------------------------------------------------------------------
126 /* This is the entry point for an item. An item calls this function when
127 it is construction which creates a queue (based on the current queue
128 mode) and puts the item in that queue. If the system is running then
129 the queue might be started. */
130 void pkgAcquire::Enqueue(ItemDesc
&Item
)
132 // Determine which queue to put the item in
133 const MethodConfig
*Config
;
134 string Name
= QueueName(Item
.URI
,Config
);
135 if (Name
.empty() == true)
138 // Find the queue structure
140 for (; I
!= 0 && I
->Name
!= Name
; I
= I
->Next
);
143 I
= new Queue(Name
,this);
151 // See if this is a local only URI
152 if (Config
->LocalOnly
== true && Item
.Owner
->Complete
== false)
153 Item
.Owner
->Local
= true;
154 Item
.Owner
->Status
= Item::StatIdle
;
156 // Queue it into the named queue
163 clog
<< "Fetching " << Item
.URI
<< endl
;
164 clog
<< " to " << Item
.Owner
->DestFile
<< endl
;
165 clog
<< " Queue is: " << Name
<< endl
;
169 // Acquire::Dequeue - Remove an item from all queues /*{{{*/
170 // ---------------------------------------------------------------------
171 /* This is called when an item is finished being fetched. It removes it
172 from all the queues */
173 void pkgAcquire::Dequeue(Item
*Itm
)
177 for (; I
!= 0; I
= I
->Next
)
178 Res
|= I
->Dequeue(Itm
);
181 clog
<< "Dequeuing " << Itm
->DestFile
<< endl
;
186 // Acquire::QueueName - Return the name of the queue for this URI /*{{{*/
187 // ---------------------------------------------------------------------
188 /* The string returned depends on the configuration settings and the
189 method parameters. Given something like http://foo.org/bar it can
190 return http://foo.org or http */
191 string
pkgAcquire::QueueName(string Uri
,MethodConfig
const *&Config
)
195 Config
= GetConfig(U
.Access
);
199 /* Single-Instance methods get exactly one queue per URI. This is
200 also used for the Access queue method */
201 if (Config
->SingleInstance
== true || QueueMode
== QueueAccess
)
204 return U
.Access
+ ':' + U
.Host
;
207 // Acquire::GetConfig - Fetch the configuration information /*{{{*/
208 // ---------------------------------------------------------------------
209 /* This locates the configuration structure for an access method. If
210 a config structure cannot be found a Worker will be created to
212 pkgAcquire::MethodConfig
*pkgAcquire::GetConfig(string Access
)
214 // Search for an existing config
216 for (Conf
= Configs
; Conf
!= 0; Conf
= Conf
->Next
)
217 if (Conf
->Access
== Access
)
220 // Create the new config class
221 Conf
= new MethodConfig
;
222 Conf
->Access
= Access
;
223 Conf
->Next
= Configs
;
226 // Create the worker to fetch the configuration
228 if (Work
.Start() == false)
234 // Acquire::SetFds - Deal with readable FDs /*{{{*/
235 // ---------------------------------------------------------------------
236 /* Collect FDs that have activity monitors into the fd sets */
237 void pkgAcquire::SetFds(int &Fd
,fd_set
*RSet
,fd_set
*WSet
)
239 for (Worker
*I
= Workers
; I
!= 0; I
= I
->NextAcquire
)
241 if (I
->InReady
== true && I
->InFd
>= 0)
245 FD_SET(I
->InFd
,RSet
);
247 if (I
->OutReady
== true && I
->OutFd
>= 0)
251 FD_SET(I
->OutFd
,WSet
);
256 // Acquire::RunFds - Deal with active FDs /*{{{*/
257 // ---------------------------------------------------------------------
258 /* Dispatch active FDs over to the proper workers. It is very important
259 that a worker never be erased while this is running! The queue class
260 should never erase a worker except during shutdown processing. */
261 void pkgAcquire::RunFds(fd_set
*RSet
,fd_set
*WSet
)
263 for (Worker
*I
= Workers
; I
!= 0; I
= I
->NextAcquire
)
265 if (I
->InFd
>= 0 && FD_ISSET(I
->InFd
,RSet
) != 0)
267 if (I
->OutFd
>= 0 && FD_ISSET(I
->OutFd
,WSet
) != 0)
272 // Acquire::Run - Run the fetch sequence /*{{{*/
273 // ---------------------------------------------------------------------
274 /* This runs the queues. It manages a select loop for all of the
275 Worker tasks. The workers interact with the queues and items to
276 manage the actual fetch. */
277 bool pkgAcquire::Run()
281 for (Queue
*I
= Queues
; I
!= 0; I
= I
->Next
)
287 // Run till all things have been acquired
298 SetFds(Highest
,&RFds
,&WFds
);
300 int Res
= select(Highest
+1,&RFds
,&WFds
,0,&tv
);
303 _error
->Errno("select","Select has failed");
308 if (_error
->PendingError() == true)
311 // Timeout, notify the log class
312 if (Res
== 0 || (Log
!= 0 && Log
->Update
== true))
315 for (Worker
*I
= Workers
; I
!= 0; I
= I
->NextAcquire
)
325 // Shut down the acquire bits
327 for (Queue
*I
= Queues
; I
!= 0; I
= I
->Next
)
330 return !_error
->PendingError();
333 // Acquire::Bump - Called when an item is dequeued /*{{{*/
334 // ---------------------------------------------------------------------
335 /* This routine bumps idle queues in hopes that they will be able to fetch
337 void pkgAcquire::Bump()
339 for (Queue
*I
= Queues
; I
!= 0; I
= I
->Next
)
343 // Acquire::WorkerStep - Step to the next worker /*{{{*/
344 // ---------------------------------------------------------------------
345 /* Not inlined to advoid including acquire-worker.h */
346 pkgAcquire::Worker
*pkgAcquire::WorkerStep(Worker
*I
)
348 return I
->NextAcquire
;
351 // Acquire::Clean - Cleans a directory /*{{{*/
352 // ---------------------------------------------------------------------
353 /* This is a bit simplistic, it looks at every file in the dir and sees
354 if it is part of the download set. */
355 bool pkgAcquire::Clean(string Dir
)
357 DIR *D
= opendir(Dir
.c_str());
359 return _error
->Errno("opendir","Unable to read %s",Dir
.c_str());
361 string StartDir
= SafeGetCWD();
362 if (chdir(Dir
.c_str()) != 0)
365 return _error
->Errno("chdir","Unable to change to ",Dir
.c_str());
368 for (struct dirent
*Dir
= readdir(D
); Dir
!= 0; Dir
= readdir(D
))
371 if (strcmp(Dir
->d_name
,"lock") == 0 ||
372 strcmp(Dir
->d_name
,"partial") == 0 ||
373 strcmp(Dir
->d_name
,".") == 0 ||
374 strcmp(Dir
->d_name
,"..") == 0)
377 // Look in the get list
378 vector
<Item
*>::iterator I
= Items
.begin();
379 for (; I
!= Items
.end(); I
++)
380 if (flNotDir((*I
)->DestFile
) == Dir
->d_name
)
383 // Nothing found, nuke it
384 if (I
== Items
.end())
388 chdir(StartDir
.c_str());
393 // Acquire::TotalNeeded - Number of bytes to fetch /*{{{*/
394 // ---------------------------------------------------------------------
395 /* This is the total number of bytes needed */
396 unsigned long pkgAcquire::TotalNeeded()
398 unsigned long Total
= 0;
399 for (pkgAcquire::Item
**I
= ItemsBegin(); I
!= ItemsEnd(); I
++)
400 Total
+= (*I
)->FileSize
;
404 // Acquire::FetchNeeded - Number of bytes needed to get /*{{{*/
405 // ---------------------------------------------------------------------
406 /* This is the number of bytes that is not local */
407 unsigned long pkgAcquire::FetchNeeded()
409 unsigned long Total
= 0;
410 for (pkgAcquire::Item
**I
= ItemsBegin(); I
!= ItemsEnd(); I
++)
411 if ((*I
)->Local
== false)
412 Total
+= (*I
)->FileSize
;
417 // Acquire::MethodConfig::MethodConfig - Constructor /*{{{*/
418 // ---------------------------------------------------------------------
420 pkgAcquire::MethodConfig::MethodConfig()
422 SingleInstance
= false;
431 // Queue::Queue - Constructor /*{{{*/
432 // ---------------------------------------------------------------------
434 pkgAcquire::Queue::Queue(string Name
,pkgAcquire
*Owner
) : Name(Name
),
442 // Queue::~Queue - Destructor /*{{{*/
443 // ---------------------------------------------------------------------
445 pkgAcquire::Queue::~Queue()
457 // Queue::Enqueue - Queue an item to the queue /*{{{*/
458 // ---------------------------------------------------------------------
460 void pkgAcquire::Queue::Enqueue(ItemDesc
&Item
)
463 for (; *I
!= 0; I
= &(*I
)->Next
);
466 QItem
*Itm
= new QItem
;
471 Item
.Owner
->QueueCounter
++;
472 if (Items
->Next
== 0)
476 // Queue::Dequeue - Remove an item from the queue /*{{{*/
477 // ---------------------------------------------------------------------
478 /* We return true if we hit something*/
479 bool pkgAcquire::Queue::Dequeue(Item
*Owner
)
486 if ((*I
)->Owner
== Owner
)
490 Owner
->QueueCounter
--;
501 // Queue::Startup - Start the worker processes /*{{{*/
502 // ---------------------------------------------------------------------
504 bool pkgAcquire::Queue::Startup()
509 pkgAcquire::MethodConfig
*Cnf
= Owner
->GetConfig(U
.Access
);
513 Workers
= new Worker(this,Cnf
,Owner
->Log
);
515 if (Workers
->Start() == false)
521 // Queue::Shutdown - Shutdown the worker processes /*{{{*/
522 // ---------------------------------------------------------------------
524 bool pkgAcquire::Queue::Shutdown()
526 // Delete all of the workers
529 pkgAcquire::Worker
*Jnk
= Workers
;
530 Workers
= Workers
->NextQueue
;
538 // Queue::Finditem - Find a URI in the item list /*{{{*/
539 // ---------------------------------------------------------------------
541 pkgAcquire::Queue::QItem
*pkgAcquire::Queue::FindItem(string URI
,pkgAcquire::Worker
*Owner
)
543 for (QItem
*I
= Items
; I
!= 0; I
= I
->Next
)
544 if (I
->URI
== URI
&& I
->Worker
== Owner
)
549 // Queue::ItemDone - Item has been completed /*{{{*/
550 // ---------------------------------------------------------------------
551 /* The worker signals this which causes the item to be removed from the
552 queue. If this is the last queue instance then it is removed from the
554 bool pkgAcquire::Queue::ItemDone(QItem
*Itm
)
556 if (Itm
->Owner
->QueueCounter
<= 1)
557 Owner
->Dequeue(Itm
->Owner
);
567 // Queue::Cycle - Queue new items into the method /*{{{*/
568 // ---------------------------------------------------------------------
569 /* This locates a new idle item and sends it to the worker */
570 bool pkgAcquire::Queue::Cycle()
572 if (Items
== 0 || Workers
== 0)
575 // Look for a queable item
577 for (; I
!= 0; I
= I
->Next
)
578 if (I
->Owner
->Status
== pkgAcquire::Item::StatIdle
)
581 // Nothing to do, queue is idle.
586 I
->Owner
->Status
= pkgAcquire::Item::StatFetching
;
587 return Workers
->QueueItem(I
);
590 // Queue::Bump - Fetch any pending objects if we are idle /*{{{*/
591 // ---------------------------------------------------------------------
593 void pkgAcquire::Queue::Bump()
598 // AcquireStatus::pkgAcquireStatus - Constructor /*{{{*/
599 // ---------------------------------------------------------------------
601 pkgAcquireStatus::pkgAcquireStatus()
606 // AcquireStatus::Pulse - Called periodically /*{{{*/
607 // ---------------------------------------------------------------------
608 /* This computes some internal state variables for the derived classes to
609 use. It generates the current downloaded bytes and total bytes to download
610 as well as the current CPS estimate. */
611 void pkgAcquireStatus::Pulse(pkgAcquire
*Owner
)
618 // Compute the total number of bytes to fetch
619 unsigned int Unknown
= 0;
620 unsigned int Count
= 0;
621 for (pkgAcquire::Item
**I
= Owner
->ItemsBegin(); I
!= Owner
->ItemsEnd();
625 if ((*I
)->Status
== pkgAcquire::Item::StatDone
)
628 // Totally ignore local items
629 if ((*I
)->Local
== true)
632 TotalBytes
+= (*I
)->FileSize
;
633 if ((*I
)->Complete
== true)
634 CurrentBytes
+= (*I
)->FileSize
;
635 if ((*I
)->FileSize
== 0 && (*I
)->Complete
== false)
639 // Compute the current completion
640 for (pkgAcquire::Worker
*I
= Owner
->WorkersBegin(); I
!= 0;
641 I
= Owner
->WorkerStep(I
))
642 if (I
->CurrentItem
!= 0 && I
->CurrentItem
->Owner
->Complete
== false)
643 CurrentBytes
+= I
->CurrentSize
;
645 // Normalize the figures and account for unknown size downloads
648 if (Unknown
== Count
)
649 TotalBytes
= Unknown
;
651 TotalBytes
+= TotalBytes
/(Count
- Unknown
)*Unknown
;
654 struct timeval NewTime
;
655 gettimeofday(&NewTime
,0);
656 if (NewTime
.tv_sec
- Time
.tv_sec
== 6 && NewTime
.tv_usec
> Time
.tv_usec
||
657 NewTime
.tv_sec
- Time
.tv_sec
> 6)
659 // Compute the delta time with full accuracy
660 long usdiff
= NewTime
.tv_usec
- Time
.tv_usec
;
661 long sdiff
= NewTime
.tv_sec
- Time
.tv_sec
;
670 // Compute the CPS value
671 if (sdiff
== 0 && usdiff
== 0)
674 CurrentCPS
= (CurrentBytes
- LastBytes
)/(sdiff
+ usdiff
/1000000.0);
675 LastBytes
= CurrentBytes
;
676 ElapsedTime
= NewTime
.tv_sec
- StartTime
.tv_sec
;
681 // AcquireStatus::Start - Called when the download is started /*{{{*/
682 // ---------------------------------------------------------------------
683 /* We just reset the counters */
684 void pkgAcquireStatus::Start()
686 gettimeofday(&Time
,0);
687 gettimeofday(&StartTime
,0);
698 // AcquireStatus::Stop - Finished downloading /*{{{*/
699 // ---------------------------------------------------------------------
700 /* This accurately computes the elapsed time and the total overall CPS. */
701 void pkgAcquireStatus::Stop()
703 // Compute the CPS and elapsed time
704 struct timeval NewTime
;
705 gettimeofday(&NewTime
,0);
707 // Compute the delta time with full accuracy
708 long usdiff
= NewTime
.tv_usec
- StartTime
.tv_usec
;
709 long sdiff
= NewTime
.tv_sec
- StartTime
.tv_sec
;
718 // Compute the CPS value
719 if (sdiff
== 0 && usdiff
== 0)
722 CurrentCPS
= FetchedBytes
/(sdiff
+ usdiff
/1000000.0);
723 LastBytes
= CurrentBytes
;
727 // AcquireStatus::Fetched - Called when a byte set has been fetched /*{{{*/
728 // ---------------------------------------------------------------------
729 /* This is used to get accurate final transfer rate reporting. */
730 void pkgAcquireStatus::Fetched(unsigned long Size
,unsigned long Resume
)
732 FetchedBytes
+= Size
- Resume
;