]>
git.saurik.com Git - apt.git/blob - apt-pkg/acquire.cc
1 // -*- mode: cpp; mode: fold -*-
3 // $Id: acquire.cc,v 1.20 1998/12/05 04:19:03 jgg Exp $
4 /* ######################################################################
6 Acquire - File Acquiration
8 The core element for the schedual system is the concept of a named
9 queue. Each queue is unique and each queue has a name derived from the
10 URI. The degree of paralization can be controled by how the queue
11 name is derived from the URI.
13 ##################################################################### */
15 // Include Files /*{{{*/
17 #pragma implementation "apt-pkg/acquire.h"
19 #include <apt-pkg/acquire.h>
20 #include <apt-pkg/acquire-item.h>
21 #include <apt-pkg/acquire-worker.h>
22 #include <apt-pkg/configuration.h>
23 #include <apt-pkg/error.h>
30 // Acquire::pkgAcquire - Constructor /*{{{*/
31 // ---------------------------------------------------------------------
32 /* We grab some runtime state from the configuration space */
33 pkgAcquire::pkgAcquire(pkgAcquireStatus
*Log
) : Log(Log
)
41 string Mode
= _config
->Find("Acquire::Queue-Mode","host");
42 if (strcasecmp(Mode
.c_str(),"host") == 0)
43 QueueMode
= QueueHost
;
44 if (strcasecmp(Mode
.c_str(),"access") == 0)
45 QueueMode
= QueueAccess
;
47 Debug
= _config
->FindB("Debug::pkgAcquire",false);
50 // Acquire::~pkgAcquire - Destructor /*{{{*/
51 // ---------------------------------------------------------------------
52 /* Free our memory, clean up the queues (destroy the workers) */
53 pkgAcquire::~pkgAcquire()
55 while (Items
.size() != 0)
60 MethodConfig
*Jnk
= Configs
;
61 Configs
= Configs
->Next
;
68 Queues
= Queues
->Next
;
73 // Acquire::Add - Add a new item /*{{{*/
74 // ---------------------------------------------------------------------
75 /* This puts an item on the acquire list. This list is mainly for tracking
77 void pkgAcquire::Add(Item
*Itm
)
82 // Acquire::Remove - Remove a item /*{{{*/
83 // ---------------------------------------------------------------------
84 /* Remove an item from the acquire list. This is usually not used.. */
85 void pkgAcquire::Remove(Item
*Itm
)
87 for (vector
<Item
*>::iterator I
= Items
.begin(); I
< Items
.end(); I
++)
94 // Acquire::Add - Add a worker /*{{{*/
95 // ---------------------------------------------------------------------
96 /* A list of workers is kept so that the select loop can direct their FD
98 void pkgAcquire::Add(Worker
*Work
)
100 Work
->NextAcquire
= Workers
;
104 // Acquire::Remove - Remove a worker /*{{{*/
105 // ---------------------------------------------------------------------
106 /* A worker has died. This can not be done while the select loop is running
107 as it would require that RunFds could handling a changing list state and
109 void pkgAcquire::Remove(Worker
*Work
)
114 Worker
**I
= &Workers
;
118 *I
= (*I
)->NextAcquire
;
120 I
= &(*I
)->NextAcquire
;
124 // Acquire::Enqueue - Queue an URI for fetching /*{{{*/
125 // ---------------------------------------------------------------------
126 /* This is the entry point for an item. An item calls this function when
127 it is construction which creates a queue (based on the current queue
128 mode) and puts the item in that queue. If the system is running then
129 the queue might be started. */
130 void pkgAcquire::Enqueue(ItemDesc
&Item
)
132 // Determine which queue to put the item in
133 const MethodConfig
*Config
;
134 string Name
= QueueName(Item
.URI
,Config
);
135 if (Name
.empty() == true)
138 // Find the queue structure
140 for (; I
!= 0 && I
->Name
!= Name
; I
= I
->Next
);
143 I
= new Queue(Name
,this);
151 // See if this is a local only URI
152 if (Config
->LocalOnly
== true && Item
.Owner
->Complete
== false)
153 Item
.Owner
->Local
= true;
154 Item
.Owner
->Status
= Item::StatIdle
;
156 // Queue it into the named queue
163 clog
<< "Fetching " << Item
.URI
<< endl
;
164 clog
<< " to " << Item
.Owner
->DestFile
<< endl
;
165 clog
<< " Queue is: " << Name
<< endl
;
169 // Acquire::Dequeue - Remove an item from all queues /*{{{*/
170 // ---------------------------------------------------------------------
171 /* This is called when an item is finished being fetched. It removes it
172 from all the queues */
173 void pkgAcquire::Dequeue(Item
*Itm
)
177 for (; I
!= 0; I
= I
->Next
)
178 Res
|= I
->Dequeue(Itm
);
181 clog
<< "Dequeuing " << Itm
->DestFile
<< endl
;
186 // Acquire::QueueName - Return the name of the queue for this URI /*{{{*/
187 // ---------------------------------------------------------------------
188 /* The string returned depends on the configuration settings and the
189 method parameters. Given something like http://foo.org/bar it can
190 return http://foo.org or http */
191 string
pkgAcquire::QueueName(string Uri
,MethodConfig
const *&Config
)
195 Config
= GetConfig(U
.Access
);
199 /* Single-Instance methods get exactly one queue per URI. This is
200 also used for the Access queue method */
201 if (Config
->SingleInstance
== true || QueueMode
== QueueAccess
)
204 return U
.Access
+ ':' + U
.Host
;
207 // Acquire::GetConfig - Fetch the configuration information /*{{{*/
208 // ---------------------------------------------------------------------
209 /* This locates the configuration structure for an access method. If
210 a config structure cannot be found a Worker will be created to
212 pkgAcquire::MethodConfig
*pkgAcquire::GetConfig(string Access
)
214 // Search for an existing config
216 for (Conf
= Configs
; Conf
!= 0; Conf
= Conf
->Next
)
217 if (Conf
->Access
== Access
)
220 // Create the new config class
221 Conf
= new MethodConfig
;
222 Conf
->Access
= Access
;
223 Conf
->Next
= Configs
;
226 // Create the worker to fetch the configuration
228 if (Work
.Start() == false)
234 // Acquire::SetFds - Deal with readable FDs /*{{{*/
235 // ---------------------------------------------------------------------
236 /* Collect FDs that have activity monitors into the fd sets */
237 void pkgAcquire::SetFds(int &Fd
,fd_set
*RSet
,fd_set
*WSet
)
239 for (Worker
*I
= Workers
; I
!= 0; I
= I
->NextAcquire
)
241 if (I
->InReady
== true && I
->InFd
>= 0)
245 FD_SET(I
->InFd
,RSet
);
247 if (I
->OutReady
== true && I
->OutFd
>= 0)
251 FD_SET(I
->OutFd
,WSet
);
256 // Acquire::RunFds - Deal with active FDs /*{{{*/
257 // ---------------------------------------------------------------------
258 /* Dispatch active FDs over to the proper workers. It is very important
259 that a worker never be erased while this is running! The queue class
260 should never erase a worker except during shutdown processing. */
261 void pkgAcquire::RunFds(fd_set
*RSet
,fd_set
*WSet
)
263 for (Worker
*I
= Workers
; I
!= 0; I
= I
->NextAcquire
)
265 if (I
->InFd
>= 0 && FD_ISSET(I
->InFd
,RSet
) != 0)
267 if (I
->OutFd
>= 0 && FD_ISSET(I
->OutFd
,WSet
) != 0)
272 // Acquire::Run - Run the fetch sequence /*{{{*/
273 // ---------------------------------------------------------------------
274 /* This runs the queues. It manages a select loop for all of the
275 Worker tasks. The workers interact with the queues and items to
276 manage the actual fetch. */
277 bool pkgAcquire::Run()
281 for (Queue
*I
= Queues
; I
!= 0; I
= I
->Next
)
287 // Run till all things have been acquired
298 SetFds(Highest
,&RFds
,&WFds
);
300 int Res
= select(Highest
+1,&RFds
,&WFds
,0,&tv
);
303 _error
->Errno("select","Select has failed");
308 if (_error
->PendingError() == true)
311 // Timeout, notify the log class
312 if (Res
== 0 || (Log
!= 0 && Log
->Update
== true))
315 for (Worker
*I
= Workers
; I
!= 0; I
= I
->NextAcquire
)
325 // Shut down the acquire bits
327 for (Queue
*I
= Queues
; I
!= 0; I
= I
->Next
)
330 return !_error
->PendingError();
333 // Acquire::Bump - Called when an item is dequeued /*{{{*/
334 // ---------------------------------------------------------------------
335 /* This routine bumps idle queues in hopes that they will be able to fetch
337 void pkgAcquire::Bump()
339 for (Queue
*I
= Queues
; I
!= 0; I
= I
->Next
)
343 // Acquire::WorkerStep - Step to the next worker /*{{{*/
344 // ---------------------------------------------------------------------
345 /* Not inlined to advoid including acquire-worker.h */
346 pkgAcquire::Worker
*pkgAcquire::WorkerStep(Worker
*I
)
348 return I
->NextAcquire
;
351 // Acquire::Clean - Cleans a directory /*{{{*/
352 // ---------------------------------------------------------------------
353 /* This is a bit simplistic, it looks at every file in the dir and sees
354 if it is part of the download set. */
355 bool pkgAcquire::Clean(string Dir
)
357 DIR *D
= opendir(Dir
.c_str());
359 return _error
->Errno("opendir","Unable to read %s",Dir
.c_str());
361 string StartDir
= SafeGetCWD();
362 if (chdir(Dir
.c_str()) != 0)
365 return _error
->Errno("chdir","Unable to change to ",Dir
.c_str());
368 for (struct dirent
*Dir
= readdir(D
); Dir
!= 0; Dir
= readdir(D
))
371 if (strcmp(Dir
->d_name
,"lock") == 0 ||
372 strcmp(Dir
->d_name
,"partial") == 0 ||
373 strcmp(Dir
->d_name
,".") == 0 ||
374 strcmp(Dir
->d_name
,"..") == 0)
377 // Look in the get list
378 vector
<Item
*>::iterator I
= Items
.begin();
379 for (; I
!= Items
.end(); I
++)
380 if (flNotDir((*I
)->DestFile
) == Dir
->d_name
)
383 // Nothing found, nuke it
384 if (I
== Items
.end())
388 chdir(StartDir
.c_str());
393 // Acquire::TotalNeeded - Number of bytes to fetch /*{{{*/
394 // ---------------------------------------------------------------------
395 /* This is the total number of bytes needed */
396 unsigned long pkgAcquire::TotalNeeded()
398 unsigned long Total
= 0;
399 for (pkgAcquire::Item
**I
= ItemsBegin(); I
!= ItemsEnd(); I
++)
400 Total
+= (*I
)->FileSize
;
404 // Acquire::FetchNeeded - Number of bytes needed to get /*{{{*/
405 // ---------------------------------------------------------------------
406 /* This is the number of bytes that is not local */
407 unsigned long pkgAcquire::FetchNeeded()
409 unsigned long Total
= 0;
410 for (pkgAcquire::Item
**I
= ItemsBegin(); I
!= ItemsEnd(); I
++)
411 if ((*I
)->Local
== false)
412 Total
+= (*I
)->FileSize
;
417 // Acquire::MethodConfig::MethodConfig - Constructor /*{{{*/
418 // ---------------------------------------------------------------------
420 pkgAcquire::MethodConfig::MethodConfig()
422 SingleInstance
= false;
430 // Queue::Queue - Constructor /*{{{*/
431 // ---------------------------------------------------------------------
433 pkgAcquire::Queue::Queue(string Name
,pkgAcquire
*Owner
) : Name(Name
),
441 // Queue::~Queue - Destructor /*{{{*/
442 // ---------------------------------------------------------------------
444 pkgAcquire::Queue::~Queue()
456 // Queue::Enqueue - Queue an item to the queue /*{{{*/
457 // ---------------------------------------------------------------------
459 void pkgAcquire::Queue::Enqueue(ItemDesc
&Item
)
462 for (; *I
!= 0; I
= &(*I
)->Next
);
465 QItem
*Itm
= new QItem
;
470 Item
.Owner
->QueueCounter
++;
471 if (Items
->Next
== 0)
475 // Queue::Dequeue - Remove an item from the queue /*{{{*/
476 // ---------------------------------------------------------------------
477 /* We return true if we hit something*/
478 bool pkgAcquire::Queue::Dequeue(Item
*Owner
)
485 if ((*I
)->Owner
== Owner
)
489 Owner
->QueueCounter
--;
500 // Queue::Startup - Start the worker processes /*{{{*/
501 // ---------------------------------------------------------------------
503 bool pkgAcquire::Queue::Startup()
508 pkgAcquire::MethodConfig
*Cnf
= Owner
->GetConfig(U
.Access
);
512 Workers
= new Worker(this,Cnf
,Owner
->Log
);
514 if (Workers
->Start() == false)
517 /* When pipelining we commit 10 items. This needs to change when we
518 added other source retry to have cycle maintain a pipeline depth
520 if (Cnf
->Pipeline
== true)
523 for (int I
= 0; I
!= 10 && Res
== true; I
++)
531 // Queue::Shutdown - Shutdown the worker processes /*{{{*/
532 // ---------------------------------------------------------------------
534 bool pkgAcquire::Queue::Shutdown()
536 // Delete all of the workers
539 pkgAcquire::Worker
*Jnk
= Workers
;
540 Workers
= Workers
->NextQueue
;
548 // Queue::Finditem - Find a URI in the item list /*{{{*/
549 // ---------------------------------------------------------------------
551 pkgAcquire::Queue::QItem
*pkgAcquire::Queue::FindItem(string URI
,pkgAcquire::Worker
*Owner
)
553 for (QItem
*I
= Items
; I
!= 0; I
= I
->Next
)
554 if (I
->URI
== URI
&& I
->Worker
== Owner
)
559 // Queue::ItemDone - Item has been completed /*{{{*/
560 // ---------------------------------------------------------------------
561 /* The worker signals this which causes the item to be removed from the
562 queue. If this is the last queue instance then it is removed from the
564 bool pkgAcquire::Queue::ItemDone(QItem
*Itm
)
566 if (Itm
->Owner
->QueueCounter
<= 1)
567 Owner
->Dequeue(Itm
->Owner
);
577 // Queue::Cycle - Queue new items into the method /*{{{*/
578 // ---------------------------------------------------------------------
579 /* This locates a new idle item and sends it to the worker */
580 bool pkgAcquire::Queue::Cycle()
582 if (Items
== 0 || Workers
== 0)
585 // Look for a queable item
587 for (; I
!= 0; I
= I
->Next
)
588 if (I
->Owner
->Status
== pkgAcquire::Item::StatIdle
)
591 // Nothing to do, queue is idle.
596 I
->Owner
->Status
= pkgAcquire::Item::StatFetching
;
597 return Workers
->QueueItem(I
);
600 // Queue::Bump - Fetch any pending objects if we are idle /*{{{*/
601 // ---------------------------------------------------------------------
603 void pkgAcquire::Queue::Bump()
608 // AcquireStatus::pkgAcquireStatus - Constructor /*{{{*/
609 // ---------------------------------------------------------------------
611 pkgAcquireStatus::pkgAcquireStatus()
616 // AcquireStatus::Pulse - Called periodically /*{{{*/
617 // ---------------------------------------------------------------------
618 /* This computes some internal state variables for the derived classes to
619 use. It generates the current downloaded bytes and total bytes to download
620 as well as the current CPS estimate. */
621 void pkgAcquireStatus::Pulse(pkgAcquire
*Owner
)
628 // Compute the total number of bytes to fetch
629 unsigned int Unknown
= 0;
630 unsigned int Count
= 0;
631 for (pkgAcquire::Item
**I
= Owner
->ItemsBegin(); I
!= Owner
->ItemsEnd();
635 if ((*I
)->Status
== pkgAcquire::Item::StatDone
)
638 // Totally ignore local items
639 if ((*I
)->Local
== true)
642 TotalBytes
+= (*I
)->FileSize
;
643 if ((*I
)->Complete
== true)
644 CurrentBytes
+= (*I
)->FileSize
;
645 if ((*I
)->FileSize
== 0 && (*I
)->Complete
== false)
649 // Compute the current completion
650 for (pkgAcquire::Worker
*I
= Owner
->WorkersBegin(); I
!= 0;
651 I
= Owner
->WorkerStep(I
))
652 if (I
->CurrentItem
!= 0 && I
->CurrentItem
->Owner
->Complete
== false)
653 CurrentBytes
+= I
->CurrentSize
;
655 // Normalize the figures and account for unknown size downloads
658 if (Unknown
== Count
)
659 TotalBytes
= Unknown
;
661 TotalBytes
+= TotalBytes
/(Count
- Unknown
)*Unknown
;
664 struct timeval NewTime
;
665 gettimeofday(&NewTime
,0);
666 if (NewTime
.tv_sec
- Time
.tv_sec
== 6 && NewTime
.tv_usec
> Time
.tv_usec
||
667 NewTime
.tv_sec
- Time
.tv_sec
> 6)
669 // Compute the delta time with full accuracy
670 long usdiff
= NewTime
.tv_usec
- Time
.tv_usec
;
671 long sdiff
= NewTime
.tv_sec
- Time
.tv_sec
;
680 // Compute the CPS value
681 if (sdiff
== 0 && usdiff
== 0)
684 CurrentCPS
= (CurrentBytes
- LastBytes
)/(sdiff
+ usdiff
/1000000.0);
685 LastBytes
= CurrentBytes
;
686 ElapsedTime
= NewTime
.tv_sec
- StartTime
.tv_sec
;
691 // AcquireStatus::Start - Called when the download is started /*{{{*/
692 // ---------------------------------------------------------------------
693 /* We just reset the counters */
694 void pkgAcquireStatus::Start()
696 gettimeofday(&Time
,0);
697 gettimeofday(&StartTime
,0);
708 // AcquireStatus::Stop - Finished downloading /*{{{*/
709 // ---------------------------------------------------------------------
710 /* This accurately computes the elapsed time and the total overall CPS. */
711 void pkgAcquireStatus::Stop()
713 // Compute the CPS and elapsed time
714 struct timeval NewTime
;
715 gettimeofday(&NewTime
,0);
717 // Compute the delta time with full accuracy
718 long usdiff
= NewTime
.tv_usec
- StartTime
.tv_usec
;
719 long sdiff
= NewTime
.tv_sec
- StartTime
.tv_sec
;
728 // Compute the CPS value
729 if (sdiff
== 0 && usdiff
== 0)
732 CurrentCPS
= FetchedBytes
/(sdiff
+ usdiff
/1000000.0);
733 LastBytes
= CurrentBytes
;
737 // AcquireStatus::Fetched - Called when a byte set has been fetched /*{{{*/
738 // ---------------------------------------------------------------------
739 /* This is used to get accurate final transfer rate reporting. */
740 void pkgAcquireStatus::Fetched(unsigned long Size
,unsigned long Resume
)
742 FetchedBytes
+= Size
- Resume
;