]>
Commit | Line | Data |
---|---|---|
0118833a AL |
1 | // -*- mode: cpp; mode: fold -*- |
2 | // Description /*{{{*/ | |
b3d44315 | 3 | // $Id: acquire.h,v 1.29.2.1 2003/12/24 23:09:17 mdz Exp $ |
0118833a AL |
4 | /* ###################################################################### |
5 | ||
6 | Acquire - File Acquiration | |
7 | ||
8 | This module contians the Acquire system. It is responsible for bringing | |
9 | files into the local pathname space. It deals with URIs for files and | |
10 | URI handlers responsible for downloading or finding the URIs. | |
11 | ||
12 | Each file to download is represented by an Acquire::Item class subclassed | |
13 | into a specialization. The Item class can add itself to several URI | |
14 | acquire queues each prioritized by the download scheduler. When the | |
15 | system is run the proper URI handlers are spawned and the the acquire | |
16 | queues are fed into the handlers by the schedular until the queues are | |
17 | empty. This allows for an Item to be downloaded from an alternate source | |
18 | if the first try turns out to fail. It also alows concurrent downloading | |
19 | of multiple items from multiple sources as well as dynamic balancing | |
20 | of load between the sources. | |
21 | ||
22 | Schedualing of downloads is done on a first ask first get basis. This | |
23 | preserves the order of the download as much as possible. And means the | |
24 | fastest source will tend to process the largest number of files. | |
25 | ||
26 | Internal methods and queues for performing gzip decompression, | |
27 | md5sum hashing and file copying are provided to allow items to apply | |
28 | a number of transformations to the data files they are working with. | |
29 | ||
30 | ##################################################################### */ | |
31 | /*}}}*/ | |
32 | #ifndef PKGLIB_ACQUIRE_H | |
33 | #define PKGLIB_ACQUIRE_H | |
34 | ||
35 | #include <vector> | |
36 | #include <string> | |
37 | ||
b4fc9b6f AL |
38 | using std::vector; |
39 | using std::string; | |
40 | ||
0118833a AL |
41 | #ifdef __GNUG__ |
42 | #pragma interface "apt-pkg/acquire.h" | |
43 | #endif | |
44 | ||
b98f2859 | 45 | #include <sys/time.h> |
0a8a80e5 AL |
46 | #include <unistd.h> |
47 | ||
8267fe24 | 48 | class pkgAcquireStatus; |
0118833a AL |
49 | class pkgAcquire |
50 | { | |
51 | public: | |
52 | ||
53 | class Item; | |
54 | class Queue; | |
55 | class Worker; | |
56 | struct MethodConfig; | |
8267fe24 | 57 | struct ItemDesc; |
b2e465d6 AL |
58 | friend class Item; |
59 | friend class Queue; | |
b4fc9b6f AL |
60 | |
61 | typedef vector<Item *>::iterator ItemIterator; | |
62 | typedef vector<Item *>::const_iterator ItemCIterator; | |
0118833a AL |
63 | |
64 | protected: | |
65 | ||
0a8a80e5 | 66 | // List of items to fetch |
0118833a | 67 | vector<Item *> Items; |
0a8a80e5 AL |
68 | |
69 | // List of active queues and fetched method configuration parameters | |
0118833a | 70 | Queue *Queues; |
0a8a80e5 | 71 | Worker *Workers; |
0118833a | 72 | MethodConfig *Configs; |
8267fe24 | 73 | pkgAcquireStatus *Log; |
0a8a80e5 | 74 | unsigned long ToFetch; |
8267fe24 | 75 | |
0a8a80e5 AL |
76 | // Configurable parameters for the schedular |
77 | enum {QueueHost,QueueAccess} QueueMode; | |
78 | bool Debug; | |
8b89e57f | 79 | bool Running; |
0118833a AL |
80 | |
81 | void Add(Item *Item); | |
82 | void Remove(Item *Item); | |
0a8a80e5 AL |
83 | void Add(Worker *Work); |
84 | void Remove(Worker *Work); | |
85 | ||
8267fe24 | 86 | void Enqueue(ItemDesc &Item); |
0a8a80e5 | 87 | void Dequeue(Item *Item); |
e331f6ed | 88 | string QueueName(string URI,MethodConfig const *&Config); |
0a8a80e5 AL |
89 | |
90 | // FDSET managers for derived classes | |
281daf46 AL |
91 | virtual void SetFds(int &Fd,fd_set *RSet,fd_set *WSet); |
92 | virtual void RunFds(fd_set *RSet,fd_set *WSet); | |
93bf083d AL |
93 | |
94 | // A queue calls this when it dequeues an item | |
95 | void Bump(); | |
0118833a AL |
96 | |
97 | public: | |
3b5421b4 | 98 | |
0a8a80e5 | 99 | MethodConfig *GetConfig(string Access); |
024d1123 AL |
100 | |
101 | enum RunResult {Continue,Failed,Cancelled}; | |
102 | ||
42ab8223 | 103 | RunResult Run(int PulseIntervall=500000); |
281daf46 AL |
104 | void Shutdown(); |
105 | ||
8267fe24 AL |
106 | // Simple iteration mechanism |
107 | inline Worker *WorkersBegin() {return Workers;}; | |
108 | Worker *WorkerStep(Worker *I); | |
b4fc9b6f AL |
109 | inline ItemIterator ItemsBegin() {return Items.begin();}; |
110 | inline ItemIterator ItemsEnd() {return Items.end();}; | |
f7a08e33 AL |
111 | |
112 | // Iterate over queued Item URIs | |
113 | class UriIterator; | |
114 | UriIterator UriBegin(); | |
115 | UriIterator UriEnd(); | |
116 | ||
7a7fa5f0 AL |
117 | // Cleans out the download dir |
118 | bool Clean(string Dir); | |
a6568219 AL |
119 | |
120 | // Returns the size of the total download set | |
b2e465d6 AL |
121 | double TotalNeeded(); |
122 | double FetchNeeded(); | |
123 | double PartialPresent(); | |
b3d44315 | 124 | |
8267fe24 | 125 | pkgAcquire(pkgAcquireStatus *Log = 0); |
58d63ae6 | 126 | virtual ~pkgAcquire(); |
0118833a AL |
127 | }; |
128 | ||
8267fe24 AL |
129 | // Description of an Item+URI |
130 | struct pkgAcquire::ItemDesc | |
131 | { | |
132 | string URI; | |
133 | string Description; | |
134 | string ShortDesc; | |
135 | Item *Owner; | |
136 | }; | |
137 | ||
0118833a AL |
138 | // List of possible items queued for download. |
139 | class pkgAcquire::Queue | |
140 | { | |
b2e465d6 AL |
141 | friend class pkgAcquire; |
142 | friend class pkgAcquire::UriIterator; | |
143 | friend class pkgAcquire::Worker; | |
0118833a AL |
144 | Queue *Next; |
145 | ||
146 | protected: | |
3b5421b4 | 147 | |
0a8a80e5 | 148 | // Queued item |
8267fe24 | 149 | struct QItem : pkgAcquire::ItemDesc |
0a8a80e5 | 150 | { |
8267fe24 | 151 | QItem *Next; |
c88edf1d | 152 | pkgAcquire::Worker *Worker; |
8267fe24 AL |
153 | |
154 | void operator =(pkgAcquire::ItemDesc const &I) | |
155 | { | |
156 | URI = I.URI; | |
157 | Description = I.Description; | |
158 | ShortDesc = I.ShortDesc; | |
159 | Owner = I.Owner; | |
160 | }; | |
161 | }; | |
0a8a80e5 AL |
162 | |
163 | // Name of the queue | |
164 | string Name; | |
165 | ||
166 | // Items queued into this queue | |
167 | QItem *Items; | |
168 | pkgAcquire::Worker *Workers; | |
169 | pkgAcquire *Owner; | |
e7432370 | 170 | signed long PipeDepth; |
b185acc2 | 171 | unsigned long MaxPipeDepth; |
0118833a AL |
172 | |
173 | public: | |
0a8a80e5 AL |
174 | |
175 | // Put an item into this queue | |
c03462c6 | 176 | bool Enqueue(ItemDesc &Item); |
bfd22fc0 | 177 | bool Dequeue(Item *Owner); |
0a8a80e5 | 178 | |
c88edf1d AL |
179 | // Find a Queued item |
180 | QItem *FindItem(string URI,pkgAcquire::Worker *Owner); | |
8267fe24 | 181 | bool ItemStart(QItem *Itm,unsigned long Size); |
c88edf1d AL |
182 | bool ItemDone(QItem *Itm); |
183 | ||
0a8a80e5 | 184 | bool Startup(); |
8e5fc8f5 | 185 | bool Shutdown(bool Final); |
93bf083d | 186 | bool Cycle(); |
be4401bf | 187 | void Bump(); |
0a8a80e5 AL |
188 | |
189 | Queue(string Name,pkgAcquire *Owner); | |
190 | ~Queue(); | |
0118833a AL |
191 | }; |
192 | ||
f7a08e33 AL |
193 | class pkgAcquire::UriIterator |
194 | { | |
195 | pkgAcquire::Queue *CurQ; | |
196 | pkgAcquire::Queue::QItem *CurItem; | |
197 | ||
198 | public: | |
199 | ||
200 | // Advance to the next item | |
201 | inline void operator ++() {operator ++();}; | |
202 | void operator ++(int) | |
203 | { | |
204 | CurItem = CurItem->Next; | |
205 | while (CurItem == 0 && CurQ != 0) | |
206 | { | |
207 | CurItem = CurQ->Items; | |
208 | CurQ = CurQ->Next; | |
209 | } | |
210 | }; | |
211 | ||
212 | // Accessors | |
213 | inline pkgAcquire::ItemDesc const *operator ->() const {return CurItem;}; | |
214 | inline bool operator !=(UriIterator const &rhs) const {return rhs.CurQ != CurQ || rhs.CurItem != CurItem;}; | |
215 | inline bool operator ==(UriIterator const &rhs) const {return rhs.CurQ == CurQ && rhs.CurItem == CurItem;}; | |
216 | ||
217 | UriIterator(pkgAcquire::Queue *Q) : CurQ(Q), CurItem(0) | |
218 | { | |
219 | while (CurItem == 0 && CurQ != 0) | |
220 | { | |
221 | CurItem = CurQ->Items; | |
222 | CurQ = CurQ->Next; | |
223 | } | |
224 | } | |
225 | }; | |
226 | ||
0118833a AL |
227 | // Configuration information from each method |
228 | struct pkgAcquire::MethodConfig | |
229 | { | |
3b5421b4 AL |
230 | MethodConfig *Next; |
231 | ||
0118833a AL |
232 | string Access; |
233 | ||
234 | string Version; | |
235 | bool SingleInstance; | |
0a8a80e5 AL |
236 | bool Pipeline; |
237 | bool SendConfig; | |
e331f6ed | 238 | bool LocalOnly; |
8e5fc8f5 | 239 | bool NeedsCleanup; |
459681d3 | 240 | bool Removable; |
8e5fc8f5 | 241 | |
0118833a | 242 | MethodConfig(); |
0118833a AL |
243 | }; |
244 | ||
8267fe24 AL |
245 | class pkgAcquireStatus |
246 | { | |
b98f2859 AL |
247 | protected: |
248 | ||
249 | struct timeval Time; | |
250 | struct timeval StartTime; | |
b2e465d6 | 251 | double LastBytes; |
b98f2859 | 252 | double CurrentCPS; |
b2e465d6 AL |
253 | double CurrentBytes; |
254 | double TotalBytes; | |
255 | double FetchedBytes; | |
b98f2859 | 256 | unsigned long ElapsedTime; |
d568ed2d AL |
257 | unsigned long TotalItems; |
258 | unsigned long CurrentItems; | |
b98f2859 | 259 | |
8267fe24 AL |
260 | public: |
261 | ||
262 | bool Update; | |
c5ccf175 AL |
263 | bool MorePulses; |
264 | ||
b98f2859 AL |
265 | // Called by items when they have finished a real download |
266 | virtual void Fetched(unsigned long Size,unsigned long ResumePoint); | |
267 | ||
542ec555 AL |
268 | // Called to change media |
269 | virtual bool MediaChange(string Media,string Drive) = 0; | |
270 | ||
8267fe24 | 271 | // Each of these is called by the workers when an event occures |
727f18af AL |
272 | virtual void IMSHit(pkgAcquire::ItemDesc &/*Itm*/) {}; |
273 | virtual void Fetch(pkgAcquire::ItemDesc &/*Itm*/) {}; | |
274 | virtual void Done(pkgAcquire::ItemDesc &/*Itm*/) {}; | |
275 | virtual void Fail(pkgAcquire::ItemDesc &/*Itm*/) {}; | |
024d1123 | 276 | virtual bool Pulse(pkgAcquire *Owner); // returns false on user cancel |
b98f2859 AL |
277 | virtual void Start(); |
278 | virtual void Stop(); | |
a6568219 | 279 | |
b98f2859 | 280 | pkgAcquireStatus(); |
8267fe24 AL |
281 | virtual ~pkgAcquireStatus() {}; |
282 | }; | |
283 | ||
0118833a | 284 | #endif |