Commit | Line | Data |
---|---|---|
5f6b130d MV |
1 | // -*- mode: cpp; mode: fold -*- |
2 | // Description /*{{{*/ | |
3 | // $Id: mirror.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $ | |
4 | /* ###################################################################### | |
5 | ||
1e3f4083 | 6 | Mirror Acquire Method - This is the Mirror acquire method for APT. |
5f6b130d MV |
7 | |
8 | ##################################################################### */ | |
9 | /*}}}*/ | |
10 | // Include Files /*{{{*/ | |
ea542140 DK |
11 | #include <config.h> |
12 | ||
ae54c535 | 13 | #include <apt-pkg/aptconfiguration.h> |
5f6b130d MV |
14 | #include <apt-pkg/fileutl.h> |
15 | #include <apt-pkg/acquire-method.h> | |
16 | #include <apt-pkg/acquire-item.h> | |
17 | #include <apt-pkg/acquire.h> | |
18 | #include <apt-pkg/error.h> | |
0c312e0e | 19 | #include <apt-pkg/sourcelist.h> |
472ff00e DK |
20 | #include <apt-pkg/configuration.h> |
21 | #include <apt-pkg/metaindex.h> | |
453b82a3 | 22 | #include <apt-pkg/strutl.h> |
78c8f3cd | 23 | |
453b82a3 DK |
24 | #include <stdlib.h> |
25 | #include <string.h> | |
26 | #include <unistd.h> | |
01a695e2 | 27 | #include <algorithm> |
5f6b130d | 28 | #include <iostream> |
453b82a3 | 29 | #include <fstream> |
d731f9c5 | 30 | #include <sys/stat.h> |
78c8f3cd | 31 | #include <sys/utsname.h> |
70288656 | 32 | #include <dirent.h> |
14e097c1 | 33 | |
5f6b130d MV |
34 | using namespace std; |
35 | ||
0ded3ad3 MV |
36 | #include<sstream> |
37 | ||
5f6b130d MV |
38 | #include "mirror.h" |
39 | #include "http.h" | |
ea542140 | 40 | #include <apti18n.h> |
5f6b130d MV |
41 | /*}}}*/ |
42 | ||
362d2934 | 43 | /* Done: |
59271f62 | 44 | * - works with http (only!) |
362d2934 MV |
45 | * - always picks the first mirror from the list |
46 | * - call out to problem reporting script | |
47 | * - supports "deb mirror://host/path/to/mirror-list/// dist component" | |
a577a938 | 48 | * - uses pkgAcqMethod::FailReason() to have a string representation |
59271f62 | 49 | * of the failure that is also send to LP |
362d2934 | 50 | * |
86c17f0a | 51 | * TODO: |
1e3f4083 | 52 | * - deal with running as non-root because we can't write to the lists |
d731f9c5 | 53 | dir then -> use the cached mirror file |
86c17f0a | 54 | * - better method to download than having a pkgAcquire interface here |
3f599bb7 | 55 | * and better error handling there! |
066b53e9 | 56 | * - support more than http |
86c17f0a | 57 | * - testing :) |
86c17f0a MV |
58 | */ |
59 | ||
5f6b130d | 60 | MirrorMethod::MirrorMethod() |
f5a34606 | 61 | : HttpMethod(), DownloadedMirrorFile(false), Debug(false) |
5f6b130d | 62 | { |
d3e8fbb3 | 63 | } |
5f6b130d | 64 | |
14e097c1 MV |
65 | // HttpMethod::Configuration - Handle a configuration message /*{{{*/ |
66 | // --------------------------------------------------------------------- | |
67 | /* We stash the desired pipeline depth */ | |
68 | bool MirrorMethod::Configuration(string Message) | |
69 | { | |
70 | if (pkgAcqMethod::Configuration(Message) == false) | |
71 | return false; | |
72 | Debug = _config->FindB("Debug::Acquire::mirror",false); | |
73 | ||
74 | return true; | |
75 | } | |
76 | /*}}}*/ | |
77 | ||
d731f9c5 | 78 | // clean the mirrors dir based on ttl information |
70288656 | 79 | bool MirrorMethod::Clean(string Dir) |
d731f9c5 | 80 | { |
0c312e0e MV |
81 | vector<metaIndex *>::const_iterator I; |
82 | ||
83 | if(Debug) | |
84 | clog << "MirrorMethod::Clean(): " << Dir << endl; | |
85 | ||
38eedeb7 MV |
86 | if(Dir == "/") |
87 | return _error->Error("will not clean: '/'"); | |
88 | ||
0c312e0e MV |
89 | // read sources.list |
90 | pkgSourceList list; | |
91 | list.ReadMainList(); | |
70288656 | 92 | |
d3e8fbb3 | 93 | DIR *D = opendir(Dir.c_str()); |
70288656 MV |
94 | if (D == 0) |
95 | return _error->Errno("opendir",_("Unable to read %s"),Dir.c_str()); | |
d3e8fbb3 | 96 | |
70288656 MV |
97 | string StartDir = SafeGetCWD(); |
98 | if (chdir(Dir.c_str()) != 0) | |
99 | { | |
100 | closedir(D); | |
101 | return _error->Errno("chdir",_("Unable to change to %s"),Dir.c_str()); | |
102 | } | |
d3e8fbb3 | 103 | |
70288656 MV |
104 | for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D)) |
105 | { | |
106 | // Skip some files.. | |
107 | if (strcmp(Dir->d_name,"lock") == 0 || | |
108 | strcmp(Dir->d_name,"partial") == 0 || | |
6aef1942 | 109 | strcmp(Dir->d_name,"lost+found") == 0 || |
70288656 MV |
110 | strcmp(Dir->d_name,".") == 0 || |
111 | strcmp(Dir->d_name,"..") == 0) | |
112 | continue; | |
0c312e0e MV |
113 | |
114 | // see if we have that uri | |
f7f0d6c7 | 115 | for(I=list.begin(); I != list.end(); ++I) |
70288656 | 116 | { |
0c312e0e | 117 | string uri = (*I)->GetURI(); |
9ce3cfc9 | 118 | if(uri.compare(0, strlen("mirror://"), "mirror://") != 0) |
0c312e0e | 119 | continue; |
066b53e9 | 120 | string BaseUri = uri.substr(0,uri.size()-1); |
0c312e0e MV |
121 | if (URItoFileName(BaseUri) == Dir->d_name) |
122 | break; | |
70288656 | 123 | } |
0c312e0e MV |
124 | // nothing found, nuke it |
125 | if (I == list.end()) | |
ce1f3a2c | 126 | RemoveFile("mirror", Dir->d_name); |
d3e8fbb3 | 127 | } |
319790f4 | 128 | |
70288656 | 129 | closedir(D); |
319790f4 DK |
130 | if (chdir(StartDir.c_str()) != 0) |
131 | return _error->Errno("chdir",_("Unable to change to %s"),StartDir.c_str()); | |
d3e8fbb3 | 132 | return true; |
d731f9c5 MV |
133 | } |
134 | ||
14e097c1 | 135 | |
65512241 | 136 | bool MirrorMethod::DownloadMirrorFile(string /*mirror_uri_str*/) |
38eedeb7 | 137 | { |
d3e8fbb3 | 138 | // not that great to use pkgAcquire here, but we do not have |
38eedeb7 MV |
139 | // any other way right now |
140 | string fetch = BaseUri; | |
141 | fetch.replace(0,strlen("mirror://"),"http://"); | |
142 | ||
d3e8fbb3 | 143 | #if 0 // no need for this, the getArchitectures() will also include the main |
ae54c535 MV |
144 | // arch |
145 | // append main architecture | |
146 | fetch += "?arch=" + _config->Find("Apt::Architecture"); | |
147 | #endif | |
148 | ||
149 | // append all architectures | |
150 | std::vector<std::string> vec = APT::Configuration::getArchitectures(); | |
151 | for (std::vector<std::string>::const_iterator I = vec.begin(); | |
13ad8ce3 | 152 | I != vec.end(); ++I) |
ae54c535 | 153 | if (I == vec.begin()) |
046e104e | 154 | fetch += "?arch=" + (*I); |
ae54c535 MV |
155 | else |
156 | fetch += "&arch=" + (*I); | |
157 | ||
6885f3de MV |
158 | // append the dist as a query string |
159 | if (Dist != "") | |
ae54c535 | 160 | fetch += "&dist=" + Dist; |
6885f3de | 161 | |
d6cc7079 MV |
162 | if(Debug) |
163 | clog << "MirrorMethod::DownloadMirrorFile(): '" << fetch << "'" | |
164 | << " to " << MirrorFile << endl; | |
165 | ||
38eedeb7 MV |
166 | pkgAcquire Fetcher; |
167 | new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile); | |
168 | bool res = (Fetcher.Run() == pkgAcquire::Continue); | |
83e6798e | 169 | if(res) { |
38eedeb7 | 170 | DownloadedMirrorFile = true; |
83e6798e MV |
171 | chmod(MirrorFile.c_str(), 0644); |
172 | } | |
38eedeb7 | 173 | Fetcher.Shutdown(); |
d6cc7079 MV |
174 | |
175 | if(Debug) | |
176 | clog << "MirrorMethod::DownloadMirrorFile() success: " << res << endl; | |
d3e8fbb3 | 177 | |
38eedeb7 MV |
178 | return res; |
179 | } | |
180 | ||
01a695e2 MV |
181 | // Randomizes the lines in the mirror file, this is used so that |
182 | // we spread the load on the mirrors evenly | |
183 | bool MirrorMethod::RandomizeMirrorFile(string mirror_file) | |
184 | { | |
185 | vector<string> content; | |
186 | string line; | |
187 | ||
0004842d MV |
188 | if (!FileExists(mirror_file)) |
189 | return false; | |
190 | ||
d3e8fbb3 | 191 | // read |
01a695e2 | 192 | ifstream in(mirror_file.c_str()); |
0190e315 | 193 | while ( !in.eof() ) { |
01a695e2 MV |
194 | getline(in, line); |
195 | content.push_back(line); | |
196 | } | |
d3e8fbb3 | 197 | |
78c8f3cd MV |
198 | // we want the file to be random for each different machine, but also |
199 | // "stable" on the same machine. this is to avoid running into out-of-sync | |
200 | // issues (i.e. Release/Release.gpg different on each mirror) | |
201 | struct utsname buf; | |
9ce3cfc9 | 202 | int seed=1; |
78c8f3cd | 203 | if(uname(&buf) == 0) { |
9ce3cfc9 | 204 | for(int i=0,seed=1; buf.nodename[i] != 0; ++i) { |
78c8f3cd MV |
205 | seed = seed * 31 + buf.nodename[i]; |
206 | } | |
207 | } | |
208 | srand( seed ); | |
01a695e2 MV |
209 | random_shuffle(content.begin(), content.end()); |
210 | ||
211 | // write | |
212 | ofstream out(mirror_file.c_str()); | |
213 | while ( !content.empty()) { | |
214 | line = content.back(); | |
215 | content.pop_back(); | |
216 | out << line << "\n"; | |
217 | } | |
218 | ||
219 | return true; | |
220 | } | |
221 | ||
03915427 MV |
222 | /* convert a the Queue->Uri back to the mirror base uri and look |
223 | * at all mirrors we have for this, this is needed as queue->uri | |
224 | * may point to different mirrors (if TryNextMirror() was run) | |
225 | */ | |
226 | void MirrorMethod::CurrentQueueUriToMirror() | |
227 | { | |
228 | // already in mirror:// style so nothing to do | |
229 | if(Queue->Uri.find("mirror://") == 0) | |
230 | return; | |
231 | ||
232 | // find current mirror and select next one | |
51561c4d DK |
233 | for (vector<string>::const_iterator mirror = AllMirrors.begin(); |
234 | mirror != AllMirrors.end(); ++mirror) | |
03915427 | 235 | { |
51561c4d | 236 | if (Queue->Uri.find(*mirror) == 0) |
03915427 | 237 | { |
51561c4d | 238 | Queue->Uri.replace(0, mirror->length(), BaseUri); |
03915427 MV |
239 | return; |
240 | } | |
241 | } | |
242 | _error->Error("Internal error: Failed to convert %s back to %s", | |
963b16dc | 243 | Queue->Uri.c_str(), BaseUri.c_str()); |
03915427 MV |
244 | } |
245 | ||
246 | bool MirrorMethod::TryNextMirror() | |
96db74ce | 247 | { |
03915427 | 248 | // find current mirror and select next one |
51561c4d DK |
249 | for (vector<string>::const_iterator mirror = AllMirrors.begin(); |
250 | mirror != AllMirrors.end(); ++mirror) | |
03915427 | 251 | { |
51561c4d DK |
252 | if (Queue->Uri.find(*mirror) != 0) |
253 | continue; | |
254 | ||
255 | vector<string>::const_iterator nextmirror = mirror + 1; | |
d6cc7079 | 256 | if (nextmirror == AllMirrors.end()) |
51561c4d DK |
257 | break; |
258 | Queue->Uri.replace(0, mirror->length(), *nextmirror); | |
259 | if (Debug) | |
260 | clog << "TryNextMirror: " << Queue->Uri << endl; | |
196fd136 MV |
261 | |
262 | // inform parent | |
263 | UsedMirror = *nextmirror; | |
264 | Log("Switching mirror"); | |
51561c4d | 265 | return true; |
03915427 MV |
266 | } |
267 | ||
963b16dc MV |
268 | if (Debug) |
269 | clog << "TryNextMirror could not find another mirror to try" << endl; | |
270 | ||
0ded3ad3 | 271 | return false; |
96db74ce MV |
272 | } |
273 | ||
274 | bool MirrorMethod::InitMirrors() | |
38eedeb7 MV |
275 | { |
276 | // if we do not have a MirrorFile, fallback | |
277 | if(!FileExists(MirrorFile)) | |
278 | { | |
279 | // FIXME: fallback to a default mirror here instead | |
280 | // and provide a config option to define that default | |
281 | return _error->Error(_("No mirror file '%s' found "), MirrorFile.c_str()); | |
282 | } | |
283 | ||
b46fb8ff MV |
284 | if (access(MirrorFile.c_str(), R_OK) != 0) |
285 | { | |
286 | // FIXME: fallback to a default mirror here instead | |
287 | // and provide a config option to define that default | |
288 | return _error->Error(_("Can not read mirror file '%s'"), MirrorFile.c_str()); | |
289 | } | |
290 | ||
38eedeb7 MV |
291 | // FIXME: make the mirror selection more clever, do not |
292 | // just use the first one! | |
293 | // BUT: we can not make this random, the mirror has to be | |
1e3f4083 | 294 | // stable across session, because otherwise we can |
38eedeb7 MV |
295 | // get into sync issues (got indexfiles from mirror A, |
296 | // but packages from mirror B - one might be out of date etc) | |
297 | ifstream in(MirrorFile.c_str()); | |
96db74ce MV |
298 | string s; |
299 | while (!in.eof()) | |
300 | { | |
301 | getline(in, s); | |
95f395cc MV |
302 | |
303 | // ignore lines that start with # | |
304 | if (s.find("#") == 0) | |
305 | continue; | |
306 | // ignore empty lines | |
307 | if (s.size() == 0) | |
308 | continue; | |
309 | // ignore non http lines | |
9ce3cfc9 | 310 | if (s.compare(0, strlen("http://"), "http://") != 0) |
95f395cc MV |
311 | continue; |
312 | ||
313 | AllMirrors.push_back(s); | |
96db74ce | 314 | } |
245ba2c3 MV |
315 | if (AllMirrors.empty()) { |
316 | return _error->Error(_("No entry found in mirror file '%s'"), MirrorFile.c_str()); | |
317 | } | |
03915427 MV |
318 | Mirror = AllMirrors[0]; |
319 | UsedMirror = Mirror; | |
38eedeb7 MV |
320 | return true; |
321 | } | |
322 | ||
323 | string MirrorMethod::GetMirrorFileName(string mirror_uri_str) | |
5f6b130d | 324 | { |
066b53e9 MV |
325 | /* |
326 | - a mirror_uri_str looks like this: | |
327 | mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors/dists/feisty/Release.gpg | |
328 | ||
329 | - the matching source.list entry | |
330 | deb mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors feisty main | |
331 | ||
332 | - we actually want to go after: | |
333 | http://people.ubuntu.com/~mvo/apt/mirror/mirrors | |
334 | ||
335 | And we need to save the BaseUri for later: | |
336 | - mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors | |
337 | ||
338 | FIXME: what if we have two similar prefixes? | |
339 | mirror://people.ubuntu.com/~mvo/mirror | |
340 | mirror://people.ubuntu.com/~mvo/mirror2 | |
341 | then mirror_uri_str looks like: | |
342 | mirror://people.ubuntu.com/~mvo/apt/mirror/dists/feisty/Release.gpg | |
343 | mirror://people.ubuntu.com/~mvo/apt/mirror2/dists/feisty/Release.gpg | |
344 | we search sources.list and find: | |
345 | mirror://people.ubuntu.com/~mvo/apt/mirror | |
346 | in both cases! So we need to apply some domain knowledge here :( and | |
347 | check for /dists/ or /Release.gpg as suffixes | |
348 | */ | |
38eedeb7 | 349 | string name; |
f0b509cd | 350 | if(Debug) |
38eedeb7 | 351 | std::cerr << "GetMirrorFileName: " << mirror_uri_str << std::endl; |
066b53e9 MV |
352 | |
353 | // read sources.list and find match | |
354 | vector<metaIndex *>::const_iterator I; | |
355 | pkgSourceList list; | |
356 | list.ReadMainList(); | |
f7f0d6c7 | 357 | for(I=list.begin(); I != list.end(); ++I) |
066b53e9 MV |
358 | { |
359 | string uristr = (*I)->GetURI(); | |
f0b509cd MV |
360 | if(Debug) |
361 | std::cerr << "Checking: " << uristr << std::endl; | |
066b53e9 MV |
362 | if(uristr.substr(0,strlen("mirror://")) != string("mirror://")) |
363 | continue; | |
364 | // find matching uri in sources.list | |
365 | if(mirror_uri_str.substr(0,uristr.size()) == uristr) | |
366 | { | |
f0b509cd MV |
367 | if(Debug) |
368 | std::cerr << "found BaseURI: " << uristr << std::endl; | |
066b53e9 | 369 | BaseUri = uristr.substr(0,uristr.size()-1); |
6885f3de | 370 | Dist = (*I)->GetDist(); |
066b53e9 MV |
371 | } |
372 | } | |
70288656 | 373 | // get new file |
38eedeb7 | 374 | name = _config->FindDir("Dir::State::mirrors") + URItoFileName(BaseUri); |
5f6b130d | 375 | |
14e097c1 MV |
376 | if(Debug) |
377 | { | |
378 | cerr << "base-uri: " << BaseUri << endl; | |
38eedeb7 | 379 | cerr << "mirror-file: " << name << endl; |
d731f9c5 | 380 | } |
38eedeb7 | 381 | return name; |
5f6b130d MV |
382 | } |
383 | ||
384 | // MirrorMethod::Fetch - Fetch an item /*{{{*/ | |
385 | // --------------------------------------------------------------------- | |
386 | /* This adds an item to the pipeline. We keep the pipeline at a fixed | |
387 | depth. */ | |
388 | bool MirrorMethod::Fetch(FetchItem *Itm) | |
389 | { | |
5dad4134 MV |
390 | if(Debug) |
391 | clog << "MirrorMethod::Fetch()" << endl; | |
392 | ||
38eedeb7 MV |
393 | // the http method uses Fetch(0) as a way to update the pipeline, |
394 | // just let it do its work in this case - Fetch() with a valid | |
395 | // Itm will always run before the first Fetch(0) | |
396 | if(Itm == NULL) | |
397 | return HttpMethod::Fetch(Itm); | |
398 | ||
399 | // if we don't have the name of the mirror file on disk yet, | |
400 | // calculate it now (can be derived from the uri) | |
401 | if(MirrorFile.empty()) | |
402 | MirrorFile = GetMirrorFileName(Itm->Uri); | |
403 | ||
404 | // download mirror file once (if we are after index files) | |
405 | if(Itm->IndexFile && !DownloadedMirrorFile) | |
5f6b130d | 406 | { |
70288656 | 407 | Clean(_config->FindDir("Dir::State::mirrors")); |
0004842d MV |
408 | if (DownloadMirrorFile(Itm->Uri)) |
409 | RandomizeMirrorFile(MirrorFile); | |
5f6b130d MV |
410 | } |
411 | ||
2ac9b90b | 412 | if(AllMirrors.empty()) { |
96db74ce | 413 | if(!InitMirrors()) { |
5dad4134 MV |
414 | // no valid mirror selected, something went wrong downloading |
415 | // from the master mirror site most likely and there is | |
416 | // no old mirror file availalbe | |
417 | return false; | |
418 | } | |
419 | } | |
5dad4134 | 420 | |
963b16dc MV |
421 | if(Itm->Uri.find("mirror://") != string::npos) |
422 | Itm->Uri.replace(0,BaseUri.size(), Mirror); | |
38eedeb7 | 423 | |
963b16dc MV |
424 | if(Debug) |
425 | clog << "Fetch: " << Itm->Uri << endl << endl; | |
d3e8fbb3 | 426 | |
14e097c1 MV |
427 | // now run the real fetcher |
428 | return HttpMethod::Fetch(Itm); | |
d3e8fbb3 | 429 | } |
5f6b130d | 430 | |
14e097c1 MV |
431 | void MirrorMethod::Fail(string Err,bool Transient) |
432 | { | |
2ac9b90b MV |
433 | // FIXME: TryNextMirror is not ideal for indexfile as we may |
434 | // run into auth issues | |
435 | ||
436 | if (Debug) | |
437 | clog << "Failure to get " << Queue->Uri << endl; | |
438 | ||
439 | // try the next mirror on fail (if its not a expected failure, | |
440 | // e.g. translations are ok to ignore) | |
d3e8fbb3 | 441 | if (!Queue->FailIgnore && TryNextMirror()) |
483dfdd8 | 442 | return; |
483dfdd8 MV |
443 | |
444 | // all mirrors failed, so bail out | |
0ded3ad3 MV |
445 | string s; |
446 | strprintf(s, _("[Mirror: %s]"), Mirror.c_str()); | |
447 | SetIP(s); | |
448 | ||
03915427 | 449 | CurrentQueueUriToMirror(); |
14e097c1 MV |
450 | pkgAcqMethod::Fail(Err, Transient); |
451 | } | |
452 | ||
453 | void MirrorMethod::URIStart(FetchResult &Res) | |
454 | { | |
03915427 | 455 | CurrentQueueUriToMirror(); |
14e097c1 MV |
456 | pkgAcqMethod::URIStart(Res); |
457 | } | |
458 | ||
459 | void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt) | |
460 | { | |
03915427 | 461 | CurrentQueueUriToMirror(); |
14e097c1 MV |
462 | pkgAcqMethod::URIDone(Res, Alt); |
463 | } | |
464 | ||
465 | ||
5f6b130d MV |
466 | int main() |
467 | { | |
468 | setlocale(LC_ALL, ""); | |
469 | ||
470 | MirrorMethod Mth; | |
471 | ||
14e097c1 | 472 | return Mth.Loop(); |
5f6b130d MV |
473 | } |
474 | ||
475 |