X-Git-Url: https://git.saurik.com/apt.git/blobdiff_plain/b86f642111954754dd9932ed2f28a9ea85035e87..ae54c5356b2d0d9776528d1eeb773c57f5b82ef4:/methods/mirror.cc diff --git a/methods/mirror.cc b/methods/mirror.cc index cfc155f58..7f28c04cb 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -8,6 +8,7 @@ ##################################################################### */ /*}}}*/ // Include Files /*{{{*/ +#include #include #include #include @@ -16,11 +17,15 @@ #include #include + +#include #include #include + #include #include #include +#include #include using namespace std; @@ -125,23 +130,90 @@ bool MirrorMethod::Clean(string Dir) bool MirrorMethod::DownloadMirrorFile(string mirror_uri_str) { - if(Debug) - clog << "MirrorMethod::DownloadMirrorFile(): " << endl; - // not that great to use pkgAcquire here, but we do not have // any other way right now string fetch = BaseUri; fetch.replace(0,strlen("mirror://"),"http://"); +#if 0 // no need for this, the getArchitectures() will also include the main + // arch + // append main architecture + fetch += "?arch=" + _config->Find("Apt::Architecture"); +#endif + + // append all architectures + std::vector vec = APT::Configuration::getArchitectures(); + for (std::vector::const_iterator I = vec.begin(); + I != vec.end(); I++) + if (I == vec.begin()) + fetch += "?arch" + (*I); + else + fetch += "&arch=" + (*I); + + // append the dist as a query string + if (Dist != "") + fetch += "&dist=" + Dist; + + if(Debug) + clog << "MirrorMethod::DownloadMirrorFile(): '" << fetch << "'" + << " to " << MirrorFile << endl; + pkgAcquire Fetcher; new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile); bool res = (Fetcher.Run() == pkgAcquire::Continue); - if(res) + if(res) { DownloadedMirrorFile = true; + chmod(MirrorFile.c_str(), 0644); + } Fetcher.Shutdown(); + + if(Debug) + clog << "MirrorMethod::DownloadMirrorFile() success: " << res << endl; + return res; } +// Randomizes the lines in the mirror file, this is used so that +// we spread the load on the mirrors evenly +bool MirrorMethod::RandomizeMirrorFile(string mirror_file) +{ + vector content; + string line; + + if (!FileExists(mirror_file)) + return false; + + // read + ifstream in(mirror_file.c_str()); + while ( !in.eof() ) { + getline(in, line); + content.push_back(line); + } + + // we want the file to be random for each different machine, but also + // "stable" on the same machine. this is to avoid running into out-of-sync + // issues (i.e. Release/Release.gpg different on each mirror) + struct utsname buf; + int seed=1, i; + if(uname(&buf) == 0) { + for(i=0,seed=1; buf.nodename[i] != 0; i++) { + seed = seed * 31 + buf.nodename[i]; + } + } + srand( seed ); + random_shuffle(content.begin(), content.end()); + + // write + ofstream out(mirror_file.c_str()); + while ( !content.empty()) { + line = content.back(); + content.pop_back(); + out << line << "\n"; + } + + return true; +} + /* convert a the Queue->Uri back to the mirror base uri and look * at all mirrors we have for this, this is needed as queue->uri * may point to different mirrors (if TryNextMirror() was run) @@ -153,33 +225,44 @@ void MirrorMethod::CurrentQueueUriToMirror() return; // find current mirror and select next one - for (int i=0; i < AllMirrors.size(); i++) + for (vector::const_iterator mirror = AllMirrors.begin(); + mirror != AllMirrors.end(); ++mirror) { - if (Queue->Uri.find(AllMirrors[i]) == 0) + if (Queue->Uri.find(*mirror) == 0) { - Queue->Uri.replace(0, AllMirrors[i].size(), BaseUri); + Queue->Uri.replace(0, mirror->length(), BaseUri); return; } } _error->Error("Internal error: Failed to convert %s back to %s", - Queue->Uri, BaseUri); + Queue->Uri.c_str(), BaseUri.c_str()); } bool MirrorMethod::TryNextMirror() { - if(Debug) - cerr << "using mirror: " << Mirror << endl; - // find current mirror and select next one - for (int i=0; i < AllMirrors.size()-1; i++) + for (vector::const_iterator mirror = AllMirrors.begin(); + mirror != AllMirrors.end(); ++mirror) { - if (Queue->Uri.find(AllMirrors[i]) == 0) - { - Queue->Uri.replace(0, AllMirrors[i].size(), AllMirrors[i+1]); - return true; - } + if (Queue->Uri.find(*mirror) != 0) + continue; + + vector::const_iterator nextmirror = mirror + 1; + if (nextmirror == AllMirrors.end()) + break; + Queue->Uri.replace(0, mirror->length(), *nextmirror); + if (Debug) + clog << "TryNextMirror: " << Queue->Uri << endl; + + // inform parent + UsedMirror = *nextmirror; + Log("Switching mirror"); + return true; } + if (Debug) + clog << "TryNextMirror could not find another mirror to try" << endl; + return false; } @@ -193,6 +276,13 @@ bool MirrorMethod::InitMirrors() return _error->Error(_("No mirror file '%s' found "), MirrorFile.c_str()); } + if (access(MirrorFile.c_str(), R_OK) != 0) + { + // FIXME: fallback to a default mirror here instead + // and provide a config option to define that default + return _error->Error(_("Can not read mirror file '%s'"), MirrorFile.c_str()); + } + // FIXME: make the mirror selection more clever, do not // just use the first one! // BUT: we can not make this random, the mirror has to be @@ -204,8 +294,18 @@ bool MirrorMethod::InitMirrors() while (!in.eof()) { getline(in, s); - if (s.size() > 0) - AllMirrors.push_back(s); + + // ignore lines that start with # + if (s.find("#") == 0) + continue; + // ignore empty lines + if (s.size() == 0) + continue; + // ignore non http lines + if (s.find("http://") != 0) + continue; + + AllMirrors.push_back(s); } Mirror = AllMirrors[0]; UsedMirror = Mirror; @@ -259,6 +359,7 @@ string MirrorMethod::GetMirrorFileName(string mirror_uri_str) if(Debug) std::cerr << "found BaseURI: " << uristr << std::endl; BaseUri = uristr.substr(0,uristr.size()-1); + Dist = (*I)->GetDist(); } } // get new file @@ -296,10 +397,11 @@ bool MirrorMethod::Fetch(FetchItem *Itm) if(Itm->IndexFile && !DownloadedMirrorFile) { Clean(_config->FindDir("Dir::State::mirrors")); - DownloadMirrorFile(Itm->Uri); + if (DownloadMirrorFile(Itm->Uri)) + RandomizeMirrorFile(MirrorFile); } - if(Mirror.empty()) { + if(AllMirrors.empty()) { if(!InitMirrors()) { // no valid mirror selected, something went wrong downloading // from the master mirror site most likely and there is @@ -307,15 +409,12 @@ bool MirrorMethod::Fetch(FetchItem *Itm) return false; } } - if(Debug) - clog << "selected mirror: " << Mirror << endl; + if(Itm->Uri.find("mirror://") != string::npos) + Itm->Uri.replace(0,BaseUri.size(), Mirror); - for (FetchItem *I = Queue; I != 0; I = I->Next) - { - if(I->Uri.find("mirror://") != string::npos) - I->Uri.replace(0,BaseUri.size(), Mirror); - } + if(Debug) + clog << "Fetch: " << Itm->Uri << endl << endl; // now run the real fetcher return HttpMethod::Fetch(Itm); @@ -323,8 +422,15 @@ bool MirrorMethod::Fetch(FetchItem *Itm) void MirrorMethod::Fail(string Err,bool Transient) { - // try the next mirror on fail - if (TryNextMirror()) + // FIXME: TryNextMirror is not ideal for indexfile as we may + // run into auth issues + + if (Debug) + clog << "Failure to get " << Queue->Uri << endl; + + // try the next mirror on fail (if its not a expected failure, + // e.g. translations are ok to ignore) + if (!Queue->FailIgnore && TryNextMirror()) return; // all mirrors failed, so bail out