X-Git-Url: https://git.saurik.com/apt.git/blobdiff_plain/70288656715d2fe4c2c33598124ae48f7bca9cdf..ea6db08d45e012d577f3081fa2070db9e034add9:/methods/mirror.cc?ds=sidebyside diff --git a/methods/mirror.cc b/methods/mirror.cc index 425a2f7f5..e8873d97b 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -24,35 +25,33 @@ using namespace std; +#include + #include "mirror.h" #include "http.h" #include "apti18n.h" /*}}}*/ -/* +/* Done: + * - works with http (only!) + * - always picks the first mirror from the list + * - call out to problem reporting script + * - supports "deb mirror://host/path/to/mirror-list/// dist component" + * - uses pkgAcqMethod::FailReason() to have a string representation + * of the failure that is also send to LP + * * TODO: - * - send expected checksum to the mirror method so that - some checking/falling back can be done here already - * - keep the mirror file around in /var/lib/apt/mirrors - * can't be put into lists/ because of the listclearer - * cleanup by time (mtime relative to the other mtimes) - * - use a TTL time the mirror file is fetched again (6h?) * - deal with runing as non-root because we can't write to the lists dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here - * - magicmarker is (a bit) evil + * and better error handling there! + * - support more than http * - testing :) */ MirrorMethod::MirrorMethod() - : HttpMethod(), HasMirrorFile(false) + : HttpMethod(), DownloadedMirrorFile(false) { -#if 0 - HasMirrorFile=true; - BaseUri="mirror://people.ubuntu.com/~mvo/mirror/mirrors"; - MirrorFile="/var/lib/apt/lists/people.ubuntu.com_%7emvo_apt_mirror_mirrors"; - Mirror="http://de.archive.ubuntu.com/ubuntu/"; -#endif }; // HttpMethod::Configuration - Handle a configuration message /*{{{*/ @@ -71,9 +70,17 @@ bool MirrorMethod::Configuration(string Message) // clean the mirrors dir based on ttl information bool MirrorMethod::Clean(string Dir) { - // FIXME: it would better to have a global idea of the mirrors - // in the sources.list and use this instead of this time - // based approach. currently apt does not support this :/ + vector::const_iterator I; + + if(Debug) + clog << "MirrorMethod::Clean(): " << Dir << endl; + + if(Dir == "/") + return _error->Error("will not clean: '/'"); + + // read sources.list + pkgSourceList list; + list.ReadMainList(); DIR *D = opendir(Dir.c_str()); if (D == 0) @@ -94,24 +101,20 @@ bool MirrorMethod::Clean(string Dir) strcmp(Dir->d_name,".") == 0 || strcmp(Dir->d_name,"..") == 0) continue; - - // Del everything not touched for MaxAge days - time_t t,now,max; - struct stat buf; - if(stat(Dir->d_name, &buf) != 0) + + // see if we have that uri + for(I=list.begin(); I != list.end(); I++) { - cerr << "Can't stat '" << Dir->d_name << "'" << endl; - continue; + string uri = (*I)->GetURI(); + if(uri.find("mirror://") != 0) + continue; + string BaseUri = uri.substr(0,uri.size()-1); + if (URItoFileName(BaseUri) == Dir->d_name) + break; } - t = std::max(buf.st_mtime, buf.st_ctime); - now = time(NULL); - max = 24*60*60*_config->FindI("Acquire::Mirror::MaxAge",90); - if(t + max < now) - { - if(Debug) - clog << "Mirror file is older than MaxAge days, deleting" << endl; + // nothing found, nuke it + if (I == list.end()) unlink(Dir->d_name); - } }; chdir(StartDir.c_str()); @@ -120,109 +123,239 @@ bool MirrorMethod::Clean(string Dir) } -bool MirrorMethod::GetMirrorFile(string uri) +bool MirrorMethod::DownloadMirrorFile(string mirror_uri_str) { - string Marker = _config->Find("Acquire::Mirror::MagicMarker","///"); - BaseUri = uri.substr(0,uri.find(Marker)); + if(Debug) + clog << "MirrorMethod::DownloadMirrorFile(): " << endl; + // not that great to use pkgAcquire here, but we do not have + // any other way right now string fetch = BaseUri; fetch.replace(0,strlen("mirror://"),"http://"); - // get new file - MirrorFile = _config->FindDir("Dir::State::mirrors") + URItoFileName(BaseUri); + pkgAcquire Fetcher; + new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile); + bool res = (Fetcher.Run() == pkgAcquire::Continue); + if(res) + DownloadedMirrorFile = true; + Fetcher.Shutdown(); + return res; +} - if(Debug) - { - cerr << "base-uri: " << BaseUri << endl; - cerr << "mirror-file: " << MirrorFile << endl; - } +/* convert a the Queue->Uri back to the mirror base uri and look + * at all mirrors we have for this, this is needed as queue->uri + * may point to different mirrors (if TryNextMirror() was run) + */ +void MirrorMethod::CurrentQueueUriToMirror() +{ + // already in mirror:// style so nothing to do + if(Queue->Uri.find("mirror://") == 0) + return; - // check the file, if it is not older than RefreshInterval just use it - // otherwise try to get a new one - if(FileExists(MirrorFile)) + // find current mirror and select next one + for (vector::const_iterator mirror = AllMirrors.begin(); + mirror != AllMirrors.end(); ++mirror) { - struct stat buf; - time_t t,now,refresh; - if(stat(MirrorFile.c_str(), &buf) != 0) - return false; - t = std::max(buf.st_mtime, buf.st_ctime); - now = time(NULL); - refresh = 60*_config->FindI("Acquire::Mirror::RefreshInterval",360); - if(t + refresh > now) + if (Queue->Uri.find(*mirror) == 0) { - if(Debug) - clog << "Mirror file is in RefreshInterval" << endl; - HasMirrorFile = true; - return true; + Queue->Uri.replace(0, mirror->length(), BaseUri); + return; } - if(Debug) - clog << "Mirror file " << MirrorFile << " older than " << refresh << "min, re-download it" << endl; } + _error->Error("Internal error: Failed to convert %s back to %s", + Queue->Uri.c_str(), BaseUri.c_str()); +} - // not that great to use pkgAcquire here, but we do not have - // any other way right now - pkgAcquire Fetcher; - new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile); - bool res = (Fetcher.Run() == pkgAcquire::Continue); - if(res) - HasMirrorFile = true; - Fetcher.Shutdown(); - return res; +bool MirrorMethod::TryNextMirror() +{ + // find current mirror and select next one + for (vector::const_iterator mirror = AllMirrors.begin(); + mirror != AllMirrors.end(); ++mirror) + { + if (Queue->Uri.find(*mirror) != 0) + continue; + + vector::const_iterator nextmirror = mirror + 1; + if (nextmirror != AllMirrors.end()) + break; + Queue->Uri.replace(0, mirror->length(), *nextmirror); + if (Debug) + clog << "TryNextMirror: " << Queue->Uri << endl; + return true; + } + + if (Debug) + clog << "TryNextMirror could not find another mirror to try" << endl; + + return false; } -bool MirrorMethod::SelectMirror() +bool MirrorMethod::InitMirrors() { + // if we do not have a MirrorFile, fallback + if(!FileExists(MirrorFile)) + { + // FIXME: fallback to a default mirror here instead + // and provide a config option to define that default + return _error->Error(_("No mirror file '%s' found "), MirrorFile.c_str()); + } + // FIXME: make the mirror selection more clever, do not // just use the first one! + // BUT: we can not make this random, the mirror has to be + // stable accross session, because otherwise we can + // get into sync issues (got indexfiles from mirror A, + // but packages from mirror B - one might be out of date etc) ifstream in(MirrorFile.c_str()); - getline(in, Mirror); - if(Debug) - cerr << "Using mirror: " << Mirror << endl; + string s; + while (!in.eof()) + { + getline(in, s); + if (s.size() > 0) + AllMirrors.push_back(s); + } + Mirror = AllMirrors[0]; + UsedMirror = Mirror; return true; } +string MirrorMethod::GetMirrorFileName(string mirror_uri_str) +{ + /* + - a mirror_uri_str looks like this: + mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors/dists/feisty/Release.gpg + + - the matching source.list entry + deb mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors feisty main + + - we actually want to go after: + http://people.ubuntu.com/~mvo/apt/mirror/mirrors + + And we need to save the BaseUri for later: + - mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors + + FIXME: what if we have two similar prefixes? + mirror://people.ubuntu.com/~mvo/mirror + mirror://people.ubuntu.com/~mvo/mirror2 + then mirror_uri_str looks like: + mirror://people.ubuntu.com/~mvo/apt/mirror/dists/feisty/Release.gpg + mirror://people.ubuntu.com/~mvo/apt/mirror2/dists/feisty/Release.gpg + we search sources.list and find: + mirror://people.ubuntu.com/~mvo/apt/mirror + in both cases! So we need to apply some domain knowledge here :( and + check for /dists/ or /Release.gpg as suffixes + */ + string name; + if(Debug) + std::cerr << "GetMirrorFileName: " << mirror_uri_str << std::endl; + + // read sources.list and find match + vector::const_iterator I; + pkgSourceList list; + list.ReadMainList(); + for(I=list.begin(); I != list.end(); I++) + { + string uristr = (*I)->GetURI(); + if(Debug) + std::cerr << "Checking: " << uristr << std::endl; + if(uristr.substr(0,strlen("mirror://")) != string("mirror://")) + continue; + // find matching uri in sources.list + if(mirror_uri_str.substr(0,uristr.size()) == uristr) + { + if(Debug) + std::cerr << "found BaseURI: " << uristr << std::endl; + BaseUri = uristr.substr(0,uristr.size()-1); + } + } + // get new file + name = _config->FindDir("Dir::State::mirrors") + URItoFileName(BaseUri); + + if(Debug) + { + cerr << "base-uri: " << BaseUri << endl; + cerr << "mirror-file: " << name << endl; + } + return name; +} + // MirrorMethod::Fetch - Fetch an item /*{{{*/ // --------------------------------------------------------------------- /* This adds an item to the pipeline. We keep the pipeline at a fixed depth. */ bool MirrorMethod::Fetch(FetchItem *Itm) { - // select mirror only once per session - if(!HasMirrorFile) + if(Debug) + clog << "MirrorMethod::Fetch()" << endl; + + // the http method uses Fetch(0) as a way to update the pipeline, + // just let it do its work in this case - Fetch() with a valid + // Itm will always run before the first Fetch(0) + if(Itm == NULL) + return HttpMethod::Fetch(Itm); + + // if we don't have the name of the mirror file on disk yet, + // calculate it now (can be derived from the uri) + if(MirrorFile.empty()) + MirrorFile = GetMirrorFileName(Itm->Uri); + + // download mirror file once (if we are after index files) + if(Itm->IndexFile && !DownloadedMirrorFile) { Clean(_config->FindDir("Dir::State::mirrors")); - GetMirrorFile(Itm->Uri); - SelectMirror(); + DownloadMirrorFile(Itm->Uri); } - for (FetchItem *I = Queue; I != 0; I = I->Next) - { - if(I->Uri.find("mirror://") != string::npos) - I->Uri.replace(0,BaseUri.size(),Mirror); + if(AllMirrors.empty()) { + if(!InitMirrors()) { + // no valid mirror selected, something went wrong downloading + // from the master mirror site most likely and there is + // no old mirror file availalbe + return false; + } } + if(Itm->Uri.find("mirror://") != string::npos) + Itm->Uri.replace(0,BaseUri.size(), Mirror); + + if(Debug) + clog << "Fetch: " << Itm->Uri << endl << endl; + // now run the real fetcher return HttpMethod::Fetch(Itm); }; void MirrorMethod::Fail(string Err,bool Transient) { - if(Queue->Uri.find("http://") != string::npos) - Queue->Uri.replace(0,Mirror.size(), BaseUri); + // FIXME: TryNextMirror is not ideal for indexfile as we may + // run into auth issues + + if (Debug) + clog << "Failure to get " << Queue->Uri << endl; + + // try the next mirror on fail (if its not a expected failure, + // e.g. translations are ok to ignore) + if (!Queue->FailIgnore && TryNextMirror()) + return; + + // all mirrors failed, so bail out + string s; + strprintf(s, _("[Mirror: %s]"), Mirror.c_str()); + SetIP(s); + + CurrentQueueUriToMirror(); pkgAcqMethod::Fail(Err, Transient); } void MirrorMethod::URIStart(FetchResult &Res) { - if(Queue->Uri.find("http://") != string::npos) - Queue->Uri.replace(0,Mirror.size(), BaseUri); + CurrentQueueUriToMirror(); pkgAcqMethod::URIStart(Res); } void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt) { - if(Queue->Uri.find("http://") != string::npos) - Queue->Uri.replace(0,Mirror.size(), BaseUri); + CurrentQueueUriToMirror(); pkgAcqMethod::URIDone(Res, Alt); }