X-Git-Url: https://git.saurik.com/apt.git/blobdiff_plain/a4221092e50af0b74040f5b4ee800c78b05fd84e..533fe3d13927798c17bdef84eba60ed9441d9608:/apt-pkg/acquire.cc diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index 1aa709381..5e5bec95c 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -23,6 +23,7 @@ #include #include +#include #include #include #include @@ -54,40 +55,62 @@ pkgAcquire::pkgAcquire() : LockFD(-1), Queues(0), Workers(0), Configs(0), Log(NU Debug(_config->FindB("Debug::pkgAcquire",false)), Running(false) { - string const Mode = _config->Find("Acquire::Queue-Mode","host"); - if (strcasecmp(Mode.c_str(),"host") == 0) - QueueMode = QueueHost; - if (strcasecmp(Mode.c_str(),"access") == 0) - QueueMode = QueueAccess; + Initialize(); } pkgAcquire::pkgAcquire(pkgAcquireStatus *Progress) : LockFD(-1), Queues(0), Workers(0), Configs(0), Log(NULL), ToFetch(0), Debug(_config->FindB("Debug::pkgAcquire",false)), Running(false) +{ + Initialize(); + SetLog(Progress); +} +void pkgAcquire::Initialize() { string const Mode = _config->Find("Acquire::Queue-Mode","host"); if (strcasecmp(Mode.c_str(),"host") == 0) QueueMode = QueueHost; if (strcasecmp(Mode.c_str(),"access") == 0) QueueMode = QueueAccess; - SetLog(Progress); + + // chown the auth.conf file as it will be accessed by our methods + std::string const SandboxUser = _config->Find("APT::Sandbox::User"); + if (getuid() == 0 && SandboxUser.empty() == false) // if we aren't root, we can't chown, so don't try it + { + struct passwd const * const pw = getpwnam(SandboxUser.c_str()); + struct group const * const gr = getgrnam("root"); + if (pw != NULL && gr != NULL) + { + std::string const AuthConf = _config->FindFile("Dir::Etc::netrc"); + if(AuthConf.empty() == false && RealFileExists(AuthConf) && + chown(AuthConf.c_str(), pw->pw_uid, gr->gr_gid) != 0) + _error->WarningE("SetupAPTPartialDirectory", "chown to %s:root of file %s failed", SandboxUser.c_str(), AuthConf.c_str()); + } + } } /*}}}*/ // Acquire::GetLock - lock directory and prepare for action /*{{{*/ static bool SetupAPTPartialDirectory(std::string const &grand, std::string const &parent) { std::string const partial = parent + "partial"; - if (CreateAPTDirectoryIfNeeded(grand, partial) == false && - CreateAPTDirectoryIfNeeded(parent, partial) == false) + mode_t const mode = umask(S_IWGRP | S_IWOTH); + bool const creation_fail = (CreateAPTDirectoryIfNeeded(grand, partial) == false && + CreateAPTDirectoryIfNeeded(parent, partial) == false); + umask(mode); + if (creation_fail == true) return false; - if (getuid() == 0) // if we aren't root, we can't chown, so don't try it + std::string const SandboxUser = _config->Find("APT::Sandbox::User"); + if (getuid() == 0 && SandboxUser.empty() == false) // if we aren't root, we can't chown, so don't try it { - std::string SandboxUser = _config->Find("APT::Sandbox::User"); - struct passwd *pw = getpwnam(SandboxUser.c_str()); - struct group *gr = getgrnam("root"); - if (pw != NULL && gr != NULL && chown(partial.c_str(), pw->pw_uid, gr->gr_gid) != 0) - _error->WarningE("SetupAPTPartialDirectory", "chown to %s:root of directory %s failed", SandboxUser.c_str(), partial.c_str()); + struct passwd const * const pw = getpwnam(SandboxUser.c_str()); + struct group const * const gr = getgrnam("root"); + if (pw != NULL && gr != NULL) + { + // chown the partial dir + if(chown(partial.c_str(), pw->pw_uid, gr->gr_gid) != 0) + _error->WarningE("SetupAPTPartialDirectory", "chown to %s:root of directory %s failed", SandboxUser.c_str(), partial.c_str()); + } } if (chmod(partial.c_str(), 0700) != 0) _error->WarningE("SetupAPTPartialDirectory", "chmod 0700 of directory %s failed", partial.c_str()); @@ -133,6 +156,8 @@ bool pkgAcquire::GetLock(std::string const &Lock) return true; // Lock the directory this acquire object will work in + if (LockFD != -1) + close(LockFD); LockFD = ::GetLock(flCombine(Lock, "lock")); if (LockFD == -1) return _error->Error(_("Unable to lock directory %s"), Lock.c_str()); @@ -655,10 +680,14 @@ bool pkgAcquire::Queue::Enqueue(ItemDesc &Item) { QItem **I = &Items; // move to the end of the queue and check for duplicates here + HashStringList const hsl = Item.Owner->GetExpectedHashes(); for (; *I != 0; I = &(*I)->Next) - if (Item.URI == (*I)->URI) + if (Item.URI == (*I)->URI || hsl == (*I)->Owner->GetExpectedHashes()) { - Item.Owner->Status = Item::StatDone; + if (_config->FindB("Debug::pkgAcquire::Worker",false) == true) + std::cerr << " @ Queue: Action combined for " << Item.URI << " and " << (*I)->URI << std::endl; + (*I)->Owners.push_back(Item.Owner); + Item.Owner->Status = (*I)->Owner->Status; return false; } @@ -681,13 +710,13 @@ bool pkgAcquire::Queue::Dequeue(Item *Owner) { if (Owner->Status == pkgAcquire::Item::StatFetching) return _error->Error("Tried to dequeue a fetching object"); - + bool Res = false; - + QItem **I = &Items; for (; *I != 0;) { - if ((*I)->Owner == Owner) + if (Owner == (*I)->Owner) { QItem *Jnk= *I; *I = (*I)->Next; @@ -698,7 +727,7 @@ bool pkgAcquire::Queue::Dequeue(Item *Owner) else I = &(*I)->Next; } - + return Res; } /*}}}*/ @@ -775,9 +804,12 @@ pkgAcquire::Queue::QItem *pkgAcquire::Queue::FindItem(string URI,pkgAcquire::Wor bool pkgAcquire::Queue::ItemDone(QItem *Itm) { PipeDepth--; - if (Itm->Owner->Status == pkgAcquire::Item::StatFetching) - Itm->Owner->Status = pkgAcquire::Item::StatDone; - + for (QItem::owner_iterator O = Itm->Owners.begin(); O != Itm->Owners.end(); ++O) + { + if ((*O)->Status == pkgAcquire::Item::StatFetching) + (*O)->Status = pkgAcquire::Item::StatDone; + } + if (Itm->Owner->QueueCounter <= 1) Owner->Dequeue(Itm->Owner); else @@ -785,7 +817,7 @@ bool pkgAcquire::Queue::ItemDone(QItem *Itm) Dequeue(Itm->Owner); Owner->Bump(); } - + return Cycle(); } /*}}}*/ @@ -800,7 +832,7 @@ bool pkgAcquire::Queue::Cycle() if (PipeDepth < 0) return _error->Error("Pipedepth failure"); - + // Look for a queable item QItem *I = Items; while (PipeDepth < (signed)MaxPipeDepth) @@ -808,18 +840,19 @@ bool pkgAcquire::Queue::Cycle() for (; I != 0; I = I->Next) if (I->Owner->Status == pkgAcquire::Item::StatIdle) break; - + // Nothing to do, queue is idle. if (I == 0) return true; - + I->Worker = Workers; - I->Owner->Status = pkgAcquire::Item::StatFetching; + for (QItem::owner_iterator O = I->Owners.begin(); O != I->Owners.end(); ++O) + (*O)->Status = pkgAcquire::Item::StatFetching; PipeDepth++; if (Workers->QueueItem(I) == false) return false; } - + return true; } /*}}}*/ @@ -831,10 +864,98 @@ void pkgAcquire::Queue::Bump() Cycle(); } /*}}}*/ +HashStringList pkgAcquire::Queue::QItem::GetExpectedHashes() const /*{{{*/ +{ + /* each Item can have multiple owners and each owner might have different + hashes, even if that is unlikely in practice and if so at least some + owners will later fail. There is one situation through which is not a + failure and still needs this handling: Two owners who expect the same + file, but one owner only knows the SHA1 while the other only knows SHA256. */ + HashStringList superhsl; + for (pkgAcquire::Queue::QItem::owner_iterator O = Owners.begin(); O != Owners.end(); ++O) + { + HashStringList const hsl = (*O)->GetExpectedHashes(); + if (hsl.usable() == false) + continue; + if (superhsl.usable() == false) + superhsl = hsl; + else + { + // we merge both lists - if we find disagreement send no hashes + HashStringList::const_iterator hs = hsl.begin(); + for (; hs != hsl.end(); ++hs) + if (superhsl.push_back(*hs) == false) + break; + if (hs != hsl.end()) + { + superhsl.clear(); + break; + } + } + } + return superhsl; +} + /*}}}*/ +APT_PURE unsigned long long pkgAcquire::Queue::QItem::GetMaximumSize() const /*{{{*/ +{ + unsigned long long Maximum = std::numeric_limits::max(); + for (pkgAcquire::Queue::QItem::owner_iterator O = Owners.begin(); O != Owners.end(); ++O) + { + if ((*O)->FileSize == 0) + continue; + Maximum = std::min(Maximum, (*O)->FileSize); + } + if (Maximum == std::numeric_limits::max()) + return 0; + return Maximum; +} + /*}}}*/ +void pkgAcquire::Queue::QItem::SyncDestinationFiles() const /*{{{*/ +{ + /* ensure that the first owner has the best partial file of all and + the rest have (potentially dangling) symlinks to it so that + everything (like progress reporting) finds it easily */ + std::string superfile = Owner->DestFile; + off_t supersize = 0; + for (pkgAcquire::Queue::QItem::owner_iterator O = Owners.begin(); O != Owners.end(); ++O) + { + if ((*O)->DestFile == superfile) + continue; + struct stat file; + if (lstat((*O)->DestFile.c_str(),&file) == 0) + { + if ((file.st_mode & S_IFREG) == 0) + unlink((*O)->DestFile.c_str()); + else if (supersize < file.st_size) + { + supersize = file.st_size; + unlink(superfile.c_str()); + rename((*O)->DestFile.c_str(), superfile.c_str()); + } + else + unlink((*O)->DestFile.c_str()); + if (symlink(superfile.c_str(), (*O)->DestFile.c_str()) != 0) + { + ; // not a problem per-se and no real alternative + } + } + } +} + /*}}}*/ +std::string pkgAcquire::Queue::QItem::Custom600Headers() const /*{{{*/ +{ + /* The others are relatively easy to merge, but this one? + Lets not merge and see how far we can run with it… + Likely, nobody will ever notice as all the items will + be of the same class and hence generate the same headers. */ + return Owner->Custom600Headers(); +} + /*}}}*/ + // AcquireStatus::pkgAcquireStatus - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ -pkgAcquireStatus::pkgAcquireStatus() : d(NULL), Percent(0), Update(true), MorePulses(false) +pkgAcquireStatus::pkgAcquireStatus() : d(NULL), Percent(-1), Update(true), MorePulses(false) { Start(); } @@ -890,9 +1011,9 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) { CurrentBytes += I->CurrentSize; ResumeSize += I->ResumePoint; - + // Files with unknown size always have 100% completion - if (I->CurrentItem->Owner->FileSize == 0 && + if (I->CurrentItem->Owner->FileSize == 0 && I->CurrentItem->Owner->Complete == false) TotalBytes += I->CurrentSize; } @@ -933,13 +1054,17 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) Time = NewTime; } + double const OldPercent = Percent; // calculate the percentage, if we have too little data assume 1% if (TotalBytes > 0 && UnfetchedReleaseFiles) Percent = 0; - else + else // use both files and bytes because bytes can be unreliable - Percent = (0.8 * (CurrentBytes/float(TotalBytes)*100.0) + + Percent = (0.8 * (CurrentBytes/float(TotalBytes)*100.0) + 0.2 * (CurrentItems/float(TotalItems)*100.0)); + double const DiffPercent = Percent - OldPercent; + if (DiffPercent < 0.001 && _config->FindB("Acquire::Progress::Diffpercent", false) == true) + return true; int fd = _config->FindI("APT::Status-Fd",-1); if(fd > 0) @@ -957,11 +1082,11 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) snprintf(msg,sizeof(msg), _("Retrieving file %li of %li (%s remaining)"), i, TotalItems, TimeToStr(ETA).c_str()); else snprintf(msg,sizeof(msg), _("Retrieving file %li of %li"), i, TotalItems); - + // build the status str status << "dlstatus:" << i << ":" << std::setprecision(3) << Percent - << ":" << msg + << ":" << msg << endl; std::string const dlstatus = status.str();