X-Git-Url: https://git.saurik.com/apt.git/blobdiff_plain/c033d41504957ed04d64ec72e7f47dfac64218b5..a4221092e50af0b74040f5b4ee800c78b05fd84e:/apt-pkg/acquire.cc?ds=sidebyside diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index 06b0f11f8..1aa709381 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -5,14 +5,16 @@ Acquire - File Acquiration - The core element for the schedual system is the concept of a named + The core element for the schedule system is the concept of a named queue. Each queue is unique and each queue has a name derived from the - URI. The degree of paralization can be controled by how the queue + URI. The degree of paralization can be controlled by how the queue name is derived from the URI. ##################################################################### */ /*}}}*/ // Include Files /*{{{*/ +#include + #include #include #include @@ -21,15 +23,26 @@ #include #include -#include - +#include +#include #include #include -#include +#include +#include +#include +#include +#include +#include +#include #include #include +#include #include +#include +#include + +#include /*}}}*/ using namespace std; @@ -47,8 +60,8 @@ pkgAcquire::pkgAcquire() : LockFD(-1), Queues(0), Workers(0), Configs(0), Log(NU if (strcasecmp(Mode.c_str(),"access") == 0) QueueMode = QueueAccess; } -pkgAcquire::pkgAcquire(pkgAcquireStatus *Progress) : LockFD(-1), Queues(0), Workers(0), - Configs(0), Log(Progress), ToFetch(0), +pkgAcquire::pkgAcquire(pkgAcquireStatus *Progress) : LockFD(-1), Queues(0), Workers(0), + Configs(0), Log(NULL), ToFetch(0), Debug(_config->FindB("Debug::pkgAcquire",false)), Running(false) { @@ -57,36 +70,70 @@ pkgAcquire::pkgAcquire(pkgAcquireStatus *Progress) : LockFD(-1), Queues(0), Wor QueueMode = QueueHost; if (strcasecmp(Mode.c_str(),"access") == 0) QueueMode = QueueAccess; - Setup(Progress, ""); + SetLog(Progress); } /*}}}*/ -// Acquire::Setup - Delayed Constructor /*{{{*/ -// --------------------------------------------------------------------- -/* Do everything needed to be a complete Acquire object and report the - success (or failure) back so the user knows that something is wrong… */ +// Acquire::GetLock - lock directory and prepare for action /*{{{*/ +static bool SetupAPTPartialDirectory(std::string const &grand, std::string const &parent) +{ + std::string const partial = parent + "partial"; + if (CreateAPTDirectoryIfNeeded(grand, partial) == false && + CreateAPTDirectoryIfNeeded(parent, partial) == false) + return false; + + if (getuid() == 0) // if we aren't root, we can't chown, so don't try it + { + std::string SandboxUser = _config->Find("APT::Sandbox::User"); + struct passwd *pw = getpwnam(SandboxUser.c_str()); + struct group *gr = getgrnam("root"); + if (pw != NULL && gr != NULL && chown(partial.c_str(), pw->pw_uid, gr->gr_gid) != 0) + _error->WarningE("SetupAPTPartialDirectory", "chown to %s:root of directory %s failed", SandboxUser.c_str(), partial.c_str()); + } + if (chmod(partial.c_str(), 0700) != 0) + _error->WarningE("SetupAPTPartialDirectory", "chmod 0700 of directory %s failed", partial.c_str()); + + return true; +} bool pkgAcquire::Setup(pkgAcquireStatus *Progress, string const &Lock) { Log = Progress; + if (Lock.empty()) + { + string const listDir = _config->FindDir("Dir::State::lists"); + if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir) == false) + return _error->Errno("Acquire", _("List directory %spartial is missing."), listDir.c_str()); + string const archivesDir = _config->FindDir("Dir::Cache::Archives"); + if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir) == false) + return _error->Errno("Acquire", _("Archives directory %spartial is missing."), archivesDir.c_str()); + return true; + } + return GetLock(Lock); +} +bool pkgAcquire::GetLock(std::string const &Lock) +{ + if (Lock.empty() == true) + return false; // check for existence and possibly create auxiliary directories string const listDir = _config->FindDir("Dir::State::lists"); - string const partialListDir = listDir + "partial/"; string const archivesDir = _config->FindDir("Dir::Cache::Archives"); - string const partialArchivesDir = archivesDir + "partial/"; - - if (CreateAPTDirectoryIfNeeded(_config->FindDir("Dir::State"), partialListDir) == false && - CreateAPTDirectoryIfNeeded(listDir, partialListDir) == false) - return _error->Errno("Acquire", _("List directory %spartial is missing."), listDir.c_str()); - if (CreateAPTDirectoryIfNeeded(_config->FindDir("Dir::Cache"), partialArchivesDir) == false && - CreateAPTDirectoryIfNeeded(archivesDir, partialArchivesDir) == false) - return _error->Errno("Acquire", _("Archives directory %spartial is missing."), archivesDir.c_str()); + if (Lock == listDir) + { + if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir) == false) + return _error->Errno("Acquire", _("List directory %spartial is missing."), listDir.c_str()); + } + if (Lock == archivesDir) + { + if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir) == false) + return _error->Errno("Acquire", _("Archives directory %spartial is missing."), archivesDir.c_str()); + } - if (Lock.empty() == true || _config->FindB("Debug::NoLocking", false) == true) + if (_config->FindB("Debug::NoLocking", false) == true) return true; // Lock the directory this acquire object will work in - LockFD = GetLock(flCombine(Lock, "lock")); + LockFD = ::GetLock(flCombine(Lock, "lock")); if (LockFD == -1) return _error->Error(_("Unable to lock directory %s"), Lock.c_str()); @@ -116,7 +163,7 @@ pkgAcquire::~pkgAcquire() /* */ void pkgAcquire::Shutdown() { - while (Items.size() != 0) + while (Items.empty() == false) { if (Items[0]->Status == Item::StatFetching) Items[0]->Status = Item::StatError; @@ -155,7 +202,7 @@ void pkgAcquire::Remove(Item *Itm) I = Items.begin(); } else - I++; + ++I; } } /*}}}*/ @@ -173,7 +220,7 @@ void pkgAcquire::Add(Worker *Work) // --------------------------------------------------------------------- /* A worker has died. This can not be done while the select loop is running as it would require that RunFds could handling a changing list state and - it cant.. */ + it can't.. */ void pkgAcquire::Remove(Worker *Work) { if (Running == true) @@ -242,11 +289,19 @@ void pkgAcquire::Dequeue(Item *Itm) { Queue *I = Queues; bool Res = false; - for (; I != 0; I = I->Next) - Res |= I->Dequeue(Itm); - if (Debug == true) clog << "Dequeuing " << Itm->DestFile << endl; + + for (; I != 0; I = I->Next) + { + if (I->Dequeue(Itm)) + { + Res = true; + if (Debug == true) + clog << "Dequeued from " << I->Name << endl; + } + } + if (Res == true) ToFetch--; } @@ -267,9 +322,30 @@ string pkgAcquire::QueueName(string Uri,MethodConfig const *&Config) /* Single-Instance methods get exactly one queue per URI. This is also used for the Access queue method */ if (Config->SingleInstance == true || QueueMode == QueueAccess) - return U.Access; + return U.Access; + + string AccessSchema = U.Access + ':', + FullQueueName = AccessSchema + U.Host; + unsigned int Instances = 0, SchemaLength = AccessSchema.length(); + + Queue *I = Queues; + for (; I != 0; I = I->Next) { + // if the queue already exists, re-use it + if (I->Name == FullQueueName) + return FullQueueName; + + if (I->Name.compare(0, SchemaLength, AccessSchema) == 0) + Instances++; + } + + if (Debug) { + clog << "Found " << Instances << " instances of " << U.Access << endl; + } + + if (Instances >= (unsigned int)_config->FindI("Acquire::QueueHost::Limit",10)) + return U.Access; - return U.Access + ':' + U.Host; + return FullQueueName; } /*}}}*/ // Acquire::GetConfig - Fetch the configuration information /*{{{*/ @@ -411,7 +487,7 @@ pkgAcquire::RunResult pkgAcquire::Run(int PulseIntervall) I->Shutdown(false); // Shut down the items - for (ItemIterator I = Items.begin(); I != Items.end(); I++) + for (ItemIterator I = Items.begin(); I != Items.end(); ++I) (*I)->Finished(); if (_error->PendingError()) @@ -437,7 +513,7 @@ void pkgAcquire::Bump() pkgAcquire::Worker *pkgAcquire::WorkerStep(Worker *I) { return I->NextAcquire; -}; +} /*}}}*/ // Acquire::Clean - Cleans a directory /*{{{*/ // --------------------------------------------------------------------- @@ -445,6 +521,13 @@ pkgAcquire::Worker *pkgAcquire::WorkerStep(Worker *I) if it is part of the download set. */ bool pkgAcquire::Clean(string Dir) { + // non-existing directories are by definition clean… + if (DirectoryExists(Dir) == false) + return true; + + if(Dir == "/") + return _error->Error(_("Clean of %s is not supported"), Dir.c_str()); + DIR *D = opendir(Dir.c_str()); if (D == 0) return _error->Errno("opendir",_("Unable to read %s"),Dir.c_str()); @@ -467,7 +550,7 @@ bool pkgAcquire::Clean(string Dir) // Look in the get list ItemCIterator I = Items.begin(); - for (; I != Items.end(); I++) + for (; I != Items.end(); ++I) if (flNotDir((*I)->DestFile) == Dir->d_name) break; @@ -485,10 +568,10 @@ bool pkgAcquire::Clean(string Dir) // Acquire::TotalNeeded - Number of bytes to fetch /*{{{*/ // --------------------------------------------------------------------- /* This is the total number of bytes needed */ -unsigned long long pkgAcquire::TotalNeeded() +APT_PURE unsigned long long pkgAcquire::TotalNeeded() { unsigned long long Total = 0; - for (ItemCIterator I = ItemsBegin(); I != ItemsEnd(); I++) + for (ItemCIterator I = ItemsBegin(); I != ItemsEnd(); ++I) Total += (*I)->FileSize; return Total; } @@ -496,10 +579,10 @@ unsigned long long pkgAcquire::TotalNeeded() // Acquire::FetchNeeded - Number of bytes needed to get /*{{{*/ // --------------------------------------------------------------------- /* This is the number of bytes that is not local */ -unsigned long long pkgAcquire::FetchNeeded() +APT_PURE unsigned long long pkgAcquire::FetchNeeded() { unsigned long long Total = 0; - for (ItemCIterator I = ItemsBegin(); I != ItemsEnd(); I++) + for (ItemCIterator I = ItemsBegin(); I != ItemsEnd(); ++I) if ((*I)->Local == false) Total += (*I)->FileSize; return Total; @@ -508,10 +591,10 @@ unsigned long long pkgAcquire::FetchNeeded() // Acquire::PartialPresent - Number of partial bytes we already have /*{{{*/ // --------------------------------------------------------------------- /* This is the number of bytes that is not local */ -unsigned long long pkgAcquire::PartialPresent() +APT_PURE unsigned long long pkgAcquire::PartialPresent() { unsigned long long Total = 0; - for (ItemCIterator I = ItemsBegin(); I != ItemsEnd(); I++) + for (ItemCIterator I = ItemsBegin(); I != ItemsEnd(); ++I) if ((*I)->Local == false) Total += (*I)->PartialSize; return Total; @@ -536,27 +619,18 @@ pkgAcquire::UriIterator pkgAcquire::UriEnd() // Acquire::MethodConfig::MethodConfig - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ -pkgAcquire::MethodConfig::MethodConfig() +pkgAcquire::MethodConfig::MethodConfig() : d(NULL), Next(0), SingleInstance(false), + Pipeline(false), SendConfig(false), LocalOnly(false), NeedsCleanup(false), + Removable(false) { - SingleInstance = false; - Pipeline = false; - SendConfig = false; - LocalOnly = false; - Removable = false; - Next = 0; } /*}}}*/ // Queue::Queue - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ -pkgAcquire::Queue::Queue(string Name,pkgAcquire *Owner) : Name(Name), - Owner(Owner) +pkgAcquire::Queue::Queue(string Name,pkgAcquire *Owner) : d(NULL), Next(0), + Name(Name), Items(0), Workers(0), Owner(Owner), PipeDepth(0), MaxPipeDepth(1) { - Items = 0; - Next = 0; - Workers = 0; - MaxPipeDepth = 1; - PipeDepth = 0; } /*}}}*/ // Queue::~Queue - Destructor /*{{{*/ @@ -760,7 +834,7 @@ void pkgAcquire::Queue::Bump() // AcquireStatus::pkgAcquireStatus - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ -pkgAcquireStatus::pkgAcquireStatus() : Update(true), MorePulses(false) +pkgAcquireStatus::pkgAcquireStatus() : d(NULL), Percent(0), Update(true), MorePulses(false) { Start(); } @@ -780,28 +854,38 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) // Compute the total number of bytes to fetch unsigned int Unknown = 0; unsigned int Count = 0; - for (pkgAcquire::ItemCIterator I = Owner->ItemsBegin(); I != Owner->ItemsEnd(); - I++, Count++) + bool UnfetchedReleaseFiles = false; + for (pkgAcquire::ItemCIterator I = Owner->ItemsBegin(); + I != Owner->ItemsEnd(); + ++I, ++Count) { TotalItems++; if ((*I)->Status == pkgAcquire::Item::StatDone) - CurrentItems++; + ++CurrentItems; // Totally ignore local items if ((*I)->Local == true) continue; + // see if the method tells us to expect more + TotalItems += (*I)->ExpectedAdditionalItems; + + // check if there are unfetched Release files + if ((*I)->Complete == false && (*I)->ExpectedAdditionalItems > 0) + UnfetchedReleaseFiles = true; + TotalBytes += (*I)->FileSize; if ((*I)->Complete == true) CurrentBytes += (*I)->FileSize; if ((*I)->FileSize == 0 && (*I)->Complete == false) - Unknown++; + ++Unknown; } // Compute the current completion unsigned long long ResumeSize = 0; for (pkgAcquire::Worker *I = Owner->WorkersBegin(); I != 0; I = Owner->WorkerStep(I)) + { if (I->CurrentItem != 0 && I->CurrentItem->Owner->Complete == false) { CurrentBytes += I->CurrentSize; @@ -812,6 +896,7 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) I->CurrentItem->Owner->Complete == false) TotalBytes += I->CurrentSize; } + } // Normalize the figures and account for unknown size downloads if (TotalBytes <= 0) @@ -822,6 +907,12 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) // Wha?! Is not supposed to happen. if (CurrentBytes > TotalBytes) CurrentBytes = TotalBytes; + + // debug + if (_config->FindB("Debug::acquire::progress", false) == true) + std::clog << " Bytes: " + << SizeToStr(CurrentBytes) << " / " << SizeToStr(TotalBytes) + << std::endl; // Compute the CPS struct timeval NewTime; @@ -842,6 +933,14 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) Time = NewTime; } + // calculate the percentage, if we have too little data assume 1% + if (TotalBytes > 0 && UnfetchedReleaseFiles) + Percent = 0; + else + // use both files and bytes because bytes can be unreliable + Percent = (0.8 * (CurrentBytes/float(TotalBytes)*100.0) + + 0.2 * (CurrentItems/float(TotalItems)*100.0)); + int fd = _config->FindI("APT::Status-Fd",-1); if(fd > 0) { @@ -859,14 +958,14 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) else snprintf(msg,sizeof(msg), _("Retrieving file %li of %li"), i, TotalItems); - - // build the status str status << "dlstatus:" << i - << ":" << (CurrentBytes/float(TotalBytes)*100.0) - << ":" << msg - << endl; - write(fd, status.str().c_str(), status.str().size()); + << ":" << std::setprecision(3) << Percent + << ":" << msg + << endl; + + std::string const dlstatus = status.str(); + FileFd::Write(fd, dlstatus.c_str(), dlstatus.size()); } return true; @@ -918,3 +1017,7 @@ void pkgAcquireStatus::Fetched(unsigned long long Size,unsigned long long Resume FetchedBytes += Size - Resume; } /*}}}*/ + +APT_CONST pkgAcquire::UriIterator::~UriIterator() {} +APT_CONST pkgAcquire::MethodConfig::~MethodConfig() {} +APT_CONST pkgAcquireStatus::~pkgAcquireStatus() {}