X-Git-Url: https://git.saurik.com/apt.git/blobdiff_plain/1cd1c398d18b78f4aa9d882a5de5385f4538e0be..54298f49d71347616df19b8d2f59c907374e07b3:/apt-pkg/acquire.cc diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index d83d80fac..e8fa68338 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -5,14 +5,16 @@ Acquire - File Acquiration - The core element for the schedual system is the concept of a named + The core element for the schedule system is the concept of a named queue. Each queue is unique and each queue has a name derived from the - URI. The degree of paralization can be controled by how the queue + URI. The degree of paralization can be controlled by how the queue name is derived from the URI. ##################################################################### */ /*}}}*/ // Include Files /*{{{*/ +#include + #include #include #include @@ -21,8 +23,6 @@ #include #include -#include - #include #include #include @@ -30,6 +30,8 @@ #include #include #include + +#include /*}}}*/ using namespace std; @@ -37,9 +39,9 @@ using namespace std; // Acquire::pkgAcquire - Constructor /*{{{*/ // --------------------------------------------------------------------- /* We grab some runtime state from the configuration space */ -pkgAcquire::pkgAcquire() : Queues(0), Workers(0), Configs(0), Log(NULL), ToFetch(0), +pkgAcquire::pkgAcquire() : LockFD(-1), Queues(0), Workers(0), Configs(0), Log(NULL), ToFetch(0), Debug(_config->FindB("Debug::pkgAcquire",false)), - Running(false), LockFD(-1) + Running(false) { string const Mode = _config->Find("Acquire::Queue-Mode","host"); if (strcasecmp(Mode.c_str(),"host") == 0) @@ -47,10 +49,10 @@ pkgAcquire::pkgAcquire() : Queues(0), Workers(0), Configs(0), Log(NULL), ToFetch if (strcasecmp(Mode.c_str(),"access") == 0) QueueMode = QueueAccess; } -pkgAcquire::pkgAcquire(pkgAcquireStatus *Progress) : Queues(0), Workers(0), +pkgAcquire::pkgAcquire(pkgAcquireStatus *Progress) : LockFD(-1), Queues(0), Workers(0), Configs(0), Log(Progress), ToFetch(0), Debug(_config->FindB("Debug::pkgAcquire",false)), - Running(false), LockFD(-1) + Running(false) { string const Mode = _config->Find("Acquire::Queue-Mode","host"); if (strcasecmp(Mode.c_str(),"host") == 0) @@ -69,9 +71,18 @@ bool pkgAcquire::Setup(pkgAcquireStatus *Progress, string const &Lock) Log = Progress; // check for existence and possibly create auxiliary directories - if (CheckDirectory(_config->FindDir("Dir::State"), _config->FindDir("Dir::State::lists") + "partial/") == false || - CheckDirectory(_config->FindDir("Dir::Cache"), _config->FindDir("Dir::Cache::Archives") + "partial/") == false) - return false; + string const listDir = _config->FindDir("Dir::State::lists"); + string const partialListDir = listDir + "partial/"; + string const archivesDir = _config->FindDir("Dir::Cache::Archives"); + string const partialArchivesDir = archivesDir + "partial/"; + + if (CreateAPTDirectoryIfNeeded(_config->FindDir("Dir::State"), partialListDir) == false && + CreateAPTDirectoryIfNeeded(listDir, partialListDir) == false) + return _error->Errno("Acquire", _("List directory %spartial is missing."), listDir.c_str()); + + if (CreateAPTDirectoryIfNeeded(_config->FindDir("Dir::Cache"), partialArchivesDir) == false && + CreateAPTDirectoryIfNeeded(archivesDir, partialArchivesDir) == false) + return _error->Errno("Acquire", _("Archives directory %spartial is missing."), archivesDir.c_str()); if (Lock.empty() == true || _config->FindB("Debug::NoLocking", false) == true) return true; @@ -84,27 +95,6 @@ bool pkgAcquire::Setup(pkgAcquireStatus *Progress, string const &Lock) return true; } /*}}}*/ -// Acquire::CheckDirectory - ensure that the given directory exists /*{{{*/ -// --------------------------------------------------------------------- -/* a small wrapper around CreateDirectory to check if it exists and to - remove the trailing "/apt/" from the parent directory if needed */ -bool pkgAcquire::CheckDirectory(string const &Parent, string const &Path) const -{ - if (DirectoryExists(Path) == true) - return true; - - size_t const len = Parent.size(); - if (len > 5 && Parent.find("/apt/", len - 6, 5) != len - 5) - { - if (CreateDirectory(Parent.substr(0,len-5), Path) == true) - return true; - } - else if (CreateDirectory(Parent, Path) == true) - return true; - - return _error->Errno("Acquire", _("Directory %s can't be created."), Path.c_str()); -} - /*}}}*/ // Acquire::~pkgAcquire - Destructor /*{{{*/ // --------------------------------------------------------------------- /* Free our memory, clean up the queues (destroy the workers) */ @@ -128,7 +118,7 @@ pkgAcquire::~pkgAcquire() /* */ void pkgAcquire::Shutdown() { - while (Items.size() != 0) + while (Items.empty() == false) { if (Items[0]->Status == Item::StatFetching) Items[0]->Status = Item::StatError; @@ -167,7 +157,7 @@ void pkgAcquire::Remove(Item *Itm) I = Items.begin(); } else - I++; + ++I; } } /*}}}*/ @@ -185,7 +175,7 @@ void pkgAcquire::Add(Worker *Work) // --------------------------------------------------------------------- /* A worker has died. This can not be done while the select loop is running as it would require that RunFds could handling a changing list state and - it cant.. */ + it can't.. */ void pkgAcquire::Remove(Worker *Work) { if (Running == true) @@ -254,11 +244,19 @@ void pkgAcquire::Dequeue(Item *Itm) { Queue *I = Queues; bool Res = false; - for (; I != 0; I = I->Next) - Res |= I->Dequeue(Itm); - if (Debug == true) clog << "Dequeuing " << Itm->DestFile << endl; + + for (; I != 0; I = I->Next) + { + if (I->Dequeue(Itm)) + { + Res = true; + if (Debug == true) + clog << "Dequeued from " << I->Name << endl; + } + } + if (Res == true) ToFetch--; } @@ -279,9 +277,30 @@ string pkgAcquire::QueueName(string Uri,MethodConfig const *&Config) /* Single-Instance methods get exactly one queue per URI. This is also used for the Access queue method */ if (Config->SingleInstance == true || QueueMode == QueueAccess) - return U.Access; + return U.Access; + + string AccessSchema = U.Access + ':', + FullQueueName = AccessSchema + U.Host; + unsigned int Instances = 0, SchemaLength = AccessSchema.length(); + + Queue *I = Queues; + for (; I != 0; I = I->Next) { + // if the queue already exists, re-use it + if (I->Name == FullQueueName) + return FullQueueName; - return U.Access + ':' + U.Host; + if (I->Name.compare(0, SchemaLength, AccessSchema) == 0) + Instances++; + } + + if (Debug) { + clog << "Found " << Instances << " instances of " << U.Access << endl; + } + + if (Instances >= (unsigned int)_config->FindI("Acquire::QueueHost::Limit",10)) + return U.Access; + + return FullQueueName; } /*}}}*/ // Acquire::GetConfig - Fetch the configuration information /*{{{*/ @@ -423,7 +442,7 @@ pkgAcquire::RunResult pkgAcquire::Run(int PulseIntervall) I->Shutdown(false); // Shut down the items - for (ItemIterator I = Items.begin(); I != Items.end(); I++) + for (ItemIterator I = Items.begin(); I != Items.end(); ++I) (*I)->Finished(); if (_error->PendingError()) @@ -449,7 +468,7 @@ void pkgAcquire::Bump() pkgAcquire::Worker *pkgAcquire::WorkerStep(Worker *I) { return I->NextAcquire; -}; +} /*}}}*/ // Acquire::Clean - Cleans a directory /*{{{*/ // --------------------------------------------------------------------- @@ -457,6 +476,10 @@ pkgAcquire::Worker *pkgAcquire::WorkerStep(Worker *I) if it is part of the download set. */ bool pkgAcquire::Clean(string Dir) { + // non-existing directories are by definition clean… + if (DirectoryExists(Dir) == false) + return true; + DIR *D = opendir(Dir.c_str()); if (D == 0) return _error->Errno("opendir",_("Unable to read %s"),Dir.c_str()); @@ -479,7 +502,7 @@ bool pkgAcquire::Clean(string Dir) // Look in the get list ItemCIterator I = Items.begin(); - for (; I != Items.end(); I++) + for (; I != Items.end(); ++I) if (flNotDir((*I)->DestFile) == Dir->d_name) break; @@ -497,10 +520,10 @@ bool pkgAcquire::Clean(string Dir) // Acquire::TotalNeeded - Number of bytes to fetch /*{{{*/ // --------------------------------------------------------------------- /* This is the total number of bytes needed */ -double pkgAcquire::TotalNeeded() +unsigned long long pkgAcquire::TotalNeeded() { - double Total = 0; - for (ItemCIterator I = ItemsBegin(); I != ItemsEnd(); I++) + unsigned long long Total = 0; + for (ItemCIterator I = ItemsBegin(); I != ItemsEnd(); ++I) Total += (*I)->FileSize; return Total; } @@ -508,10 +531,10 @@ double pkgAcquire::TotalNeeded() // Acquire::FetchNeeded - Number of bytes needed to get /*{{{*/ // --------------------------------------------------------------------- /* This is the number of bytes that is not local */ -double pkgAcquire::FetchNeeded() +unsigned long long pkgAcquire::FetchNeeded() { - double Total = 0; - for (ItemCIterator I = ItemsBegin(); I != ItemsEnd(); I++) + unsigned long long Total = 0; + for (ItemCIterator I = ItemsBegin(); I != ItemsEnd(); ++I) if ((*I)->Local == false) Total += (*I)->FileSize; return Total; @@ -520,10 +543,10 @@ double pkgAcquire::FetchNeeded() // Acquire::PartialPresent - Number of partial bytes we already have /*{{{*/ // --------------------------------------------------------------------- /* This is the number of bytes that is not local */ -double pkgAcquire::PartialPresent() +unsigned long long pkgAcquire::PartialPresent() { - double Total = 0; - for (ItemCIterator I = ItemsBegin(); I != ItemsEnd(); I++) + unsigned long long Total = 0; + for (ItemCIterator I = ItemsBegin(); I != ItemsEnd(); ++I) if ((*I)->Local == false) Total += (*I)->PartialSize; return Total; @@ -772,7 +795,7 @@ void pkgAcquire::Queue::Bump() // AcquireStatus::pkgAcquireStatus - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ -pkgAcquireStatus::pkgAcquireStatus() : Update(true), MorePulses(false) +pkgAcquireStatus::pkgAcquireStatus() : d(NULL), Update(true), MorePulses(false) { Start(); } @@ -793,11 +816,11 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) unsigned int Unknown = 0; unsigned int Count = 0; for (pkgAcquire::ItemCIterator I = Owner->ItemsBegin(); I != Owner->ItemsEnd(); - I++, Count++) + ++I, ++Count) { TotalItems++; if ((*I)->Status == pkgAcquire::Item::StatDone) - CurrentItems++; + ++CurrentItems; // Totally ignore local items if ((*I)->Local == true) @@ -807,11 +830,11 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) if ((*I)->Complete == true) CurrentBytes += (*I)->FileSize; if ((*I)->FileSize == 0 && (*I)->Complete == false) - Unknown++; + ++Unknown; } // Compute the current completion - unsigned long ResumeSize = 0; + unsigned long long ResumeSize = 0; for (pkgAcquire::Worker *I = Owner->WorkersBegin(); I != 0; I = Owner->WorkerStep(I)) if (I->CurrentItem != 0 && I->CurrentItem->Owner->Complete == false) @@ -850,7 +873,7 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) else CurrentCPS = ((CurrentBytes - ResumeSize) - LastBytes)/Delta; LastBytes = CurrentBytes - ResumeSize; - ElapsedTime = (unsigned long)Delta; + ElapsedTime = (unsigned long long)Delta; Time = NewTime; } @@ -861,8 +884,9 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) char msg[200]; long i = CurrentItems < TotalItems ? CurrentItems + 1 : CurrentItems; - unsigned long ETA = - (unsigned long)((TotalBytes - CurrentBytes) / CurrentCPS); + unsigned long long ETA = 0; + if(CurrentCPS > 0) + ETA = (TotalBytes - CurrentBytes) / CurrentCPS; // only show the ETA if it makes sense if (ETA > 0 && ETA < 172800 /* two days */ ) @@ -877,7 +901,9 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) << ":" << (CurrentBytes/float(TotalBytes)*100.0) << ":" << msg << endl; - write(fd, status.str().c_str(), status.str().size()); + + std::string const dlstatus = status.str(); + FileFd::Write(fd, dlstatus.c_str(), dlstatus.size()); } return true; @@ -918,13 +944,13 @@ void pkgAcquireStatus::Stop() else CurrentCPS = FetchedBytes/Delta; LastBytes = CurrentBytes; - ElapsedTime = (unsigned int)Delta; + ElapsedTime = (unsigned long long)Delta; } /*}}}*/ // AcquireStatus::Fetched - Called when a byte set has been fetched /*{{{*/ // --------------------------------------------------------------------- /* This is used to get accurate final transfer rate reporting. */ -void pkgAcquireStatus::Fetched(unsigned long Size,unsigned long Resume) +void pkgAcquireStatus::Fetched(unsigned long long Size,unsigned long long Resume) { FetchedBytes += Size - Resume; }