X-Git-Url: https://git.saurik.com/apt.git/blobdiff_plain/c6e9cc582c7093b08c7c057c1f7885eb07e06959..2e2865ae53a65c00dd55a892d5b48458f3110366:/apt-pkg/acquire.cc diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index 57cbba169..affc0c791 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -23,20 +23,27 @@ #include #include +#include +#include #include #include #include #include +#include +#include + #include #include #include #include -#include - +#include +#include +#include #include #include #include #include +#include #include /*}}}*/ @@ -46,56 +53,114 @@ using namespace std; // Acquire::pkgAcquire - Constructor /*{{{*/ // --------------------------------------------------------------------- /* We grab some runtime state from the configuration space */ -pkgAcquire::pkgAcquire() : LockFD(-1), Queues(0), Workers(0), Configs(0), Log(NULL), ToFetch(0), +pkgAcquire::pkgAcquire() : LockFD(-1), d(NULL), Queues(0), Workers(0), Configs(0), Log(NULL), ToFetch(0), Debug(_config->FindB("Debug::pkgAcquire",false)), Running(false) { - string const Mode = _config->Find("Acquire::Queue-Mode","host"); - if (strcasecmp(Mode.c_str(),"host") == 0) - QueueMode = QueueHost; - if (strcasecmp(Mode.c_str(),"access") == 0) - QueueMode = QueueAccess; + Initialize(); } -pkgAcquire::pkgAcquire(pkgAcquireStatus *Progress) : LockFD(-1), Queues(0), Workers(0), - Configs(0), Log(Progress), ToFetch(0), +pkgAcquire::pkgAcquire(pkgAcquireStatus *Progress) : LockFD(-1), d(NULL), Queues(0), Workers(0), + Configs(0), Log(NULL), ToFetch(0), Debug(_config->FindB("Debug::pkgAcquire",false)), Running(false) +{ + Initialize(); + SetLog(Progress); +} +void pkgAcquire::Initialize() { string const Mode = _config->Find("Acquire::Queue-Mode","host"); if (strcasecmp(Mode.c_str(),"host") == 0) QueueMode = QueueHost; if (strcasecmp(Mode.c_str(),"access") == 0) QueueMode = QueueAccess; - Setup(Progress, ""); + + // chown the auth.conf file as it will be accessed by our methods + std::string const SandboxUser = _config->Find("APT::Sandbox::User"); + if (getuid() == 0 && SandboxUser.empty() == false && SandboxUser != "root") // if we aren't root, we can't chown, so don't try it + { + struct passwd const * const pw = getpwnam(SandboxUser.c_str()); + struct group const * const gr = getgrnam("root"); + if (pw != NULL && gr != NULL) + { + std::string const AuthConf = _config->FindFile("Dir::Etc::netrc"); + if(AuthConf.empty() == false && RealFileExists(AuthConf) && + chown(AuthConf.c_str(), pw->pw_uid, gr->gr_gid) != 0) + _error->WarningE("SetupAPTPartialDirectory", "chown to %s:root of file %s failed", SandboxUser.c_str(), AuthConf.c_str()); + } + } } /*}}}*/ -// Acquire::Setup - Delayed Constructor /*{{{*/ -// --------------------------------------------------------------------- -/* Do everything needed to be a complete Acquire object and report the - success (or failure) back so the user knows that something is wrong… */ +// Acquire::GetLock - lock directory and prepare for action /*{{{*/ +static bool SetupAPTPartialDirectory(std::string const &grand, std::string const &parent) +{ + std::string const partial = parent + "partial"; + mode_t const mode = umask(S_IWGRP | S_IWOTH); + bool const creation_fail = (CreateAPTDirectoryIfNeeded(grand, partial) == false && + CreateAPTDirectoryIfNeeded(parent, partial) == false); + umask(mode); + if (creation_fail == true) + return false; + + std::string const SandboxUser = _config->Find("APT::Sandbox::User"); + if (getuid() == 0 && SandboxUser.empty() == false && SandboxUser != "root") // if we aren't root, we can't chown, so don't try it + { + struct passwd const * const pw = getpwnam(SandboxUser.c_str()); + struct group const * const gr = getgrnam("root"); + if (pw != NULL && gr != NULL) + { + // chown the partial dir + if(chown(partial.c_str(), pw->pw_uid, gr->gr_gid) != 0) + _error->WarningE("SetupAPTPartialDirectory", "chown to %s:root of directory %s failed", SandboxUser.c_str(), partial.c_str()); + } + } + if (chmod(partial.c_str(), 0700) != 0) + _error->WarningE("SetupAPTPartialDirectory", "chmod 0700 of directory %s failed", partial.c_str()); + + return true; +} bool pkgAcquire::Setup(pkgAcquireStatus *Progress, string const &Lock) { Log = Progress; + if (Lock.empty()) + { + string const listDir = _config->FindDir("Dir::State::lists"); + if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir) == false) + return _error->Errno("Acquire", _("List directory %spartial is missing."), listDir.c_str()); + string const archivesDir = _config->FindDir("Dir::Cache::Archives"); + if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir) == false) + return _error->Errno("Acquire", _("Archives directory %spartial is missing."), archivesDir.c_str()); + return true; + } + return GetLock(Lock); +} +bool pkgAcquire::GetLock(std::string const &Lock) +{ + if (Lock.empty() == true) + return false; // check for existence and possibly create auxiliary directories string const listDir = _config->FindDir("Dir::State::lists"); - string const partialListDir = listDir + "partial/"; string const archivesDir = _config->FindDir("Dir::Cache::Archives"); - string const partialArchivesDir = archivesDir + "partial/"; - if (CreateAPTDirectoryIfNeeded(_config->FindDir("Dir::State"), partialListDir) == false && - CreateAPTDirectoryIfNeeded(listDir, partialListDir) == false) - return _error->Errno("Acquire", _("List directory %spartial is missing."), listDir.c_str()); - - if (CreateAPTDirectoryIfNeeded(_config->FindDir("Dir::Cache"), partialArchivesDir) == false && - CreateAPTDirectoryIfNeeded(archivesDir, partialArchivesDir) == false) - return _error->Errno("Acquire", _("Archives directory %spartial is missing."), archivesDir.c_str()); + if (Lock == listDir) + { + if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir) == false) + return _error->Errno("Acquire", _("List directory %spartial is missing."), listDir.c_str()); + } + if (Lock == archivesDir) + { + if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir) == false) + return _error->Errno("Acquire", _("Archives directory %spartial is missing."), archivesDir.c_str()); + } - if (Lock.empty() == true || _config->FindB("Debug::NoLocking", false) == true) + if (_config->FindB("Debug::NoLocking", false) == true) return true; // Lock the directory this acquire object will work in - LockFD = GetLock(flCombine(Lock, "lock")); + if (LockFD != -1) + close(LockFD); + LockFD = ::GetLock(flCombine(Lock, "lock")); if (LockFD == -1) return _error->Error(_("Unable to lock directory %s"), Lock.c_str()); @@ -204,6 +269,42 @@ void pkgAcquire::Remove(Worker *Work) it is constructed which creates a queue (based on the current queue mode) and puts the item in that queue. If the system is running then the queue might be started. */ +static bool DoesAcquireResultInInstantFailure(pkgAcquire::Item * const Item, + pkgAcquire::MethodConfig const * const Config, pkgAcquireStatus * const Log) +{ + auto SavedDesc = Item->GetItemDesc(); + if (Item->IsRedirectionLoop(SavedDesc.URI)) + { + std::string const Message = "400 URI Failure" + "\nURI: " + SavedDesc.URI + + "\nFilename: " + Item->DestFile + + "\nFailReason: RedirectionLoop"; + + Item->Status = pkgAcquire::Item::StatError; + Item->Failed(Message, Config); + if (Log != nullptr) + Log->Fail(SavedDesc); + return true; + } + + HashStringList const hsl = Item->GetExpectedHashes(); + if (hsl.usable() == false && Item->HashesRequired() && + _config->Exists("Acquire::ForceHash") == false) + { + std::string const Message = "400 URI Failure" + "\nURI: " + SavedDesc.URI + + "\nFilename: " + Item->DestFile + + "\nFailReason: WeakHashSums"; + + auto SavedDesc = Item->GetItemDesc(); + Item->Status = pkgAcquire::Item::StatAuthError; + Item->Failed(Message, Config); + if (Log != nullptr) + Log->Fail(SavedDesc); + return true; + } + return false; +} void pkgAcquire::Enqueue(ItemDesc &Item) { // Determine which queue to put the item in @@ -212,6 +313,13 @@ void pkgAcquire::Enqueue(ItemDesc &Item) if (Name.empty() == true) return; + /* the check for running avoids that we produce errors + in logging before we actually have started, which would + be easier to implement but would confuse users/implementations + so we check the items skipped here in #Startup */ + if (Running && DoesAcquireResultInInstantFailure(Item.Owner, Config, Log)) + return; + // Find the queue structure Queue *I = Queues; for (; I != 0 && I->Name != Name; I = I->Next); @@ -286,8 +394,42 @@ string pkgAcquire::QueueName(string Uri,MethodConfig const *&Config) if (Config->SingleInstance == true || QueueMode == QueueAccess) return U.Access; - string AccessSchema = U.Access + ':', - FullQueueName = AccessSchema + U.Host; + string AccessSchema = U.Access + ':'; + string FullQueueName; + + if (U.Host.empty()) + { + long existing = 0; + // check how many queues exist already and reuse empty ones + for (Queue const *I = Queues; I != 0; I = I->Next) + if (I->Name.compare(0, AccessSchema.length(), AccessSchema) == 0) + { + if (I->Items == nullptr) + return I->Name; + ++existing; + } + +#ifdef _SC_NPROCESSORS_ONLN + long cpuCount = sysconf(_SC_NPROCESSORS_ONLN) * 2; +#else + long cpuCount = 10; +#endif + cpuCount = _config->FindI("Acquire::QueueHost::Limit", cpuCount); + + if (cpuCount <= 0 || existing < cpuCount) + strprintf(FullQueueName, "%s%ld", AccessSchema.c_str(), existing); + else + { + long const randomQueue = random() % cpuCount; + strprintf(FullQueueName, "%s%ld", AccessSchema.c_str(), randomQueue); + } + + if (Debug) + clog << "Chose random queue " << FullQueueName << " for " << Uri << endl; + } else + { + FullQueueName = AccessSchema + U.Host; + } unsigned int Instances = 0, SchemaLength = AccessSchema.length(); Queue *I = Queues; @@ -363,20 +505,30 @@ void pkgAcquire::SetFds(int &Fd,fd_set *RSet,fd_set *WSet) } } /*}}}*/ -// Acquire::RunFds - Deal with active FDs /*{{{*/ +// Acquire::RunFds - compatibility remove on next abi/api break /*{{{*/ +void pkgAcquire::RunFds(fd_set *RSet,fd_set *WSet) +{ + RunFdsSane(RSet, WSet); +} + /*}}}*/ +// Acquire::RunFdsSane - Deal with active FDs /*{{{*/ // --------------------------------------------------------------------- /* Dispatch active FDs over to the proper workers. It is very important that a worker never be erased while this is running! The queue class should never erase a worker except during shutdown processing. */ -void pkgAcquire::RunFds(fd_set *RSet,fd_set *WSet) +bool pkgAcquire::RunFdsSane(fd_set *RSet,fd_set *WSet) { + bool Res = true; + for (Worker *I = Workers; I != 0; I = I->NextAcquire) { if (I->InFd >= 0 && FD_ISSET(I->InFd,RSet) != 0) - I->InFdReady(); + Res &= I->InFdReady(); if (I->OutFd >= 0 && FD_ISSET(I->OutFd,WSet) != 0) - I->OutFdReady(); + Res &= I->OutFdReady(); } + + return Res; } /*}}}*/ // Acquire::Run - Run the fetch sequence /*{{{*/ @@ -384,8 +536,129 @@ void pkgAcquire::RunFds(fd_set *RSet,fd_set *WSet) /* This runs the queues. It manages a select loop for all of the Worker tasks. The workers interact with the queues and items to manage the actual fetch. */ +static bool IsAccessibleBySandboxUser(std::string const &filename, bool const ReadWrite) +{ + // you would think this is easily to answer with faccessat, right? Wrong! + // It e.g. gets groups wrong, so the only thing which works reliable is trying + // to open the file we want to open later on… + if (unlikely(filename.empty())) + return true; + + if (ReadWrite == false) + { + errno = 0; + // can we read a file? Note that non-existing files are "fine" + int const fd = open(filename.c_str(), O_RDONLY | O_CLOEXEC); + if (fd == -1 && errno == EACCES) + return false; + close(fd); + return true; + } + else + { + // the file might not exist yet and even if it does we will fix permissions, + // so important is here just that the directory it is in allows that + std::string const dirname = flNotFile(filename); + if (unlikely(dirname.empty())) + return true; + + char const * const filetag = ".apt-acquire-privs-test.XXXXXX"; + std::string const tmpfile_tpl = flCombine(dirname, filetag); + std::unique_ptr tmpfile { strdup(tmpfile_tpl.c_str()), std::free }; + int const fd = mkstemp(tmpfile.get()); + if (fd == -1 && errno == EACCES) + return false; + RemoveFile("IsAccessibleBySandboxUser", tmpfile.get()); + close(fd); + return true; + } +} +static void CheckDropPrivsMustBeDisabled(pkgAcquire const &Fetcher) +{ + if(getuid() != 0) + return; + + std::string const SandboxUser = _config->Find("APT::Sandbox::User"); + if (SandboxUser.empty() || SandboxUser == "root") + return; + + struct passwd const * const pw = getpwnam(SandboxUser.c_str()); + if (pw == NULL) + { + _error->Warning(_("No sandbox user '%s' on the system, can not drop privileges"), SandboxUser.c_str()); + _config->Set("APT::Sandbox::User", ""); + return; + } + + gid_t const old_euid = geteuid(); + gid_t const old_egid = getegid(); + + long const ngroups_max = sysconf(_SC_NGROUPS_MAX); + std::unique_ptr old_gidlist(new gid_t[ngroups_max]); + if (unlikely(old_gidlist == NULL)) + return; + ssize_t old_gidlist_nr; + if ((old_gidlist_nr = getgroups(ngroups_max, old_gidlist.get())) < 0) + { + _error->FatalE("getgroups", "getgroups %lu failed", ngroups_max); + old_gidlist[0] = 0; + old_gidlist_nr = 1; + } + if (setgroups(1, &pw->pw_gid)) + _error->FatalE("setgroups", "setgroups %u failed", pw->pw_gid); + + if (setegid(pw->pw_gid) != 0) + _error->FatalE("setegid", "setegid %u failed", pw->pw_gid); + if (seteuid(pw->pw_uid) != 0) + _error->FatalE("seteuid", "seteuid %u failed", pw->pw_uid); + + for (pkgAcquire::ItemCIterator I = Fetcher.ItemsBegin(); + I != Fetcher.ItemsEnd(); ++I) + { + // no need to drop privileges for a complete file + if ((*I)->Complete == true || (*I)->Status != pkgAcquire::Item::StatIdle) + continue; + + // if destination file is inaccessible all hope is lost for privilege dropping + if (IsAccessibleBySandboxUser((*I)->DestFile, true) == false) + { + _error->WarningE("pkgAcquire::Run", _("Can't drop privileges for downloading as file '%s' couldn't be accessed by user '%s'."), + (*I)->DestFile.c_str(), SandboxUser.c_str()); + _config->Set("APT::Sandbox::User", ""); + break; + } + + // if its the source file (e.g. local sources) we might be lucky + // by dropping the dropping only for some methods. + URI const source = (*I)->DescURI(); + if (source.Access == "file" || source.Access == "copy") + { + std::string const conf = "Binary::" + source.Access + "::APT::Sandbox::User"; + if (_config->Exists(conf) == true) + continue; + + if (IsAccessibleBySandboxUser(source.Path, false) == false) + { + _error->NoticeE("pkgAcquire::Run", _("Can't drop privileges for downloading as file '%s' couldn't be accessed by user '%s'."), + source.Path.c_str(), SandboxUser.c_str()); + _config->CndSet("Binary::file::APT::Sandbox::User", "root"); + _config->CndSet("Binary::copy::APT::Sandbox::User", "root"); + } + } + } + + if (seteuid(old_euid) != 0) + _error->FatalE("seteuid", "seteuid %u failed", old_euid); + if (setegid(old_egid) != 0) + _error->FatalE("setegid", "setegid %u failed", old_egid); + if (setgroups(old_gidlist_nr, old_gidlist.get())) + _error->FatalE("setgroups", "setgroups %u failed", 0); +} pkgAcquire::RunResult pkgAcquire::Run(int PulseIntervall) { + _error->PushToStack(); + CheckDropPrivsMustBeDisabled(*this); + Running = true; for (Queue *I = Queues; I != 0; I = I->Next) @@ -421,11 +694,10 @@ pkgAcquire::RunResult pkgAcquire::Run(int PulseIntervall) _error->Errno("select","Select has failed"); break; } - - RunFds(&RFds,&WFds); - if (_error->PendingError() == true) - break; - + + if(RunFdsSane(&RFds,&WFds) == false) + break; + // Timeout, notify the log class if (Res == 0 || (Log != 0 && Log->Update == true)) { @@ -450,9 +722,11 @@ pkgAcquire::RunResult pkgAcquire::Run(int PulseIntervall) // Shut down the items for (ItemIterator I = Items.begin(); I != Items.end(); ++I) - (*I)->Finished(); - - if (_error->PendingError()) + (*I)->Finished(); + + bool const newError = _error->PendingError(); + _error->MergeWithStack(); + if (newError) return Failed; if (WasCancelled) return Cancelled; @@ -487,6 +761,9 @@ bool pkgAcquire::Clean(string Dir) if (DirectoryExists(Dir) == false) return true; + if(Dir == "/") + return _error->Error(_("Clean of %s is not supported"), Dir.c_str()); + DIR *D = opendir(Dir.c_str()); if (D == 0) return _error->Errno("opendir",_("Unable to read %s"),Dir.c_str()); @@ -503,6 +780,7 @@ bool pkgAcquire::Clean(string Dir) // Skip some files.. if (strcmp(Dir->d_name,"lock") == 0 || strcmp(Dir->d_name,"partial") == 0 || + strcmp(Dir->d_name,"lost+found") == 0 || strcmp(Dir->d_name,".") == 0 || strcmp(Dir->d_name,"..") == 0) continue; @@ -515,7 +793,7 @@ bool pkgAcquire::Clean(string Dir) // Nothing found, nuke it if (I == Items.end()) - unlink(Dir->d_name); + RemoveFile("Clean", Dir->d_name); }; closedir(D); @@ -529,10 +807,10 @@ bool pkgAcquire::Clean(string Dir) /* This is the total number of bytes needed */ APT_PURE unsigned long long pkgAcquire::TotalNeeded() { - unsigned long long Total = 0; - for (ItemCIterator I = ItemsBegin(); I != ItemsEnd(); ++I) - Total += (*I)->FileSize; - return Total; + return std::accumulate(ItemsBegin(), ItemsEnd(), 0llu, + [](unsigned long long const T, Item const * const I) { + return T + I->FileSize; + }); } /*}}}*/ // Acquire::FetchNeeded - Number of bytes needed to get /*{{{*/ @@ -540,11 +818,13 @@ APT_PURE unsigned long long pkgAcquire::TotalNeeded() /* This is the number of bytes that is not local */ APT_PURE unsigned long long pkgAcquire::FetchNeeded() { - unsigned long long Total = 0; - for (ItemCIterator I = ItemsBegin(); I != ItemsEnd(); ++I) - if ((*I)->Local == false) - Total += (*I)->FileSize; - return Total; + return std::accumulate(ItemsBegin(), ItemsEnd(), 0llu, + [](unsigned long long const T, Item const * const I) { + if (I->Local == false) + return T + I->FileSize; + else + return T; + }); } /*}}}*/ // Acquire::PartialPresent - Number of partial bytes we already have /*{{{*/ @@ -552,11 +832,13 @@ APT_PURE unsigned long long pkgAcquire::FetchNeeded() /* This is the number of bytes that is not local */ APT_PURE unsigned long long pkgAcquire::PartialPresent() { - unsigned long long Total = 0; - for (ItemCIterator I = ItemsBegin(); I != ItemsEnd(); ++I) - if ((*I)->Local == false) - Total += (*I)->PartialSize; - return Total; + return std::accumulate(ItemsBegin(), ItemsEnd(), 0llu, + [](unsigned long long const T, Item const * const I) { + if (I->Local == false) + return T + I->PartialSize; + else + return T; + }); } /*}}}*/ // Acquire::UriBegin - Start iterator for the uri list /*{{{*/ @@ -578,27 +860,18 @@ pkgAcquire::UriIterator pkgAcquire::UriEnd() // Acquire::MethodConfig::MethodConfig - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ -pkgAcquire::MethodConfig::MethodConfig() +pkgAcquire::MethodConfig::MethodConfig() : d(NULL), Next(0), SingleInstance(false), + Pipeline(false), SendConfig(false), LocalOnly(false), NeedsCleanup(false), + Removable(false) { - SingleInstance = false; - Pipeline = false; - SendConfig = false; - LocalOnly = false; - Removable = false; - Next = 0; } /*}}}*/ // Queue::Queue - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ -pkgAcquire::Queue::Queue(string Name,pkgAcquire *Owner) : Name(Name), - Owner(Owner) +pkgAcquire::Queue::Queue(string const &name,pkgAcquire * const owner) : d(NULL), Next(0), + Name(name), Items(0), Workers(0), Owner(owner), PipeDepth(0), MaxPipeDepth(1) { - Items = 0; - Next = 0; - Workers = 0; - MaxPipeDepth = 1; - PipeDepth = 0; } /*}}}*/ // Queue::~Queue - Destructor /*{{{*/ @@ -624,9 +897,12 @@ bool pkgAcquire::Queue::Enqueue(ItemDesc &Item) QItem **I = &Items; // move to the end of the queue and check for duplicates here for (; *I != 0; I = &(*I)->Next) - if (Item.URI == (*I)->URI) + if (Item.URI == (*I)->URI) { - Item.Owner->Status = Item::StatDone; + if (_config->FindB("Debug::pkgAcquire::Worker",false) == true) + std::cerr << " @ Queue: Action combined for " << Item.URI << " and " << (*I)->URI << std::endl; + (*I)->Owners.push_back(Item.Owner); + Item.Owner->Status = (*I)->Owner->Status; return false; } @@ -649,13 +925,13 @@ bool pkgAcquire::Queue::Dequeue(Item *Owner) { if (Owner->Status == pkgAcquire::Item::StatFetching) return _error->Error("Tried to dequeue a fetching object"); - + bool Res = false; - + QItem **I = &Items; for (; *I != 0;) { - if ((*I)->Owner == Owner) + if (Owner == (*I)->Owner) { QItem *Jnk= *I; *I = (*I)->Next; @@ -666,7 +942,7 @@ bool pkgAcquire::Queue::Dequeue(Item *Owner) else I = &(*I)->Next; } - + return Res; } /*}}}*/ @@ -679,10 +955,20 @@ bool pkgAcquire::Queue::Startup() if (Workers == 0) { URI U(Name); - pkgAcquire::MethodConfig *Cnf = Owner->GetConfig(U.Access); - if (Cnf == 0) + pkgAcquire::MethodConfig * const Cnf = Owner->GetConfig(U.Access); + if (unlikely(Cnf == nullptr)) return false; - + + // now-running twin of the pkgAcquire::Enqueue call + for (QItem *I = Items; I != 0; ) + { + bool pointless = false; + for (auto &&O: I->Owners) + if (DoesAcquireResultInInstantFailure(O, Cnf, Owner->Log)) + pointless = true; + I = pointless ? Items : I->Next; + } + Workers = new Worker(this,Cnf,Owner->Log); Owner->Add(Workers); if (Workers->Start() == false) @@ -743,9 +1029,12 @@ pkgAcquire::Queue::QItem *pkgAcquire::Queue::FindItem(string URI,pkgAcquire::Wor bool pkgAcquire::Queue::ItemDone(QItem *Itm) { PipeDepth--; - if (Itm->Owner->Status == pkgAcquire::Item::StatFetching) - Itm->Owner->Status = pkgAcquire::Item::StatDone; - + for (QItem::owner_iterator O = Itm->Owners.begin(); O != Itm->Owners.end(); ++O) + { + if ((*O)->Status == pkgAcquire::Item::StatFetching) + (*O)->Status = pkgAcquire::Item::StatDone; + } + if (Itm->Owner->QueueCounter <= 1) Owner->Dequeue(Itm->Owner); else @@ -753,7 +1042,7 @@ bool pkgAcquire::Queue::ItemDone(QItem *Itm) Dequeue(Itm->Owner); Owner->Bump(); } - + return Cycle(); } /*}}}*/ @@ -768,7 +1057,7 @@ bool pkgAcquire::Queue::Cycle() if (PipeDepth < 0) return _error->Error("Pipedepth failure"); - + // Look for a queable item QItem *I = Items; while (PipeDepth < (signed)MaxPipeDepth) @@ -776,18 +1065,19 @@ bool pkgAcquire::Queue::Cycle() for (; I != 0; I = I->Next) if (I->Owner->Status == pkgAcquire::Item::StatIdle) break; - + // Nothing to do, queue is idle. if (I == 0) return true; - + I->Worker = Workers; - I->Owner->Status = pkgAcquire::Item::StatFetching; + for (auto const &O: I->Owners) + O->Status = pkgAcquire::Item::StatFetching; PipeDepth++; if (Workers->QueueItem(I) == false) return false; } - + return true; } /*}}}*/ @@ -799,10 +1089,98 @@ void pkgAcquire::Queue::Bump() Cycle(); } /*}}}*/ +HashStringList pkgAcquire::Queue::QItem::GetExpectedHashes() const /*{{{*/ +{ + /* each Item can have multiple owners and each owner might have different + hashes, even if that is unlikely in practice and if so at least some + owners will later fail. There is one situation through which is not a + failure and still needs this handling: Two owners who expect the same + file, but one owner only knows the SHA1 while the other only knows SHA256. */ + HashStringList superhsl; + for (pkgAcquire::Queue::QItem::owner_iterator O = Owners.begin(); O != Owners.end(); ++O) + { + HashStringList const hsl = (*O)->GetExpectedHashes(); + if (hsl.usable() == false) + continue; + if (superhsl.usable() == false) + superhsl = hsl; + else + { + // we merge both lists - if we find disagreement send no hashes + HashStringList::const_iterator hs = hsl.begin(); + for (; hs != hsl.end(); ++hs) + if (superhsl.push_back(*hs) == false) + break; + if (hs != hsl.end()) + { + superhsl.clear(); + break; + } + } + } + return superhsl; +} + /*}}}*/ +APT_PURE unsigned long long pkgAcquire::Queue::QItem::GetMaximumSize() const /*{{{*/ +{ + unsigned long long Maximum = std::numeric_limits::max(); + for (auto const &O: Owners) + { + if (O->FileSize == 0) + continue; + Maximum = std::min(Maximum, O->FileSize); + } + if (Maximum == std::numeric_limits::max()) + return 0; + return Maximum; +} + /*}}}*/ +void pkgAcquire::Queue::QItem::SyncDestinationFiles() const /*{{{*/ +{ + /* ensure that the first owner has the best partial file of all and + the rest have (potentially dangling) symlinks to it so that + everything (like progress reporting) finds it easily */ + std::string superfile = Owner->DestFile; + off_t supersize = 0; + for (pkgAcquire::Queue::QItem::owner_iterator O = Owners.begin(); O != Owners.end(); ++O) + { + if ((*O)->DestFile == superfile) + continue; + struct stat file; + if (lstat((*O)->DestFile.c_str(),&file) == 0) + { + if ((file.st_mode & S_IFREG) == 0) + RemoveFile("SyncDestinationFiles", (*O)->DestFile); + else if (supersize < file.st_size) + { + supersize = file.st_size; + RemoveFile("SyncDestinationFiles", superfile); + rename((*O)->DestFile.c_str(), superfile.c_str()); + } + else + RemoveFile("SyncDestinationFiles", (*O)->DestFile); + if (symlink(superfile.c_str(), (*O)->DestFile.c_str()) != 0) + { + ; // not a problem per-se and no real alternative + } + } + } +} + /*}}}*/ +std::string pkgAcquire::Queue::QItem::Custom600Headers() const /*{{{*/ +{ + /* The others are relatively easy to merge, but this one? + Lets not merge and see how far we can run with it… + Likely, nobody will ever notice as all the items will + be of the same class and hence generate the same headers. */ + return Owner->Custom600Headers(); +} + /*}}}*/ + // AcquireStatus::pkgAcquireStatus - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ -pkgAcquireStatus::pkgAcquireStatus() : d(NULL), Update(true), MorePulses(false) +pkgAcquireStatus::pkgAcquireStatus() : d(NULL), Percent(-1), Update(true), MorePulses(false) { Start(); } @@ -818,11 +1196,11 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) CurrentBytes = 0; TotalItems = 0; CurrentItems = 0; - + // Compute the total number of bytes to fetch unsigned int Unknown = 0; unsigned int Count = 0; - bool UnfetchedReleaseFiles = false; + bool ExpectAdditionalItems = false; for (pkgAcquire::ItemCIterator I = Owner->ItemsBegin(); I != Owner->ItemsEnd(); ++I, ++Count) @@ -830,17 +1208,10 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) TotalItems++; if ((*I)->Status == pkgAcquire::Item::StatDone) ++CurrentItems; - - // Totally ignore local items - if ((*I)->Local == true) - continue; - - // see if the method tells us to expect more - TotalItems += (*I)->ExpectedAdditionalItems; - // check if there are unfetched Release files - if ((*I)->Complete == false && (*I)->ExpectedAdditionalItems > 0) - UnfetchedReleaseFiles = true; + // do we expect to acquire more files than we know of yet? + if ((*I)->ExpectedAdditionalItems > 0) + ExpectAdditionalItems = true; TotalBytes += (*I)->FileSize; if ((*I)->Complete == true) @@ -848,7 +1219,7 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) if ((*I)->FileSize == 0 && (*I)->Complete == false) ++Unknown; } - + // Compute the current completion unsigned long long ResumeSize = 0; for (pkgAcquire::Worker *I = Owner->WorkersBegin(); I != 0; @@ -858,9 +1229,9 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) { CurrentBytes += I->CurrentSize; ResumeSize += I->ResumePoint; - + // Files with unknown size always have 100% completion - if (I->CurrentItem->Owner->FileSize == 0 && + if (I->CurrentItem->Owner->FileSize == 0 && I->CurrentItem->Owner->Complete == false) TotalBytes += I->CurrentSize; } @@ -876,12 +1247,6 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) if (CurrentBytes > TotalBytes) CurrentBytes = TotalBytes; - // debug - if (_config->FindB("Debug::acquire::progress", false) == true) - std::clog << " Bytes: " - << SizeToStr(CurrentBytes) << " / " << SizeToStr(TotalBytes) - << std::endl; - // Compute the CPS struct timeval NewTime; gettimeofday(&NewTime,0); @@ -901,14 +1266,33 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) Time = NewTime; } + double const OldPercent = Percent; // calculate the percentage, if we have too little data assume 1% - if (TotalBytes > 0 && UnfetchedReleaseFiles) + if (ExpectAdditionalItems) Percent = 0; - else + else // use both files and bytes because bytes can be unreliable - Percent = (0.8 * (CurrentBytes/float(TotalBytes)*100.0) + + Percent = (0.8 * (CurrentBytes/float(TotalBytes)*100.0) + 0.2 * (CurrentItems/float(TotalItems)*100.0)); + // debug + if (_config->FindB("Debug::acquire::progress", false) == true) + { + std::clog + << "[" + << std::setw(5) << std::setprecision(4) << std::showpoint << Percent + << "]" + << " Bytes: " + << SizeToStr(CurrentBytes) << " / " << SizeToStr(TotalBytes) + << " # Files: " + << CurrentItems << " / " << TotalItems + << std::endl; + } + + double const DiffPercent = Percent - OldPercent; + if (DiffPercent < 0.001 && _config->FindB("Acquire::Progress::Diffpercent", false) == true) + return true; + int fd = _config->FindI("APT::Status-Fd",-1); if(fd > 0) { @@ -925,15 +1309,14 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) snprintf(msg,sizeof(msg), _("Retrieving file %li of %li (%s remaining)"), i, TotalItems, TimeToStr(ETA).c_str()); else snprintf(msg,sizeof(msg), _("Retrieving file %li of %li"), i, TotalItems); - - // build the status str - status << "dlstatus:" << i - << ":" << std::setprecision(3) << Percent - << ":" << msg - << endl; - std::string const dlstatus = status.str(); - FileFd::Write(fd, dlstatus.c_str(), dlstatus.size()); + // build the status str + std::ostringstream str; + str.imbue(std::locale("C.UTF-8")); + str.precision(4); + str << "dlstatus" << ':' << std::fixed << i << ':' << Percent << ':' << msg << '\n'; + auto const dlstatus = str.str(); + FileFd::Write(fd, dlstatus.data(), dlstatus.size()); } return true; @@ -985,3 +1368,16 @@ void pkgAcquireStatus::Fetched(unsigned long long Size,unsigned long long Resume FetchedBytes += Size - Resume; } /*}}}*/ + +pkgAcquire::UriIterator::UriIterator(pkgAcquire::Queue *Q) : d(NULL), CurQ(Q), CurItem(0) +{ + while (CurItem == 0 && CurQ != 0) + { + CurItem = CurQ->Items; + CurQ = CurQ->Next; + } +} + +APT_CONST pkgAcquire::UriIterator::~UriIterator() {} +APT_CONST pkgAcquire::MethodConfig::~MethodConfig() {} +APT_CONST pkgAcquireStatus::~pkgAcquireStatus() {}