]> git.saurik.com Git - apt.git/blob - apt-pkg/acquire.cc
show more details for "Hash Sum mismatch" errors
[apt.git] / apt-pkg / acquire.cc
1 // -*- mode: cpp; mode: fold -*-
2 // Description /*{{{*/
3 // $Id: acquire.cc,v 1.50 2004/03/17 05:17:11 mdz Exp $
4 /* ######################################################################
5
6 Acquire - File Acquiration
7
8 The core element for the schedule system is the concept of a named
9 queue. Each queue is unique and each queue has a name derived from the
10 URI. The degree of paralization can be controlled by how the queue
11 name is derived from the URI.
12
13 ##################################################################### */
14 /*}}}*/
15 // Include Files /*{{{*/
16 #include <config.h>
17
18 #include <apt-pkg/acquire.h>
19 #include <apt-pkg/acquire-item.h>
20 #include <apt-pkg/acquire-worker.h>
21 #include <apt-pkg/configuration.h>
22 #include <apt-pkg/error.h>
23 #include <apt-pkg/strutl.h>
24 #include <apt-pkg/fileutl.h>
25
26 #include <algorithm>
27 #include <numeric>
28 #include <string>
29 #include <vector>
30 #include <iostream>
31 #include <sstream>
32 #include <iomanip>
33 #include <memory>
34
35 #include <stdio.h>
36 #include <stdlib.h>
37 #include <string.h>
38 #include <unistd.h>
39 #include <fcntl.h>
40 #include <pwd.h>
41 #include <grp.h>
42 #include <dirent.h>
43 #include <sys/time.h>
44 #include <sys/select.h>
45 #include <errno.h>
46 #include <sys/stat.h>
47
48 #include <apti18n.h>
49 /*}}}*/
50
51 using namespace std;
52
53 // Acquire::pkgAcquire - Constructor /*{{{*/
54 // ---------------------------------------------------------------------
55 /* We grab some runtime state from the configuration space */
56 pkgAcquire::pkgAcquire() : LockFD(-1), d(NULL), Queues(0), Workers(0), Configs(0), Log(NULL), ToFetch(0),
57 Debug(_config->FindB("Debug::pkgAcquire",false)),
58 Running(false)
59 {
60 Initialize();
61 }
62 pkgAcquire::pkgAcquire(pkgAcquireStatus *Progress) : LockFD(-1), d(NULL), Queues(0), Workers(0),
63 Configs(0), Log(NULL), ToFetch(0),
64 Debug(_config->FindB("Debug::pkgAcquire",false)),
65 Running(false)
66 {
67 Initialize();
68 SetLog(Progress);
69 }
70 void pkgAcquire::Initialize()
71 {
72 string const Mode = _config->Find("Acquire::Queue-Mode","host");
73 if (strcasecmp(Mode.c_str(),"host") == 0)
74 QueueMode = QueueHost;
75 if (strcasecmp(Mode.c_str(),"access") == 0)
76 QueueMode = QueueAccess;
77
78 // chown the auth.conf file as it will be accessed by our methods
79 std::string const SandboxUser = _config->Find("APT::Sandbox::User");
80 if (getuid() == 0 && SandboxUser.empty() == false && SandboxUser != "root") // if we aren't root, we can't chown, so don't try it
81 {
82 struct passwd const * const pw = getpwnam(SandboxUser.c_str());
83 struct group const * const gr = getgrnam("root");
84 if (pw != NULL && gr != NULL)
85 {
86 std::string const AuthConf = _config->FindFile("Dir::Etc::netrc");
87 if(AuthConf.empty() == false && RealFileExists(AuthConf) &&
88 chown(AuthConf.c_str(), pw->pw_uid, gr->gr_gid) != 0)
89 _error->WarningE("SetupAPTPartialDirectory", "chown to %s:root of file %s failed", SandboxUser.c_str(), AuthConf.c_str());
90 }
91 }
92 }
93 /*}}}*/
94 // Acquire::GetLock - lock directory and prepare for action /*{{{*/
95 static bool SetupAPTPartialDirectory(std::string const &grand, std::string const &parent)
96 {
97 std::string const partial = parent + "partial";
98 mode_t const mode = umask(S_IWGRP | S_IWOTH);
99 bool const creation_fail = (CreateAPTDirectoryIfNeeded(grand, partial) == false &&
100 CreateAPTDirectoryIfNeeded(parent, partial) == false);
101 umask(mode);
102 if (creation_fail == true)
103 return false;
104
105 std::string const SandboxUser = _config->Find("APT::Sandbox::User");
106 if (getuid() == 0 && SandboxUser.empty() == false && SandboxUser != "root") // if we aren't root, we can't chown, so don't try it
107 {
108 struct passwd const * const pw = getpwnam(SandboxUser.c_str());
109 struct group const * const gr = getgrnam("root");
110 if (pw != NULL && gr != NULL)
111 {
112 // chown the partial dir
113 if(chown(partial.c_str(), pw->pw_uid, gr->gr_gid) != 0)
114 _error->WarningE("SetupAPTPartialDirectory", "chown to %s:root of directory %s failed", SandboxUser.c_str(), partial.c_str());
115 }
116 }
117 if (chmod(partial.c_str(), 0700) != 0)
118 _error->WarningE("SetupAPTPartialDirectory", "chmod 0700 of directory %s failed", partial.c_str());
119
120 return true;
121 }
122 bool pkgAcquire::Setup(pkgAcquireStatus *Progress, string const &Lock)
123 {
124 Log = Progress;
125 if (Lock.empty())
126 {
127 string const listDir = _config->FindDir("Dir::State::lists");
128 if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir) == false)
129 return _error->Errno("Acquire", _("List directory %spartial is missing."), listDir.c_str());
130 string const archivesDir = _config->FindDir("Dir::Cache::Archives");
131 if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir) == false)
132 return _error->Errno("Acquire", _("Archives directory %spartial is missing."), archivesDir.c_str());
133 return true;
134 }
135 return GetLock(Lock);
136 }
137 bool pkgAcquire::GetLock(std::string const &Lock)
138 {
139 if (Lock.empty() == true)
140 return false;
141
142 // check for existence and possibly create auxiliary directories
143 string const listDir = _config->FindDir("Dir::State::lists");
144 string const archivesDir = _config->FindDir("Dir::Cache::Archives");
145
146 if (Lock == listDir)
147 {
148 if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir) == false)
149 return _error->Errno("Acquire", _("List directory %spartial is missing."), listDir.c_str());
150 }
151 if (Lock == archivesDir)
152 {
153 if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir) == false)
154 return _error->Errno("Acquire", _("Archives directory %spartial is missing."), archivesDir.c_str());
155 }
156
157 if (_config->FindB("Debug::NoLocking", false) == true)
158 return true;
159
160 // Lock the directory this acquire object will work in
161 if (LockFD != -1)
162 close(LockFD);
163 LockFD = ::GetLock(flCombine(Lock, "lock"));
164 if (LockFD == -1)
165 return _error->Error(_("Unable to lock directory %s"), Lock.c_str());
166
167 return true;
168 }
169 /*}}}*/
170 // Acquire::~pkgAcquire - Destructor /*{{{*/
171 // ---------------------------------------------------------------------
172 /* Free our memory, clean up the queues (destroy the workers) */
173 pkgAcquire::~pkgAcquire()
174 {
175 Shutdown();
176
177 if (LockFD != -1)
178 close(LockFD);
179
180 while (Configs != 0)
181 {
182 MethodConfig *Jnk = Configs;
183 Configs = Configs->Next;
184 delete Jnk;
185 }
186 }
187 /*}}}*/
188 // Acquire::Shutdown - Clean out the acquire object /*{{{*/
189 // ---------------------------------------------------------------------
190 /* */
191 void pkgAcquire::Shutdown()
192 {
193 while (Items.empty() == false)
194 {
195 if (Items[0]->Status == Item::StatFetching)
196 Items[0]->Status = Item::StatError;
197 delete Items[0];
198 }
199
200 while (Queues != 0)
201 {
202 Queue *Jnk = Queues;
203 Queues = Queues->Next;
204 delete Jnk;
205 }
206 }
207 /*}}}*/
208 // Acquire::Add - Add a new item /*{{{*/
209 // ---------------------------------------------------------------------
210 /* This puts an item on the acquire list. This list is mainly for tracking
211 item status */
212 void pkgAcquire::Add(Item *Itm)
213 {
214 Items.push_back(Itm);
215 }
216 /*}}}*/
217 // Acquire::Remove - Remove a item /*{{{*/
218 // ---------------------------------------------------------------------
219 /* Remove an item from the acquire list. This is usually not used.. */
220 void pkgAcquire::Remove(Item *Itm)
221 {
222 Dequeue(Itm);
223
224 for (ItemIterator I = Items.begin(); I != Items.end();)
225 {
226 if (*I == Itm)
227 {
228 Items.erase(I);
229 I = Items.begin();
230 }
231 else
232 ++I;
233 }
234 }
235 /*}}}*/
236 // Acquire::Add - Add a worker /*{{{*/
237 // ---------------------------------------------------------------------
238 /* A list of workers is kept so that the select loop can direct their FD
239 usage. */
240 void pkgAcquire::Add(Worker *Work)
241 {
242 Work->NextAcquire = Workers;
243 Workers = Work;
244 }
245 /*}}}*/
246 // Acquire::Remove - Remove a worker /*{{{*/
247 // ---------------------------------------------------------------------
248 /* A worker has died. This can not be done while the select loop is running
249 as it would require that RunFds could handling a changing list state and
250 it can't.. */
251 void pkgAcquire::Remove(Worker *Work)
252 {
253 if (Running == true)
254 abort();
255
256 Worker **I = &Workers;
257 for (; *I != 0;)
258 {
259 if (*I == Work)
260 *I = (*I)->NextAcquire;
261 else
262 I = &(*I)->NextAcquire;
263 }
264 }
265 /*}}}*/
266 // Acquire::Enqueue - Queue an URI for fetching /*{{{*/
267 // ---------------------------------------------------------------------
268 /* This is the entry point for an item. An item calls this function when
269 it is constructed which creates a queue (based on the current queue
270 mode) and puts the item in that queue. If the system is running then
271 the queue might be started. */
272 void pkgAcquire::Enqueue(ItemDesc &Item)
273 {
274 // Determine which queue to put the item in
275 const MethodConfig *Config;
276 string Name = QueueName(Item.URI,Config);
277 if (Name.empty() == true)
278 return;
279
280 // Find the queue structure
281 Queue *I = Queues;
282 for (; I != 0 && I->Name != Name; I = I->Next);
283 if (I == 0)
284 {
285 I = new Queue(Name,this);
286 I->Next = Queues;
287 Queues = I;
288
289 if (Running == true)
290 I->Startup();
291 }
292
293 // See if this is a local only URI
294 if (Config->LocalOnly == true && Item.Owner->Complete == false)
295 Item.Owner->Local = true;
296 Item.Owner->Status = Item::StatIdle;
297
298 // Queue it into the named queue
299 if(I->Enqueue(Item))
300 ToFetch++;
301
302 // Some trace stuff
303 if (Debug == true)
304 {
305 clog << "Fetching " << Item.URI << endl;
306 clog << " to " << Item.Owner->DestFile << endl;
307 clog << " Queue is: " << Name << endl;
308 }
309 }
310 /*}}}*/
311 // Acquire::Dequeue - Remove an item from all queues /*{{{*/
312 // ---------------------------------------------------------------------
313 /* This is called when an item is finished being fetched. It removes it
314 from all the queues */
315 void pkgAcquire::Dequeue(Item *Itm)
316 {
317 Queue *I = Queues;
318 bool Res = false;
319 if (Debug == true)
320 clog << "Dequeuing " << Itm->DestFile << endl;
321
322 for (; I != 0; I = I->Next)
323 {
324 if (I->Dequeue(Itm))
325 {
326 Res = true;
327 if (Debug == true)
328 clog << "Dequeued from " << I->Name << endl;
329 }
330 }
331
332 if (Res == true)
333 ToFetch--;
334 }
335 /*}}}*/
336 // Acquire::QueueName - Return the name of the queue for this URI /*{{{*/
337 // ---------------------------------------------------------------------
338 /* The string returned depends on the configuration settings and the
339 method parameters. Given something like http://foo.org/bar it can
340 return http://foo.org or http */
341 string pkgAcquire::QueueName(string Uri,MethodConfig const *&Config)
342 {
343 URI U(Uri);
344
345 Config = GetConfig(U.Access);
346 if (Config == 0)
347 return string();
348
349 /* Single-Instance methods get exactly one queue per URI. This is
350 also used for the Access queue method */
351 if (Config->SingleInstance == true || QueueMode == QueueAccess)
352 return U.Access;
353
354 string AccessSchema = U.Access + ':';
355 string FullQueueName;
356
357 if (U.Host.empty())
358 {
359 long existing = 0;
360 // check how many queues exist already and reuse empty ones
361 for (Queue const *I = Queues; I != 0; I = I->Next)
362 if (I->Name.compare(0, AccessSchema.length(), AccessSchema) == 0)
363 {
364 if (I->Items == nullptr)
365 return I->Name;
366 ++existing;
367 }
368
369 #ifdef _SC_NPROCESSORS_ONLN
370 long cpuCount = sysconf(_SC_NPROCESSORS_ONLN) * 2;
371 #else
372 long cpuCount = 10;
373 #endif
374 cpuCount = _config->FindI("Acquire::QueueHost::Limit", cpuCount);
375
376 if (cpuCount <= 0 || existing < cpuCount)
377 strprintf(FullQueueName, "%s%ld", AccessSchema.c_str(), existing);
378 else
379 {
380 long const randomQueue = random() % cpuCount;
381 strprintf(FullQueueName, "%s%ld", AccessSchema.c_str(), randomQueue);
382 }
383
384 if (Debug)
385 clog << "Chose random queue " << FullQueueName << " for " << Uri << endl;
386 } else
387 {
388 FullQueueName = AccessSchema + U.Host;
389 }
390 unsigned int Instances = 0, SchemaLength = AccessSchema.length();
391
392 Queue *I = Queues;
393 for (; I != 0; I = I->Next) {
394 // if the queue already exists, re-use it
395 if (I->Name == FullQueueName)
396 return FullQueueName;
397
398 if (I->Name.compare(0, SchemaLength, AccessSchema) == 0)
399 Instances++;
400 }
401
402 if (Debug) {
403 clog << "Found " << Instances << " instances of " << U.Access << endl;
404 }
405
406 if (Instances >= (unsigned int)_config->FindI("Acquire::QueueHost::Limit",10))
407 return U.Access;
408
409 return FullQueueName;
410 }
411 /*}}}*/
412 // Acquire::GetConfig - Fetch the configuration information /*{{{*/
413 // ---------------------------------------------------------------------
414 /* This locates the configuration structure for an access method. If
415 a config structure cannot be found a Worker will be created to
416 retrieve it */
417 pkgAcquire::MethodConfig *pkgAcquire::GetConfig(string Access)
418 {
419 // Search for an existing config
420 MethodConfig *Conf;
421 for (Conf = Configs; Conf != 0; Conf = Conf->Next)
422 if (Conf->Access == Access)
423 return Conf;
424
425 // Create the new config class
426 Conf = new MethodConfig;
427 Conf->Access = Access;
428 Conf->Next = Configs;
429 Configs = Conf;
430
431 // Create the worker to fetch the configuration
432 Worker Work(Conf);
433 if (Work.Start() == false)
434 return 0;
435
436 /* if a method uses DownloadLimit, we switch to SingleInstance mode */
437 if(_config->FindI("Acquire::"+Access+"::Dl-Limit",0) > 0)
438 Conf->SingleInstance = true;
439
440 return Conf;
441 }
442 /*}}}*/
443 // Acquire::SetFds - Deal with readable FDs /*{{{*/
444 // ---------------------------------------------------------------------
445 /* Collect FDs that have activity monitors into the fd sets */
446 void pkgAcquire::SetFds(int &Fd,fd_set *RSet,fd_set *WSet)
447 {
448 for (Worker *I = Workers; I != 0; I = I->NextAcquire)
449 {
450 if (I->InReady == true && I->InFd >= 0)
451 {
452 if (Fd < I->InFd)
453 Fd = I->InFd;
454 FD_SET(I->InFd,RSet);
455 }
456 if (I->OutReady == true && I->OutFd >= 0)
457 {
458 if (Fd < I->OutFd)
459 Fd = I->OutFd;
460 FD_SET(I->OutFd,WSet);
461 }
462 }
463 }
464 /*}}}*/
465 // Acquire::RunFds - compatibility remove on next abi/api break /*{{{*/
466 void pkgAcquire::RunFds(fd_set *RSet,fd_set *WSet)
467 {
468 RunFdsSane(RSet, WSet);
469 };
470 /*}}}*/
471 // Acquire::RunFdsSane - Deal with active FDs /*{{{*/
472 // ---------------------------------------------------------------------
473 /* Dispatch active FDs over to the proper workers. It is very important
474 that a worker never be erased while this is running! The queue class
475 should never erase a worker except during shutdown processing. */
476 bool pkgAcquire::RunFdsSane(fd_set *RSet,fd_set *WSet)
477 {
478 bool Res = true;
479
480 for (Worker *I = Workers; I != 0; I = I->NextAcquire)
481 {
482 if (I->InFd >= 0 && FD_ISSET(I->InFd,RSet) != 0)
483 Res &= I->InFdReady();
484 if (I->OutFd >= 0 && FD_ISSET(I->OutFd,WSet) != 0)
485 Res &= I->OutFdReady();
486 }
487
488 return Res;
489 }
490 /*}}}*/
491 // Acquire::Run - Run the fetch sequence /*{{{*/
492 // ---------------------------------------------------------------------
493 /* This runs the queues. It manages a select loop for all of the
494 Worker tasks. The workers interact with the queues and items to
495 manage the actual fetch. */
496 static bool IsAccessibleBySandboxUser(std::string const &filename, bool const ReadWrite)
497 {
498 // you would think this is easily to answer with faccessat, right? Wrong!
499 // It e.g. gets groups wrong, so the only thing which works reliable is trying
500 // to open the file we want to open later on…
501 if (unlikely(filename.empty()))
502 return true;
503
504 if (ReadWrite == false)
505 {
506 errno = 0;
507 // can we read a file? Note that non-existing files are "fine"
508 int const fd = open(filename.c_str(), O_RDONLY | O_CLOEXEC);
509 if (fd == -1 && errno == EACCES)
510 return false;
511 close(fd);
512 return true;
513 }
514 else
515 {
516 // the file might not exist yet and even if it does we will fix permissions,
517 // so important is here just that the directory it is in allows that
518 std::string const dirname = flNotFile(filename);
519 if (unlikely(dirname.empty()))
520 return true;
521
522 char const * const filetag = ".apt-acquire-privs-test.XXXXXX";
523 std::string const tmpfile_tpl = flCombine(dirname, filetag);
524 std::unique_ptr<char, decltype(std::free) *> tmpfile { strdup(tmpfile_tpl.c_str()), std::free };
525 int const fd = mkstemp(tmpfile.get());
526 if (fd == -1 && errno == EACCES)
527 return false;
528 RemoveFile("IsAccessibleBySandboxUser", tmpfile.get());
529 close(fd);
530 return true;
531 }
532 }
533 static void CheckDropPrivsMustBeDisabled(pkgAcquire const &Fetcher)
534 {
535 if(getuid() != 0)
536 return;
537
538 std::string const SandboxUser = _config->Find("APT::Sandbox::User");
539 if (SandboxUser.empty() || SandboxUser == "root")
540 return;
541
542 struct passwd const * const pw = getpwnam(SandboxUser.c_str());
543 if (pw == NULL)
544 {
545 _error->Warning(_("No sandbox user '%s' on the system, can not drop privileges"), SandboxUser.c_str());
546 _config->Set("APT::Sandbox::User", "");
547 return;
548 }
549
550 gid_t const old_euid = geteuid();
551 gid_t const old_egid = getegid();
552
553 long const ngroups_max = sysconf(_SC_NGROUPS_MAX);
554 std::unique_ptr<gid_t[]> old_gidlist(new gid_t[ngroups_max]);
555 if (unlikely(old_gidlist == NULL))
556 return;
557 ssize_t old_gidlist_nr;
558 if ((old_gidlist_nr = getgroups(ngroups_max, old_gidlist.get())) < 0)
559 {
560 _error->FatalE("getgroups", "getgroups %lu failed", ngroups_max);
561 old_gidlist[0] = 0;
562 old_gidlist_nr = 1;
563 }
564 if (setgroups(1, &pw->pw_gid))
565 _error->FatalE("setgroups", "setgroups %u failed", pw->pw_gid);
566
567 if (setegid(pw->pw_gid) != 0)
568 _error->FatalE("setegid", "setegid %u failed", pw->pw_gid);
569 if (seteuid(pw->pw_uid) != 0)
570 _error->FatalE("seteuid", "seteuid %u failed", pw->pw_uid);
571
572 for (pkgAcquire::ItemCIterator I = Fetcher.ItemsBegin();
573 I != Fetcher.ItemsEnd(); ++I)
574 {
575 // no need to drop privileges for a complete file
576 if ((*I)->Complete == true || (*I)->Status != pkgAcquire::Item::StatIdle)
577 continue;
578
579 // if destination file is inaccessible all hope is lost for privilege dropping
580 if (IsAccessibleBySandboxUser((*I)->DestFile, true) == false)
581 {
582 _error->WarningE("pkgAcquire::Run", _("Can't drop privileges for downloading as file '%s' couldn't be accessed by user '%s'."),
583 (*I)->DestFile.c_str(), SandboxUser.c_str());
584 _config->Set("APT::Sandbox::User", "");
585 break;
586 }
587
588 // if its the source file (e.g. local sources) we might be lucky
589 // by dropping the dropping only for some methods.
590 URI const source = (*I)->DescURI();
591 if (source.Access == "file" || source.Access == "copy")
592 {
593 std::string const conf = "Binary::" + source.Access + "::APT::Sandbox::User";
594 if (_config->Exists(conf) == true)
595 continue;
596
597 if (IsAccessibleBySandboxUser(source.Path, false) == false)
598 {
599 _error->NoticeE("pkgAcquire::Run", _("Can't drop privileges for downloading as file '%s' couldn't be accessed by user '%s'."),
600 source.Path.c_str(), SandboxUser.c_str());
601 _config->CndSet("Binary::file::APT::Sandbox::User", "root");
602 _config->CndSet("Binary::copy::APT::Sandbox::User", "root");
603 }
604 }
605 }
606
607 if (seteuid(old_euid) != 0)
608 _error->FatalE("seteuid", "seteuid %u failed", old_euid);
609 if (setegid(old_egid) != 0)
610 _error->FatalE("setegid", "setegid %u failed", old_egid);
611 if (setgroups(old_gidlist_nr, old_gidlist.get()))
612 _error->FatalE("setgroups", "setgroups %u failed", 0);
613 }
614 pkgAcquire::RunResult pkgAcquire::Run(int PulseIntervall)
615 {
616 _error->PushToStack();
617 CheckDropPrivsMustBeDisabled(*this);
618
619 Running = true;
620
621 for (Queue *I = Queues; I != 0; I = I->Next)
622 I->Startup();
623
624 if (Log != 0)
625 Log->Start();
626
627 bool WasCancelled = false;
628
629 // Run till all things have been acquired
630 struct timeval tv;
631 tv.tv_sec = 0;
632 tv.tv_usec = PulseIntervall;
633 while (ToFetch > 0)
634 {
635 fd_set RFds;
636 fd_set WFds;
637 int Highest = 0;
638 FD_ZERO(&RFds);
639 FD_ZERO(&WFds);
640 SetFds(Highest,&RFds,&WFds);
641
642 int Res;
643 do
644 {
645 Res = select(Highest+1,&RFds,&WFds,0,&tv);
646 }
647 while (Res < 0 && errno == EINTR);
648
649 if (Res < 0)
650 {
651 _error->Errno("select","Select has failed");
652 break;
653 }
654
655 if(RunFdsSane(&RFds,&WFds) == false)
656 break;
657
658 // Timeout, notify the log class
659 if (Res == 0 || (Log != 0 && Log->Update == true))
660 {
661 tv.tv_usec = PulseIntervall;
662 for (Worker *I = Workers; I != 0; I = I->NextAcquire)
663 I->Pulse();
664 if (Log != 0 && Log->Pulse(this) == false)
665 {
666 WasCancelled = true;
667 break;
668 }
669 }
670 }
671
672 if (Log != 0)
673 Log->Stop();
674
675 // Shut down the acquire bits
676 Running = false;
677 for (Queue *I = Queues; I != 0; I = I->Next)
678 I->Shutdown(false);
679
680 // Shut down the items
681 for (ItemIterator I = Items.begin(); I != Items.end(); ++I)
682 (*I)->Finished();
683
684 bool const newError = _error->PendingError();
685 _error->MergeWithStack();
686 if (newError)
687 return Failed;
688 if (WasCancelled)
689 return Cancelled;
690 return Continue;
691 }
692 /*}}}*/
693 // Acquire::Bump - Called when an item is dequeued /*{{{*/
694 // ---------------------------------------------------------------------
695 /* This routine bumps idle queues in hopes that they will be able to fetch
696 the dequeued item */
697 void pkgAcquire::Bump()
698 {
699 for (Queue *I = Queues; I != 0; I = I->Next)
700 I->Bump();
701 }
702 /*}}}*/
703 // Acquire::WorkerStep - Step to the next worker /*{{{*/
704 // ---------------------------------------------------------------------
705 /* Not inlined to advoid including acquire-worker.h */
706 pkgAcquire::Worker *pkgAcquire::WorkerStep(Worker *I)
707 {
708 return I->NextAcquire;
709 }
710 /*}}}*/
711 // Acquire::Clean - Cleans a directory /*{{{*/
712 // ---------------------------------------------------------------------
713 /* This is a bit simplistic, it looks at every file in the dir and sees
714 if it is part of the download set. */
715 bool pkgAcquire::Clean(string Dir)
716 {
717 // non-existing directories are by definition clean…
718 if (DirectoryExists(Dir) == false)
719 return true;
720
721 if(Dir == "/")
722 return _error->Error(_("Clean of %s is not supported"), Dir.c_str());
723
724 DIR *D = opendir(Dir.c_str());
725 if (D == 0)
726 return _error->Errno("opendir",_("Unable to read %s"),Dir.c_str());
727
728 string StartDir = SafeGetCWD();
729 if (chdir(Dir.c_str()) != 0)
730 {
731 closedir(D);
732 return _error->Errno("chdir",_("Unable to change to %s"),Dir.c_str());
733 }
734
735 for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D))
736 {
737 // Skip some files..
738 if (strcmp(Dir->d_name,"lock") == 0 ||
739 strcmp(Dir->d_name,"partial") == 0 ||
740 strcmp(Dir->d_name,"lost+found") == 0 ||
741 strcmp(Dir->d_name,".") == 0 ||
742 strcmp(Dir->d_name,"..") == 0)
743 continue;
744
745 // Look in the get list
746 ItemCIterator I = Items.begin();
747 for (; I != Items.end(); ++I)
748 if (flNotDir((*I)->DestFile) == Dir->d_name)
749 break;
750
751 // Nothing found, nuke it
752 if (I == Items.end())
753 RemoveFile("Clean", Dir->d_name);
754 };
755
756 closedir(D);
757 if (chdir(StartDir.c_str()) != 0)
758 return _error->Errno("chdir",_("Unable to change to %s"),StartDir.c_str());
759 return true;
760 }
761 /*}}}*/
762 // Acquire::TotalNeeded - Number of bytes to fetch /*{{{*/
763 // ---------------------------------------------------------------------
764 /* This is the total number of bytes needed */
765 APT_PURE unsigned long long pkgAcquire::TotalNeeded()
766 {
767 return std::accumulate(ItemsBegin(), ItemsEnd(), 0llu,
768 [](unsigned long long const T, Item const * const I) {
769 return T + I->FileSize;
770 });
771 }
772 /*}}}*/
773 // Acquire::FetchNeeded - Number of bytes needed to get /*{{{*/
774 // ---------------------------------------------------------------------
775 /* This is the number of bytes that is not local */
776 APT_PURE unsigned long long pkgAcquire::FetchNeeded()
777 {
778 return std::accumulate(ItemsBegin(), ItemsEnd(), 0llu,
779 [](unsigned long long const T, Item const * const I) {
780 if (I->Local == false)
781 return T + I->FileSize;
782 else
783 return T;
784 });
785 }
786 /*}}}*/
787 // Acquire::PartialPresent - Number of partial bytes we already have /*{{{*/
788 // ---------------------------------------------------------------------
789 /* This is the number of bytes that is not local */
790 APT_PURE unsigned long long pkgAcquire::PartialPresent()
791 {
792 return std::accumulate(ItemsBegin(), ItemsEnd(), 0llu,
793 [](unsigned long long const T, Item const * const I) {
794 if (I->Local == false)
795 return T + I->PartialSize;
796 else
797 return T;
798 });
799 }
800 /*}}}*/
801 // Acquire::UriBegin - Start iterator for the uri list /*{{{*/
802 // ---------------------------------------------------------------------
803 /* */
804 pkgAcquire::UriIterator pkgAcquire::UriBegin()
805 {
806 return UriIterator(Queues);
807 }
808 /*}}}*/
809 // Acquire::UriEnd - End iterator for the uri list /*{{{*/
810 // ---------------------------------------------------------------------
811 /* */
812 pkgAcquire::UriIterator pkgAcquire::UriEnd()
813 {
814 return UriIterator(0);
815 }
816 /*}}}*/
817 // Acquire::MethodConfig::MethodConfig - Constructor /*{{{*/
818 // ---------------------------------------------------------------------
819 /* */
820 pkgAcquire::MethodConfig::MethodConfig() : d(NULL), Next(0), SingleInstance(false),
821 Pipeline(false), SendConfig(false), LocalOnly(false), NeedsCleanup(false),
822 Removable(false)
823 {
824 }
825 /*}}}*/
826 // Queue::Queue - Constructor /*{{{*/
827 // ---------------------------------------------------------------------
828 /* */
829 pkgAcquire::Queue::Queue(string const &name,pkgAcquire * const owner) : d(NULL), Next(0),
830 Name(name), Items(0), Workers(0), Owner(owner), PipeDepth(0), MaxPipeDepth(1)
831 {
832 }
833 /*}}}*/
834 // Queue::~Queue - Destructor /*{{{*/
835 // ---------------------------------------------------------------------
836 /* */
837 pkgAcquire::Queue::~Queue()
838 {
839 Shutdown(true);
840
841 while (Items != 0)
842 {
843 QItem *Jnk = Items;
844 Items = Items->Next;
845 delete Jnk;
846 }
847 }
848 /*}}}*/
849 // Queue::Enqueue - Queue an item to the queue /*{{{*/
850 // ---------------------------------------------------------------------
851 /* */
852 bool pkgAcquire::Queue::Enqueue(ItemDesc &Item)
853 {
854 QItem **I = &Items;
855 // move to the end of the queue and check for duplicates here
856 for (; *I != 0; I = &(*I)->Next)
857 if (Item.URI == (*I)->URI)
858 {
859 if (_config->FindB("Debug::pkgAcquire::Worker",false) == true)
860 std::cerr << " @ Queue: Action combined for " << Item.URI << " and " << (*I)->URI << std::endl;
861 (*I)->Owners.push_back(Item.Owner);
862 Item.Owner->Status = (*I)->Owner->Status;
863 return false;
864 }
865
866 // Create a new item
867 QItem *Itm = new QItem;
868 *Itm = Item;
869 Itm->Next = 0;
870 *I = Itm;
871
872 Item.Owner->QueueCounter++;
873 if (Items->Next == 0)
874 Cycle();
875 return true;
876 }
877 /*}}}*/
878 // Queue::Dequeue - Remove an item from the queue /*{{{*/
879 // ---------------------------------------------------------------------
880 /* We return true if we hit something */
881 bool pkgAcquire::Queue::Dequeue(Item *Owner)
882 {
883 if (Owner->Status == pkgAcquire::Item::StatFetching)
884 return _error->Error("Tried to dequeue a fetching object");
885
886 bool Res = false;
887
888 QItem **I = &Items;
889 for (; *I != 0;)
890 {
891 if (Owner == (*I)->Owner)
892 {
893 QItem *Jnk= *I;
894 *I = (*I)->Next;
895 Owner->QueueCounter--;
896 delete Jnk;
897 Res = true;
898 }
899 else
900 I = &(*I)->Next;
901 }
902
903 return Res;
904 }
905 /*}}}*/
906 // Queue::Startup - Start the worker processes /*{{{*/
907 // ---------------------------------------------------------------------
908 /* It is possible for this to be called with a pre-existing set of
909 workers. */
910 bool pkgAcquire::Queue::Startup()
911 {
912 if (Workers == 0)
913 {
914 URI U(Name);
915 pkgAcquire::MethodConfig *Cnf = Owner->GetConfig(U.Access);
916 if (Cnf == 0)
917 return false;
918
919 Workers = new Worker(this,Cnf,Owner->Log);
920 Owner->Add(Workers);
921 if (Workers->Start() == false)
922 return false;
923
924 /* When pipelining we commit 10 items. This needs to change when we
925 added other source retry to have cycle maintain a pipeline depth
926 on its own. */
927 if (Cnf->Pipeline == true)
928 MaxPipeDepth = _config->FindI("Acquire::Max-Pipeline-Depth",10);
929 else
930 MaxPipeDepth = 1;
931 }
932
933 return Cycle();
934 }
935 /*}}}*/
936 // Queue::Shutdown - Shutdown the worker processes /*{{{*/
937 // ---------------------------------------------------------------------
938 /* If final is true then all workers are eliminated, otherwise only workers
939 that do not need cleanup are removed */
940 bool pkgAcquire::Queue::Shutdown(bool Final)
941 {
942 // Delete all of the workers
943 pkgAcquire::Worker **Cur = &Workers;
944 while (*Cur != 0)
945 {
946 pkgAcquire::Worker *Jnk = *Cur;
947 if (Final == true || Jnk->GetConf()->NeedsCleanup == false)
948 {
949 *Cur = Jnk->NextQueue;
950 Owner->Remove(Jnk);
951 delete Jnk;
952 }
953 else
954 Cur = &(*Cur)->NextQueue;
955 }
956
957 return true;
958 }
959 /*}}}*/
960 // Queue::FindItem - Find a URI in the item list /*{{{*/
961 // ---------------------------------------------------------------------
962 /* */
963 pkgAcquire::Queue::QItem *pkgAcquire::Queue::FindItem(string URI,pkgAcquire::Worker *Owner)
964 {
965 for (QItem *I = Items; I != 0; I = I->Next)
966 if (I->URI == URI && I->Worker == Owner)
967 return I;
968 return 0;
969 }
970 /*}}}*/
971 // Queue::ItemDone - Item has been completed /*{{{*/
972 // ---------------------------------------------------------------------
973 /* The worker signals this which causes the item to be removed from the
974 queue. If this is the last queue instance then it is removed from the
975 main queue too.*/
976 bool pkgAcquire::Queue::ItemDone(QItem *Itm)
977 {
978 PipeDepth--;
979 for (QItem::owner_iterator O = Itm->Owners.begin(); O != Itm->Owners.end(); ++O)
980 {
981 if ((*O)->Status == pkgAcquire::Item::StatFetching)
982 (*O)->Status = pkgAcquire::Item::StatDone;
983 }
984
985 if (Itm->Owner->QueueCounter <= 1)
986 Owner->Dequeue(Itm->Owner);
987 else
988 {
989 Dequeue(Itm->Owner);
990 Owner->Bump();
991 }
992
993 return Cycle();
994 }
995 /*}}}*/
996 // Queue::Cycle - Queue new items into the method /*{{{*/
997 // ---------------------------------------------------------------------
998 /* This locates a new idle item and sends it to the worker. If pipelining
999 is enabled then it keeps the pipe full. */
1000 bool pkgAcquire::Queue::Cycle()
1001 {
1002 if (Items == 0 || Workers == 0)
1003 return true;
1004
1005 if (PipeDepth < 0)
1006 return _error->Error("Pipedepth failure");
1007
1008 // Look for a queable item
1009 QItem *I = Items;
1010 while (PipeDepth < (signed)MaxPipeDepth)
1011 {
1012 for (; I != 0; I = I->Next)
1013 if (I->Owner->Status == pkgAcquire::Item::StatIdle)
1014 break;
1015
1016 // Nothing to do, queue is idle.
1017 if (I == 0)
1018 return true;
1019
1020 I->Worker = Workers;
1021 for (auto const &O: I->Owners)
1022 O->Status = pkgAcquire::Item::StatFetching;
1023 PipeDepth++;
1024 if (Workers->QueueItem(I) == false)
1025 return false;
1026 }
1027
1028 return true;
1029 }
1030 /*}}}*/
1031 // Queue::Bump - Fetch any pending objects if we are idle /*{{{*/
1032 // ---------------------------------------------------------------------
1033 /* This is called when an item in multiple queues is dequeued */
1034 void pkgAcquire::Queue::Bump()
1035 {
1036 Cycle();
1037 }
1038 /*}}}*/
1039 HashStringList pkgAcquire::Queue::QItem::GetExpectedHashes() const /*{{{*/
1040 {
1041 /* each Item can have multiple owners and each owner might have different
1042 hashes, even if that is unlikely in practice and if so at least some
1043 owners will later fail. There is one situation through which is not a
1044 failure and still needs this handling: Two owners who expect the same
1045 file, but one owner only knows the SHA1 while the other only knows SHA256. */
1046 HashStringList superhsl;
1047 for (pkgAcquire::Queue::QItem::owner_iterator O = Owners.begin(); O != Owners.end(); ++O)
1048 {
1049 HashStringList const hsl = (*O)->GetExpectedHashes();
1050 if (hsl.usable() == false)
1051 continue;
1052 if (superhsl.usable() == false)
1053 superhsl = hsl;
1054 else
1055 {
1056 // we merge both lists - if we find disagreement send no hashes
1057 HashStringList::const_iterator hs = hsl.begin();
1058 for (; hs != hsl.end(); ++hs)
1059 if (superhsl.push_back(*hs) == false)
1060 break;
1061 if (hs != hsl.end())
1062 {
1063 superhsl.clear();
1064 break;
1065 }
1066 }
1067 }
1068 return superhsl;
1069 }
1070 /*}}}*/
1071 APT_PURE unsigned long long pkgAcquire::Queue::QItem::GetMaximumSize() const /*{{{*/
1072 {
1073 unsigned long long Maximum = std::numeric_limits<unsigned long long>::max();
1074 for (auto const &O: Owners)
1075 {
1076 if (O->FileSize == 0)
1077 continue;
1078 Maximum = std::min(Maximum, O->FileSize);
1079 }
1080 if (Maximum == std::numeric_limits<unsigned long long>::max())
1081 return 0;
1082 return Maximum;
1083 }
1084 /*}}}*/
1085 void pkgAcquire::Queue::QItem::SyncDestinationFiles() const /*{{{*/
1086 {
1087 /* ensure that the first owner has the best partial file of all and
1088 the rest have (potentially dangling) symlinks to it so that
1089 everything (like progress reporting) finds it easily */
1090 std::string superfile = Owner->DestFile;
1091 off_t supersize = 0;
1092 for (pkgAcquire::Queue::QItem::owner_iterator O = Owners.begin(); O != Owners.end(); ++O)
1093 {
1094 if ((*O)->DestFile == superfile)
1095 continue;
1096 struct stat file;
1097 if (lstat((*O)->DestFile.c_str(),&file) == 0)
1098 {
1099 if ((file.st_mode & S_IFREG) == 0)
1100 RemoveFile("SyncDestinationFiles", (*O)->DestFile);
1101 else if (supersize < file.st_size)
1102 {
1103 supersize = file.st_size;
1104 RemoveFile("SyncDestinationFiles", superfile);
1105 rename((*O)->DestFile.c_str(), superfile.c_str());
1106 }
1107 else
1108 RemoveFile("SyncDestinationFiles", (*O)->DestFile);
1109 if (symlink(superfile.c_str(), (*O)->DestFile.c_str()) != 0)
1110 {
1111 ; // not a problem per-se and no real alternative
1112 }
1113 }
1114 }
1115 }
1116 /*}}}*/
1117 std::string pkgAcquire::Queue::QItem::Custom600Headers() const /*{{{*/
1118 {
1119 /* The others are relatively easy to merge, but this one?
1120 Lets not merge and see how far we can run with it…
1121 Likely, nobody will ever notice as all the items will
1122 be of the same class and hence generate the same headers. */
1123 return Owner->Custom600Headers();
1124 }
1125 /*}}}*/
1126
1127 // AcquireStatus::pkgAcquireStatus - Constructor /*{{{*/
1128 // ---------------------------------------------------------------------
1129 /* */
1130 pkgAcquireStatus::pkgAcquireStatus() : d(NULL), Percent(-1), Update(true), MorePulses(false)
1131 {
1132 Start();
1133 }
1134 /*}}}*/
1135 // AcquireStatus::Pulse - Called periodically /*{{{*/
1136 // ---------------------------------------------------------------------
1137 /* This computes some internal state variables for the derived classes to
1138 use. It generates the current downloaded bytes and total bytes to download
1139 as well as the current CPS estimate. */
1140 bool pkgAcquireStatus::Pulse(pkgAcquire *Owner)
1141 {
1142 TotalBytes = 0;
1143 CurrentBytes = 0;
1144 TotalItems = 0;
1145 CurrentItems = 0;
1146
1147 // Compute the total number of bytes to fetch
1148 unsigned int Unknown = 0;
1149 unsigned int Count = 0;
1150 bool UnfetchedReleaseFiles = false;
1151 for (pkgAcquire::ItemCIterator I = Owner->ItemsBegin();
1152 I != Owner->ItemsEnd();
1153 ++I, ++Count)
1154 {
1155 TotalItems++;
1156 if ((*I)->Status == pkgAcquire::Item::StatDone)
1157 ++CurrentItems;
1158
1159 // see if the method tells us to expect more
1160 TotalItems += (*I)->ExpectedAdditionalItems;
1161
1162 // check if there are unfetched Release files
1163 if ((*I)->Status != pkgAcquire::Item::StatDone && (*I)->ExpectedAdditionalItems > 0)
1164 UnfetchedReleaseFiles = true;
1165
1166 TotalBytes += (*I)->FileSize;
1167 if ((*I)->Complete == true)
1168 CurrentBytes += (*I)->FileSize;
1169 if ((*I)->FileSize == 0 && (*I)->Complete == false)
1170 ++Unknown;
1171 }
1172
1173 // Compute the current completion
1174 unsigned long long ResumeSize = 0;
1175 for (pkgAcquire::Worker *I = Owner->WorkersBegin(); I != 0;
1176 I = Owner->WorkerStep(I))
1177 {
1178 if (I->CurrentItem != 0 && I->CurrentItem->Owner->Complete == false)
1179 {
1180 CurrentBytes += I->CurrentSize;
1181 ResumeSize += I->ResumePoint;
1182
1183 // Files with unknown size always have 100% completion
1184 if (I->CurrentItem->Owner->FileSize == 0 &&
1185 I->CurrentItem->Owner->Complete == false)
1186 TotalBytes += I->CurrentSize;
1187 }
1188 }
1189
1190 // Normalize the figures and account for unknown size downloads
1191 if (TotalBytes <= 0)
1192 TotalBytes = 1;
1193 if (Unknown == Count)
1194 TotalBytes = Unknown;
1195
1196 // Wha?! Is not supposed to happen.
1197 if (CurrentBytes > TotalBytes)
1198 CurrentBytes = TotalBytes;
1199
1200 // Compute the CPS
1201 struct timeval NewTime;
1202 gettimeofday(&NewTime,0);
1203 if ((NewTime.tv_sec - Time.tv_sec == 6 && NewTime.tv_usec > Time.tv_usec) ||
1204 NewTime.tv_sec - Time.tv_sec > 6)
1205 {
1206 double Delta = NewTime.tv_sec - Time.tv_sec +
1207 (NewTime.tv_usec - Time.tv_usec)/1000000.0;
1208
1209 // Compute the CPS value
1210 if (Delta < 0.01)
1211 CurrentCPS = 0;
1212 else
1213 CurrentCPS = ((CurrentBytes - ResumeSize) - LastBytes)/Delta;
1214 LastBytes = CurrentBytes - ResumeSize;
1215 ElapsedTime = (unsigned long long)Delta;
1216 Time = NewTime;
1217 }
1218
1219 double const OldPercent = Percent;
1220 // calculate the percentage, if we have too little data assume 1%
1221 if (TotalBytes > 0 && UnfetchedReleaseFiles)
1222 Percent = 0;
1223 else
1224 // use both files and bytes because bytes can be unreliable
1225 Percent = (0.8 * (CurrentBytes/float(TotalBytes)*100.0) +
1226 0.2 * (CurrentItems/float(TotalItems)*100.0));
1227
1228 // debug
1229 if (_config->FindB("Debug::acquire::progress", false) == true)
1230 {
1231 std::clog
1232 << "["
1233 << std::setw(5) << std::setprecision(4) << std::showpoint << Percent
1234 << "]"
1235 << " Bytes: "
1236 << SizeToStr(CurrentBytes) << " / " << SizeToStr(TotalBytes)
1237 << " # Files: "
1238 << CurrentItems << " / " << TotalItems
1239 << std::endl;
1240 }
1241
1242 double const DiffPercent = Percent - OldPercent;
1243 if (DiffPercent < 0.001 && _config->FindB("Acquire::Progress::Diffpercent", false) == true)
1244 return true;
1245
1246 int fd = _config->FindI("APT::Status-Fd",-1);
1247 if(fd > 0)
1248 {
1249 ostringstream status;
1250
1251 char msg[200];
1252 long i = CurrentItems < TotalItems ? CurrentItems + 1 : CurrentItems;
1253 unsigned long long ETA = 0;
1254 if(CurrentCPS > 0)
1255 ETA = (TotalBytes - CurrentBytes) / CurrentCPS;
1256
1257 // only show the ETA if it makes sense
1258 if (ETA > 0 && ETA < 172800 /* two days */ )
1259 snprintf(msg,sizeof(msg), _("Retrieving file %li of %li (%s remaining)"), i, TotalItems, TimeToStr(ETA).c_str());
1260 else
1261 snprintf(msg,sizeof(msg), _("Retrieving file %li of %li"), i, TotalItems);
1262
1263 // build the status str
1264 status << "dlstatus:" << i
1265 << ":" << std::setprecision(3) << Percent
1266 << ":" << msg
1267 << endl;
1268
1269 std::string const dlstatus = status.str();
1270 FileFd::Write(fd, dlstatus.c_str(), dlstatus.size());
1271 }
1272
1273 return true;
1274 }
1275 /*}}}*/
1276 // AcquireStatus::Start - Called when the download is started /*{{{*/
1277 // ---------------------------------------------------------------------
1278 /* We just reset the counters */
1279 void pkgAcquireStatus::Start()
1280 {
1281 gettimeofday(&Time,0);
1282 gettimeofday(&StartTime,0);
1283 LastBytes = 0;
1284 CurrentCPS = 0;
1285 CurrentBytes = 0;
1286 TotalBytes = 0;
1287 FetchedBytes = 0;
1288 ElapsedTime = 0;
1289 TotalItems = 0;
1290 CurrentItems = 0;
1291 }
1292 /*}}}*/
1293 // AcquireStatus::Stop - Finished downloading /*{{{*/
1294 // ---------------------------------------------------------------------
1295 /* This accurately computes the elapsed time and the total overall CPS. */
1296 void pkgAcquireStatus::Stop()
1297 {
1298 // Compute the CPS and elapsed time
1299 struct timeval NewTime;
1300 gettimeofday(&NewTime,0);
1301
1302 double Delta = NewTime.tv_sec - StartTime.tv_sec +
1303 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
1304
1305 // Compute the CPS value
1306 if (Delta < 0.01)
1307 CurrentCPS = 0;
1308 else
1309 CurrentCPS = FetchedBytes/Delta;
1310 LastBytes = CurrentBytes;
1311 ElapsedTime = (unsigned long long)Delta;
1312 }
1313 /*}}}*/
1314 // AcquireStatus::Fetched - Called when a byte set has been fetched /*{{{*/
1315 // ---------------------------------------------------------------------
1316 /* This is used to get accurate final transfer rate reporting. */
1317 void pkgAcquireStatus::Fetched(unsigned long long Size,unsigned long long Resume)
1318 {
1319 FetchedBytes += Size - Resume;
1320 }
1321 /*}}}*/
1322
1323 pkgAcquire::UriIterator::UriIterator(pkgAcquire::Queue *Q) : d(NULL), CurQ(Q), CurItem(0)
1324 {
1325 while (CurItem == 0 && CurQ != 0)
1326 {
1327 CurItem = CurQ->Items;
1328 CurQ = CurQ->Next;
1329 }
1330 }
1331
1332 APT_CONST pkgAcquire::UriIterator::~UriIterator() {}
1333 APT_CONST pkgAcquire::MethodConfig::~MethodConfig() {}
1334 APT_CONST pkgAcquireStatus::~pkgAcquireStatus() {}