]> git.saurik.com Git - apt.git/blame - apt-pkg/acquire.cc
detect redirection loops in acquire instead of workers
[apt.git] / apt-pkg / acquire.cc
CommitLineData
0118833a
AL
1// -*- mode: cpp; mode: fold -*-
2// Description /*{{{*/
1b480911 3// $Id: acquire.cc,v 1.50 2004/03/17 05:17:11 mdz Exp $
0118833a
AL
4/* ######################################################################
5
6 Acquire - File Acquiration
7
1e3f4083 8 The core element for the schedule system is the concept of a named
0a8a80e5 9 queue. Each queue is unique and each queue has a name derived from the
1e3f4083 10 URI. The degree of paralization can be controlled by how the queue
0a8a80e5
AL
11 name is derived from the URI.
12
0118833a
AL
13 ##################################################################### */
14 /*}}}*/
15// Include Files /*{{{*/
ea542140
DK
16#include <config.h>
17
0118833a
AL
18#include <apt-pkg/acquire.h>
19#include <apt-pkg/acquire-item.h>
20#include <apt-pkg/acquire-worker.h>
0a8a80e5
AL
21#include <apt-pkg/configuration.h>
22#include <apt-pkg/error.h>
cdcc6d34 23#include <apt-pkg/strutl.h>
1cd1c398 24#include <apt-pkg/fileutl.h>
8267fe24 25
08ea7806 26#include <algorithm>
ae732225 27#include <numeric>
453b82a3
DK
28#include <string>
29#include <vector>
b4fc9b6f 30#include <iostream>
ac7f8f79
MV
31#include <sstream>
32#include <iomanip>
514a25cb 33#include <memory>
04a54261 34
526334a0 35#include <stdio.h>
453b82a3
DK
36#include <stdlib.h>
37#include <string.h>
38#include <unistd.h>
7c8206bf 39#include <fcntl.h>
04a54261
DK
40#include <pwd.h>
41#include <grp.h>
7a7fa5f0 42#include <dirent.h>
8267fe24 43#include <sys/time.h>
453b82a3 44#include <sys/select.h>
524f8105 45#include <errno.h>
56472095 46#include <sys/stat.h>
ea542140
DK
47
48#include <apti18n.h>
0118833a
AL
49 /*}}}*/
50
b4fc9b6f
AL
51using namespace std;
52
0118833a
AL
53// Acquire::pkgAcquire - Constructor /*{{{*/
54// ---------------------------------------------------------------------
93bf083d 55/* We grab some runtime state from the configuration space */
6c55f07a 56pkgAcquire::pkgAcquire() : LockFD(-1), d(NULL), Queues(0), Workers(0), Configs(0), Log(NULL), ToFetch(0),
1cd1c398 57 Debug(_config->FindB("Debug::pkgAcquire",false)),
5efbd596 58 Running(false)
0118833a 59{
03aa0847 60 Initialize();
1cd1c398 61}
6c55f07a 62pkgAcquire::pkgAcquire(pkgAcquireStatus *Progress) : LockFD(-1), d(NULL), Queues(0), Workers(0),
04a54261 63 Configs(0), Log(NULL), ToFetch(0),
1cd1c398 64 Debug(_config->FindB("Debug::pkgAcquire",false)),
5efbd596 65 Running(false)
03aa0847
DK
66{
67 Initialize();
68 SetLog(Progress);
69}
70void pkgAcquire::Initialize()
1cd1c398
DK
71{
72 string const Mode = _config->Find("Acquire::Queue-Mode","host");
73 if (strcasecmp(Mode.c_str(),"host") == 0)
74 QueueMode = QueueHost;
75 if (strcasecmp(Mode.c_str(),"access") == 0)
76 QueueMode = QueueAccess;
03aa0847
DK
77
78 // chown the auth.conf file as it will be accessed by our methods
79 std::string const SandboxUser = _config->Find("APT::Sandbox::User");
514a25cb 80 if (getuid() == 0 && SandboxUser.empty() == false && SandboxUser != "root") // if we aren't root, we can't chown, so don't try it
03aa0847
DK
81 {
82 struct passwd const * const pw = getpwnam(SandboxUser.c_str());
83 struct group const * const gr = getgrnam("root");
84 if (pw != NULL && gr != NULL)
85 {
86 std::string const AuthConf = _config->FindFile("Dir::Etc::netrc");
87 if(AuthConf.empty() == false && RealFileExists(AuthConf) &&
88 chown(AuthConf.c_str(), pw->pw_uid, gr->gr_gid) != 0)
89 _error->WarningE("SetupAPTPartialDirectory", "chown to %s:root of file %s failed", SandboxUser.c_str(), AuthConf.c_str());
90 }
91 }
1cd1c398
DK
92}
93 /*}}}*/
04a54261
DK
94// Acquire::GetLock - lock directory and prepare for action /*{{{*/
95static bool SetupAPTPartialDirectory(std::string const &grand, std::string const &parent)
1cd1c398 96{
04a54261 97 std::string const partial = parent + "partial";
8fe964f1
DK
98 mode_t const mode = umask(S_IWGRP | S_IWOTH);
99 bool const creation_fail = (CreateAPTDirectoryIfNeeded(grand, partial) == false &&
100 CreateAPTDirectoryIfNeeded(parent, partial) == false);
101 umask(mode);
102 if (creation_fail == true)
04a54261 103 return false;
0a8a80e5 104
03aa0847 105 std::string const SandboxUser = _config->Find("APT::Sandbox::User");
514a25cb 106 if (getuid() == 0 && SandboxUser.empty() == false && SandboxUser != "root") // if we aren't root, we can't chown, so don't try it
04a54261 107 {
03aa0847
DK
108 struct passwd const * const pw = getpwnam(SandboxUser.c_str());
109 struct group const * const gr = getgrnam("root");
1924b1e5
MV
110 if (pw != NULL && gr != NULL)
111 {
112 // chown the partial dir
113 if(chown(partial.c_str(), pw->pw_uid, gr->gr_gid) != 0)
114 _error->WarningE("SetupAPTPartialDirectory", "chown to %s:root of directory %s failed", SandboxUser.c_str(), partial.c_str());
1924b1e5 115 }
04a54261
DK
116 }
117 if (chmod(partial.c_str(), 0700) != 0)
118 _error->WarningE("SetupAPTPartialDirectory", "chmod 0700 of directory %s failed", partial.c_str());
119
120 return true;
121}
122bool pkgAcquire::Setup(pkgAcquireStatus *Progress, string const &Lock)
123{
124 Log = Progress;
125 if (Lock.empty())
43acd019
DK
126 {
127 string const listDir = _config->FindDir("Dir::State::lists");
04a54261
DK
128 if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir) == false)
129 return _error->Errno("Acquire", _("List directory %spartial is missing."), listDir.c_str());
43acd019 130 string const archivesDir = _config->FindDir("Dir::Cache::Archives");
04a54261
DK
131 if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir) == false)
132 return _error->Errno("Acquire", _("Archives directory %spartial is missing."), archivesDir.c_str());
133 return true;
134 }
135 return GetLock(Lock);
136}
137bool pkgAcquire::GetLock(std::string const &Lock)
138{
139 if (Lock.empty() == true)
140 return false;
9c2c9c24 141
04a54261
DK
142 // check for existence and possibly create auxiliary directories
143 string const listDir = _config->FindDir("Dir::State::lists");
144 string const archivesDir = _config->FindDir("Dir::Cache::Archives");
9c2c9c24 145
04a54261
DK
146 if (Lock == listDir)
147 {
148 if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir) == false)
149 return _error->Errno("Acquire", _("List directory %spartial is missing."), listDir.c_str());
150 }
151 if (Lock == archivesDir)
152 {
153 if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir) == false)
43acd019
DK
154 return _error->Errno("Acquire", _("Archives directory %spartial is missing."), archivesDir.c_str());
155 }
1cd1c398 156
04a54261 157 if (_config->FindB("Debug::NoLocking", false) == true)
1cd1c398
DK
158 return true;
159
160 // Lock the directory this acquire object will work in
7e04a6bf
DK
161 if (LockFD != -1)
162 close(LockFD);
04a54261 163 LockFD = ::GetLock(flCombine(Lock, "lock"));
1cd1c398
DK
164 if (LockFD == -1)
165 return _error->Error(_("Unable to lock directory %s"), Lock.c_str());
166
167 return true;
168}
169 /*}}}*/
0118833a
AL
170// Acquire::~pkgAcquire - Destructor /*{{{*/
171// ---------------------------------------------------------------------
93bf083d 172/* Free our memory, clean up the queues (destroy the workers) */
0118833a
AL
173pkgAcquire::~pkgAcquire()
174{
459681d3 175 Shutdown();
1cd1c398
DK
176
177 if (LockFD != -1)
178 close(LockFD);
179
3b5421b4
AL
180 while (Configs != 0)
181 {
182 MethodConfig *Jnk = Configs;
183 Configs = Configs->Next;
184 delete Jnk;
185 }
281daf46
AL
186}
187 /*}}}*/
8e5fc8f5 188// Acquire::Shutdown - Clean out the acquire object /*{{{*/
281daf46
AL
189// ---------------------------------------------------------------------
190/* */
191void pkgAcquire::Shutdown()
192{
f7f0d6c7 193 while (Items.empty() == false)
1b480911
AL
194 {
195 if (Items[0]->Status == Item::StatFetching)
196 Items[0]->Status = Item::StatError;
281daf46 197 delete Items[0];
1b480911 198 }
0a8a80e5
AL
199
200 while (Queues != 0)
201 {
202 Queue *Jnk = Queues;
203 Queues = Queues->Next;
204 delete Jnk;
205 }
0118833a
AL
206}
207 /*}}}*/
208// Acquire::Add - Add a new item /*{{{*/
209// ---------------------------------------------------------------------
93bf083d
AL
210/* This puts an item on the acquire list. This list is mainly for tracking
211 item status */
0118833a
AL
212void pkgAcquire::Add(Item *Itm)
213{
214 Items.push_back(Itm);
215}
216 /*}}}*/
217// Acquire::Remove - Remove a item /*{{{*/
218// ---------------------------------------------------------------------
93bf083d 219/* Remove an item from the acquire list. This is usually not used.. */
0118833a
AL
220void pkgAcquire::Remove(Item *Itm)
221{
a3eaf954
AL
222 Dequeue(Itm);
223
753b3525 224 for (ItemIterator I = Items.begin(); I != Items.end();)
0118833a
AL
225 {
226 if (*I == Itm)
b4fc9b6f 227 {
0118833a 228 Items.erase(I);
b4fc9b6f
AL
229 I = Items.begin();
230 }
753b3525 231 else
f7f0d6c7 232 ++I;
8267fe24 233 }
0118833a
AL
234}
235 /*}}}*/
0a8a80e5
AL
236// Acquire::Add - Add a worker /*{{{*/
237// ---------------------------------------------------------------------
93bf083d
AL
238/* A list of workers is kept so that the select loop can direct their FD
239 usage. */
0a8a80e5
AL
240void pkgAcquire::Add(Worker *Work)
241{
242 Work->NextAcquire = Workers;
243 Workers = Work;
244}
245 /*}}}*/
246// Acquire::Remove - Remove a worker /*{{{*/
247// ---------------------------------------------------------------------
93bf083d
AL
248/* A worker has died. This can not be done while the select loop is running
249 as it would require that RunFds could handling a changing list state and
1e3f4083 250 it can't.. */
0a8a80e5
AL
251void pkgAcquire::Remove(Worker *Work)
252{
93bf083d
AL
253 if (Running == true)
254 abort();
255
0a8a80e5
AL
256 Worker **I = &Workers;
257 for (; *I != 0;)
258 {
259 if (*I == Work)
260 *I = (*I)->NextAcquire;
261 else
262 I = &(*I)->NextAcquire;
263 }
264}
265 /*}}}*/
0118833a
AL
266// Acquire::Enqueue - Queue an URI for fetching /*{{{*/
267// ---------------------------------------------------------------------
93bf083d 268/* This is the entry point for an item. An item calls this function when
281daf46 269 it is constructed which creates a queue (based on the current queue
93bf083d
AL
270 mode) and puts the item in that queue. If the system is running then
271 the queue might be started. */
8267fe24 272void pkgAcquire::Enqueue(ItemDesc &Item)
0118833a 273{
0a8a80e5 274 // Determine which queue to put the item in
e331f6ed
AL
275 const MethodConfig *Config;
276 string Name = QueueName(Item.URI,Config);
0a8a80e5
AL
277 if (Name.empty() == true)
278 return;
279
280 // Find the queue structure
281 Queue *I = Queues;
282 for (; I != 0 && I->Name != Name; I = I->Next);
283 if (I == 0)
284 {
285 I = new Queue(Name,this);
286 I->Next = Queues;
287 Queues = I;
93bf083d
AL
288
289 if (Running == true)
290 I->Startup();
0a8a80e5 291 }
bfd22fc0 292
e331f6ed
AL
293 // See if this is a local only URI
294 if (Config->LocalOnly == true && Item.Owner->Complete == false)
295 Item.Owner->Local = true;
8267fe24 296 Item.Owner->Status = Item::StatIdle;
0a8a80e5
AL
297
298 // Queue it into the named queue
c03462c6
MV
299 if(I->Enqueue(Item))
300 ToFetch++;
301
0a8a80e5
AL
302 // Some trace stuff
303 if (Debug == true)
304 {
8267fe24
AL
305 clog << "Fetching " << Item.URI << endl;
306 clog << " to " << Item.Owner->DestFile << endl;
e331f6ed 307 clog << " Queue is: " << Name << endl;
0a8a80e5 308 }
3b5421b4
AL
309}
310 /*}}}*/
0a8a80e5 311// Acquire::Dequeue - Remove an item from all queues /*{{{*/
3b5421b4 312// ---------------------------------------------------------------------
93bf083d
AL
313/* This is called when an item is finished being fetched. It removes it
314 from all the queues */
0a8a80e5
AL
315void pkgAcquire::Dequeue(Item *Itm)
316{
317 Queue *I = Queues;
bfd22fc0 318 bool Res = false;
93bf083d
AL
319 if (Debug == true)
320 clog << "Dequeuing " << Itm->DestFile << endl;
5674f6b3
RG
321
322 for (; I != 0; I = I->Next)
323 {
324 if (I->Dequeue(Itm))
325 {
326 Res = true;
327 if (Debug == true)
328 clog << "Dequeued from " << I->Name << endl;
329 }
330 }
331
bfd22fc0
AL
332 if (Res == true)
333 ToFetch--;
0a8a80e5
AL
334}
335 /*}}}*/
336// Acquire::QueueName - Return the name of the queue for this URI /*{{{*/
337// ---------------------------------------------------------------------
338/* The string returned depends on the configuration settings and the
339 method parameters. Given something like http://foo.org/bar it can
340 return http://foo.org or http */
e331f6ed 341string pkgAcquire::QueueName(string Uri,MethodConfig const *&Config)
3b5421b4 342{
93bf083d
AL
343 URI U(Uri);
344
e331f6ed 345 Config = GetConfig(U.Access);
0a8a80e5
AL
346 if (Config == 0)
347 return string();
348
349 /* Single-Instance methods get exactly one queue per URI. This is
350 also used for the Access queue method */
351 if (Config->SingleInstance == true || QueueMode == QueueAccess)
5674f6b3
RG
352 return U.Access;
353
b89cd2e3 354 string AccessSchema = U.Access + ':';
4aa6ebf6 355 string FullQueueName;
b89cd2e3
JAK
356
357 if (U.Host.empty())
358 {
4aa6ebf6
DK
359 long existing = 0;
360 // check how many queues exist already and reuse empty ones
361 for (Queue const *I = Queues; I != 0; I = I->Next)
362 if (I->Name.compare(0, AccessSchema.length(), AccessSchema) == 0)
363 {
364 if (I->Items == nullptr)
365 return I->Name;
366 ++existing;
367 }
368
b89cd2e3
JAK
369#ifdef _SC_NPROCESSORS_ONLN
370 long cpuCount = sysconf(_SC_NPROCESSORS_ONLN) * 2;
371#else
4aa6ebf6 372 long cpuCount = 10;
b89cd2e3 373#endif
4aa6ebf6 374 cpuCount = _config->FindI("Acquire::QueueHost::Limit", cpuCount);
b89cd2e3 375
4aa6ebf6
DK
376 if (cpuCount <= 0 || existing < cpuCount)
377 strprintf(FullQueueName, "%s%ld", AccessSchema.c_str(), existing);
378 else
379 {
380 long const randomQueue = random() % cpuCount;
381 strprintf(FullQueueName, "%s%ld", AccessSchema.c_str(), randomQueue);
b89cd2e3 382 }
4aa6ebf6
DK
383
384 if (Debug)
385 clog << "Chose random queue " << FullQueueName << " for " << Uri << endl;
b89cd2e3
JAK
386 } else
387 {
388 FullQueueName = AccessSchema + U.Host;
389 }
5674f6b3
RG
390 unsigned int Instances = 0, SchemaLength = AccessSchema.length();
391
392 Queue *I = Queues;
393 for (; I != 0; I = I->Next) {
394 // if the queue already exists, re-use it
395 if (I->Name == FullQueueName)
396 return FullQueueName;
397
398 if (I->Name.compare(0, SchemaLength, AccessSchema) == 0)
399 Instances++;
400 }
401
402 if (Debug) {
403 clog << "Found " << Instances << " instances of " << U.Access << endl;
404 }
405
406 if (Instances >= (unsigned int)_config->FindI("Acquire::QueueHost::Limit",10))
407 return U.Access;
93bf083d 408
5674f6b3 409 return FullQueueName;
0118833a
AL
410}
411 /*}}}*/
3b5421b4
AL
412// Acquire::GetConfig - Fetch the configuration information /*{{{*/
413// ---------------------------------------------------------------------
414/* This locates the configuration structure for an access method. If
415 a config structure cannot be found a Worker will be created to
416 retrieve it */
0a8a80e5 417pkgAcquire::MethodConfig *pkgAcquire::GetConfig(string Access)
3b5421b4
AL
418{
419 // Search for an existing config
420 MethodConfig *Conf;
421 for (Conf = Configs; Conf != 0; Conf = Conf->Next)
422 if (Conf->Access == Access)
423 return Conf;
424
425 // Create the new config class
426 Conf = new MethodConfig;
427 Conf->Access = Access;
428 Conf->Next = Configs;
429 Configs = Conf;
0118833a 430
3b5421b4
AL
431 // Create the worker to fetch the configuration
432 Worker Work(Conf);
433 if (Work.Start() == false)
434 return 0;
7c6e2dc7
MV
435
436 /* if a method uses DownloadLimit, we switch to SingleInstance mode */
4b65cc13 437 if(_config->FindI("Acquire::"+Access+"::Dl-Limit",0) > 0)
7c6e2dc7
MV
438 Conf->SingleInstance = true;
439
3b5421b4
AL
440 return Conf;
441}
442 /*}}}*/
0a8a80e5
AL
443// Acquire::SetFds - Deal with readable FDs /*{{{*/
444// ---------------------------------------------------------------------
445/* Collect FDs that have activity monitors into the fd sets */
446void pkgAcquire::SetFds(int &Fd,fd_set *RSet,fd_set *WSet)
447{
448 for (Worker *I = Workers; I != 0; I = I->NextAcquire)
449 {
450 if (I->InReady == true && I->InFd >= 0)
451 {
452 if (Fd < I->InFd)
453 Fd = I->InFd;
454 FD_SET(I->InFd,RSet);
455 }
456 if (I->OutReady == true && I->OutFd >= 0)
457 {
458 if (Fd < I->OutFd)
459 Fd = I->OutFd;
460 FD_SET(I->OutFd,WSet);
461 }
462 }
463}
464 /*}}}*/
a416a90b
MV
465// Acquire::RunFds - compatibility remove on next abi/api break /*{{{*/
466void pkgAcquire::RunFds(fd_set *RSet,fd_set *WSet)
467{
468 RunFdsSane(RSet, WSet);
124e6916 469}
a416a90b
MV
470 /*}}}*/
471// Acquire::RunFdsSane - Deal with active FDs /*{{{*/
0a8a80e5 472// ---------------------------------------------------------------------
93bf083d
AL
473/* Dispatch active FDs over to the proper workers. It is very important
474 that a worker never be erased while this is running! The queue class
475 should never erase a worker except during shutdown processing. */
a416a90b 476bool pkgAcquire::RunFdsSane(fd_set *RSet,fd_set *WSet)
0a8a80e5 477{
a416a90b
MV
478 bool Res = true;
479
0a8a80e5
AL
480 for (Worker *I = Workers; I != 0; I = I->NextAcquire)
481 {
482 if (I->InFd >= 0 && FD_ISSET(I->InFd,RSet) != 0)
a416a90b 483 Res &= I->InFdReady();
0a8a80e5 484 if (I->OutFd >= 0 && FD_ISSET(I->OutFd,WSet) != 0)
a416a90b 485 Res &= I->OutFdReady();
0a8a80e5 486 }
a416a90b
MV
487
488 return Res;
0a8a80e5
AL
489}
490 /*}}}*/
491// Acquire::Run - Run the fetch sequence /*{{{*/
492// ---------------------------------------------------------------------
493/* This runs the queues. It manages a select loop for all of the
494 Worker tasks. The workers interact with the queues and items to
495 manage the actual fetch. */
514a25cb
DK
496static bool IsAccessibleBySandboxUser(std::string const &filename, bool const ReadWrite)
497{
498 // you would think this is easily to answer with faccessat, right? Wrong!
499 // It e.g. gets groups wrong, so the only thing which works reliable is trying
500 // to open the file we want to open later on…
501 if (unlikely(filename.empty()))
502 return true;
503
504 if (ReadWrite == false)
505 {
506 errno = 0;
507 // can we read a file? Note that non-existing files are "fine"
508 int const fd = open(filename.c_str(), O_RDONLY | O_CLOEXEC);
509 if (fd == -1 && errno == EACCES)
510 return false;
511 close(fd);
512 return true;
513 }
514 else
515 {
516 // the file might not exist yet and even if it does we will fix permissions,
517 // so important is here just that the directory it is in allows that
518 std::string const dirname = flNotFile(filename);
519 if (unlikely(dirname.empty()))
520 return true;
521
522 char const * const filetag = ".apt-acquire-privs-test.XXXXXX";
523 std::string const tmpfile_tpl = flCombine(dirname, filetag);
524 std::unique_ptr<char, decltype(std::free) *> tmpfile { strdup(tmpfile_tpl.c_str()), std::free };
525 int const fd = mkstemp(tmpfile.get());
526 if (fd == -1 && errno == EACCES)
527 return false;
528 RemoveFile("IsAccessibleBySandboxUser", tmpfile.get());
529 close(fd);
530 return true;
531 }
532}
7c8206bf
DK
533static void CheckDropPrivsMustBeDisabled(pkgAcquire const &Fetcher)
534{
535 if(getuid() != 0)
536 return;
537
514a25cb
DK
538 std::string const SandboxUser = _config->Find("APT::Sandbox::User");
539 if (SandboxUser.empty() || SandboxUser == "root")
7c8206bf
DK
540 return;
541
542 struct passwd const * const pw = getpwnam(SandboxUser.c_str());
543 if (pw == NULL)
ef39c148
MV
544 {
545 _error->Warning(_("No sandbox user '%s' on the system, can not drop privileges"), SandboxUser.c_str());
546 _config->Set("APT::Sandbox::User", "");
7c8206bf 547 return;
ef39c148 548 }
7c8206bf 549
226c0f64
DK
550 gid_t const old_euid = geteuid();
551 gid_t const old_egid = getegid();
514a25cb
DK
552
553 long const ngroups_max = sysconf(_SC_NGROUPS_MAX);
554 std::unique_ptr<gid_t[]> old_gidlist(new gid_t[ngroups_max]);
555 if (unlikely(old_gidlist == NULL))
556 return;
557 ssize_t old_gidlist_nr;
558 if ((old_gidlist_nr = getgroups(ngroups_max, old_gidlist.get())) < 0)
559 {
560 _error->FatalE("getgroups", "getgroups %lu failed", ngroups_max);
561 old_gidlist[0] = 0;
562 old_gidlist_nr = 1;
563 }
564 if (setgroups(1, &pw->pw_gid))
565 _error->FatalE("setgroups", "setgroups %u failed", pw->pw_gid);
566
7c8206bf 567 if (setegid(pw->pw_gid) != 0)
514a25cb 568 _error->FatalE("setegid", "setegid %u failed", pw->pw_gid);
7c8206bf 569 if (seteuid(pw->pw_uid) != 0)
514a25cb 570 _error->FatalE("seteuid", "seteuid %u failed", pw->pw_uid);
7c8206bf 571
7c8206bf 572 for (pkgAcquire::ItemCIterator I = Fetcher.ItemsBegin();
69b76544 573 I != Fetcher.ItemsEnd(); ++I)
7c8206bf 574 {
226c0f64 575 // no need to drop privileges for a complete file
6fd4b4c0 576 if ((*I)->Complete == true || (*I)->Status != pkgAcquire::Item::StatIdle)
7c8206bf
DK
577 continue;
578
514a25cb
DK
579 // if destination file is inaccessible all hope is lost for privilege dropping
580 if (IsAccessibleBySandboxUser((*I)->DestFile, true) == false)
7c8206bf 581 {
7c8206bf 582 _error->WarningE("pkgAcquire::Run", _("Can't drop privileges for downloading as file '%s' couldn't be accessed by user '%s'."),
514a25cb 583 (*I)->DestFile.c_str(), SandboxUser.c_str());
7c8206bf
DK
584 _config->Set("APT::Sandbox::User", "");
585 break;
586 }
514a25cb
DK
587
588 // if its the source file (e.g. local sources) we might be lucky
589 // by dropping the dropping only for some methods.
590 URI const source = (*I)->DescURI();
591 if (source.Access == "file" || source.Access == "copy")
592 {
593 std::string const conf = "Binary::" + source.Access + "::APT::Sandbox::User";
594 if (_config->Exists(conf) == true)
595 continue;
596
597 if (IsAccessibleBySandboxUser(source.Path, false) == false)
598 {
599 _error->NoticeE("pkgAcquire::Run", _("Can't drop privileges for downloading as file '%s' couldn't be accessed by user '%s'."),
600 source.Path.c_str(), SandboxUser.c_str());
601 _config->CndSet("Binary::file::APT::Sandbox::User", "root");
602 _config->CndSet("Binary::copy::APT::Sandbox::User", "root");
603 }
604 }
7c8206bf
DK
605 }
606
226c0f64 607 if (seteuid(old_euid) != 0)
514a25cb 608 _error->FatalE("seteuid", "seteuid %u failed", old_euid);
226c0f64 609 if (setegid(old_egid) != 0)
514a25cb
DK
610 _error->FatalE("setegid", "setegid %u failed", old_egid);
611 if (setgroups(old_gidlist_nr, old_gidlist.get()))
612 _error->FatalE("setgroups", "setgroups %u failed", 0);
7c8206bf 613}
1c5f7e5f 614pkgAcquire::RunResult pkgAcquire::Run(int PulseIntervall)
0a8a80e5 615{
95278287 616 _error->PushToStack();
7c8206bf
DK
617 CheckDropPrivsMustBeDisabled(*this);
618
8b89e57f
AL
619 Running = true;
620
0a8a80e5
AL
621 for (Queue *I = Queues; I != 0; I = I->Next)
622 I->Startup();
623
b98f2859
AL
624 if (Log != 0)
625 Log->Start();
626
024d1123
AL
627 bool WasCancelled = false;
628
0a8a80e5 629 // Run till all things have been acquired
8267fe24
AL
630 struct timeval tv;
631 tv.tv_sec = 0;
1c5f7e5f 632 tv.tv_usec = PulseIntervall;
0a8a80e5
AL
633 while (ToFetch > 0)
634 {
635 fd_set RFds;
636 fd_set WFds;
637 int Highest = 0;
638 FD_ZERO(&RFds);
639 FD_ZERO(&WFds);
640 SetFds(Highest,&RFds,&WFds);
641
b0db36b1
AL
642 int Res;
643 do
644 {
645 Res = select(Highest+1,&RFds,&WFds,0,&tv);
646 }
647 while (Res < 0 && errno == EINTR);
648
8267fe24 649 if (Res < 0)
8b89e57f 650 {
8267fe24
AL
651 _error->Errno("select","Select has failed");
652 break;
8b89e57f 653 }
95278287 654
a416a90b
MV
655 if(RunFdsSane(&RFds,&WFds) == false)
656 break;
95278287 657
8267fe24
AL
658 // Timeout, notify the log class
659 if (Res == 0 || (Log != 0 && Log->Update == true))
660 {
1c5f7e5f 661 tv.tv_usec = PulseIntervall;
8267fe24
AL
662 for (Worker *I = Workers; I != 0; I = I->NextAcquire)
663 I->Pulse();
024d1123
AL
664 if (Log != 0 && Log->Pulse(this) == false)
665 {
666 WasCancelled = true;
667 break;
668 }
8267fe24 669 }
0a8a80e5 670 }
be4401bf 671
b98f2859
AL
672 if (Log != 0)
673 Log->Stop();
674
be4401bf
AL
675 // Shut down the acquire bits
676 Running = false;
0a8a80e5 677 for (Queue *I = Queues; I != 0; I = I->Next)
8e5fc8f5 678 I->Shutdown(false);
0a8a80e5 679
ab559b35 680 // Shut down the items
f7f0d6c7 681 for (ItemIterator I = Items.begin(); I != Items.end(); ++I)
95278287
DK
682 (*I)->Finished();
683
684 bool const newError = _error->PendingError();
685 _error->MergeWithStack();
686 if (newError)
024d1123
AL
687 return Failed;
688 if (WasCancelled)
689 return Cancelled;
690 return Continue;
93bf083d
AL
691}
692 /*}}}*/
be4401bf 693// Acquire::Bump - Called when an item is dequeued /*{{{*/
93bf083d
AL
694// ---------------------------------------------------------------------
695/* This routine bumps idle queues in hopes that they will be able to fetch
696 the dequeued item */
697void pkgAcquire::Bump()
698{
be4401bf
AL
699 for (Queue *I = Queues; I != 0; I = I->Next)
700 I->Bump();
0a8a80e5
AL
701}
702 /*}}}*/
8267fe24
AL
703// Acquire::WorkerStep - Step to the next worker /*{{{*/
704// ---------------------------------------------------------------------
705/* Not inlined to advoid including acquire-worker.h */
706pkgAcquire::Worker *pkgAcquire::WorkerStep(Worker *I)
707{
708 return I->NextAcquire;
d3e8fbb3 709}
8267fe24 710 /*}}}*/
a6568219 711// Acquire::Clean - Cleans a directory /*{{{*/
7a7fa5f0
AL
712// ---------------------------------------------------------------------
713/* This is a bit simplistic, it looks at every file in the dir and sees
714 if it is part of the download set. */
715bool pkgAcquire::Clean(string Dir)
716{
95b5f6c1
DK
717 // non-existing directories are by definition clean…
718 if (DirectoryExists(Dir) == false)
719 return true;
720
10ecfe4f
MV
721 if(Dir == "/")
722 return _error->Error(_("Clean of %s is not supported"), Dir.c_str());
723
7a7fa5f0
AL
724 DIR *D = opendir(Dir.c_str());
725 if (D == 0)
b2e465d6 726 return _error->Errno("opendir",_("Unable to read %s"),Dir.c_str());
7a7fa5f0
AL
727
728 string StartDir = SafeGetCWD();
729 if (chdir(Dir.c_str()) != 0)
730 {
731 closedir(D);
b2e465d6 732 return _error->Errno("chdir",_("Unable to change to %s"),Dir.c_str());
7a7fa5f0
AL
733 }
734
735 for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D))
736 {
737 // Skip some files..
738 if (strcmp(Dir->d_name,"lock") == 0 ||
739 strcmp(Dir->d_name,"partial") == 0 ||
6aef1942 740 strcmp(Dir->d_name,"lost+found") == 0 ||
7a7fa5f0
AL
741 strcmp(Dir->d_name,".") == 0 ||
742 strcmp(Dir->d_name,"..") == 0)
743 continue;
744
745 // Look in the get list
b4fc9b6f 746 ItemCIterator I = Items.begin();
f7f0d6c7 747 for (; I != Items.end(); ++I)
7a7fa5f0
AL
748 if (flNotDir((*I)->DestFile) == Dir->d_name)
749 break;
750
751 // Nothing found, nuke it
752 if (I == Items.end())
ce1f3a2c 753 RemoveFile("Clean", Dir->d_name);
7a7fa5f0
AL
754 };
755
7a7fa5f0 756 closedir(D);
3c8cda8b
MV
757 if (chdir(StartDir.c_str()) != 0)
758 return _error->Errno("chdir",_("Unable to change to %s"),StartDir.c_str());
7a7fa5f0
AL
759 return true;
760}
761 /*}}}*/
a6568219
AL
762// Acquire::TotalNeeded - Number of bytes to fetch /*{{{*/
763// ---------------------------------------------------------------------
764/* This is the total number of bytes needed */
a02db58f 765APT_PURE unsigned long long pkgAcquire::TotalNeeded()
a6568219 766{
e551e123 767 return std::accumulate(ItemsBegin(), ItemsEnd(), 0llu,
ae732225
DK
768 [](unsigned long long const T, Item const * const I) {
769 return T + I->FileSize;
770 });
a6568219
AL
771}
772 /*}}}*/
773// Acquire::FetchNeeded - Number of bytes needed to get /*{{{*/
774// ---------------------------------------------------------------------
775/* This is the number of bytes that is not local */
a02db58f 776APT_PURE unsigned long long pkgAcquire::FetchNeeded()
a6568219 777{
5a978348 778 return std::accumulate(ItemsBegin(), ItemsEnd(), 0llu,
ae732225
DK
779 [](unsigned long long const T, Item const * const I) {
780 if (I->Local == false)
781 return T + I->FileSize;
782 else
783 return T;
784 });
a6568219
AL
785}
786 /*}}}*/
6b1ff003
AL
787// Acquire::PartialPresent - Number of partial bytes we already have /*{{{*/
788// ---------------------------------------------------------------------
789/* This is the number of bytes that is not local */
a02db58f 790APT_PURE unsigned long long pkgAcquire::PartialPresent()
6b1ff003 791{
5a978348 792 return std::accumulate(ItemsBegin(), ItemsEnd(), 0llu,
ae732225
DK
793 [](unsigned long long const T, Item const * const I) {
794 if (I->Local == false)
795 return T + I->PartialSize;
796 else
797 return T;
798 });
6b1ff003 799}
92fcbfc1 800 /*}}}*/
8e5fc8f5 801// Acquire::UriBegin - Start iterator for the uri list /*{{{*/
f7a08e33
AL
802// ---------------------------------------------------------------------
803/* */
804pkgAcquire::UriIterator pkgAcquire::UriBegin()
805{
806 return UriIterator(Queues);
807}
808 /*}}}*/
8e5fc8f5 809// Acquire::UriEnd - End iterator for the uri list /*{{{*/
f7a08e33
AL
810// ---------------------------------------------------------------------
811/* */
812pkgAcquire::UriIterator pkgAcquire::UriEnd()
813{
814 return UriIterator(0);
815}
816 /*}}}*/
e331f6ed
AL
817// Acquire::MethodConfig::MethodConfig - Constructor /*{{{*/
818// ---------------------------------------------------------------------
819/* */
25613a61
DK
820pkgAcquire::MethodConfig::MethodConfig() : d(NULL), Next(0), SingleInstance(false),
821 Pipeline(false), SendConfig(false), LocalOnly(false), NeedsCleanup(false),
822 Removable(false)
e331f6ed 823{
e331f6ed
AL
824}
825 /*}}}*/
0a8a80e5
AL
826// Queue::Queue - Constructor /*{{{*/
827// ---------------------------------------------------------------------
828/* */
e8afd168
DK
829pkgAcquire::Queue::Queue(string const &name,pkgAcquire * const owner) : d(NULL), Next(0),
830 Name(name), Items(0), Workers(0), Owner(owner), PipeDepth(0), MaxPipeDepth(1)
0a8a80e5 831{
0a8a80e5
AL
832}
833 /*}}}*/
834// Queue::~Queue - Destructor /*{{{*/
835// ---------------------------------------------------------------------
836/* */
837pkgAcquire::Queue::~Queue()
838{
8e5fc8f5 839 Shutdown(true);
0a8a80e5
AL
840
841 while (Items != 0)
842 {
843 QItem *Jnk = Items;
844 Items = Items->Next;
845 delete Jnk;
846 }
847}
848 /*}}}*/
849// Queue::Enqueue - Queue an item to the queue /*{{{*/
850// ---------------------------------------------------------------------
851/* */
c03462c6 852bool pkgAcquire::Queue::Enqueue(ItemDesc &Item)
0a8a80e5 853{
7a1b1f8b 854 QItem **I = &Items;
c03462c6
MV
855 // move to the end of the queue and check for duplicates here
856 for (; *I != 0; I = &(*I)->Next)
feb674ab 857 if (Item.URI == (*I)->URI)
c03462c6 858 {
9d2a8a73
DK
859 if (_config->FindB("Debug::pkgAcquire::Worker",false) == true)
860 std::cerr << " @ Queue: Action combined for " << Item.URI << " and " << (*I)->URI << std::endl;
08ea7806
DK
861 (*I)->Owners.push_back(Item.Owner);
862 Item.Owner->Status = (*I)->Owner->Status;
c03462c6
MV
863 return false;
864 }
865
0a8a80e5 866 // Create a new item
7a1b1f8b
AL
867 QItem *Itm = new QItem;
868 *Itm = Item;
869 Itm->Next = 0;
870 *I = Itm;
0a8a80e5 871
8267fe24 872 Item.Owner->QueueCounter++;
93bf083d
AL
873 if (Items->Next == 0)
874 Cycle();
c03462c6 875 return true;
0a8a80e5
AL
876}
877 /*}}}*/
c88edf1d 878// Queue::Dequeue - Remove an item from the queue /*{{{*/
0a8a80e5 879// ---------------------------------------------------------------------
b185acc2 880/* We return true if we hit something */
bfd22fc0 881bool pkgAcquire::Queue::Dequeue(Item *Owner)
0a8a80e5 882{
b185acc2
AL
883 if (Owner->Status == pkgAcquire::Item::StatFetching)
884 return _error->Error("Tried to dequeue a fetching object");
08ea7806 885
bfd22fc0 886 bool Res = false;
08ea7806 887
0a8a80e5
AL
888 QItem **I = &Items;
889 for (; *I != 0;)
890 {
08ea7806 891 if (Owner == (*I)->Owner)
0a8a80e5
AL
892 {
893 QItem *Jnk= *I;
894 *I = (*I)->Next;
895 Owner->QueueCounter--;
896 delete Jnk;
bfd22fc0 897 Res = true;
0a8a80e5
AL
898 }
899 else
900 I = &(*I)->Next;
901 }
08ea7806 902
bfd22fc0 903 return Res;
0a8a80e5
AL
904}
905 /*}}}*/
906// Queue::Startup - Start the worker processes /*{{{*/
907// ---------------------------------------------------------------------
8e5fc8f5
AL
908/* It is possible for this to be called with a pre-existing set of
909 workers. */
0a8a80e5
AL
910bool pkgAcquire::Queue::Startup()
911{
8e5fc8f5
AL
912 if (Workers == 0)
913 {
914 URI U(Name);
915 pkgAcquire::MethodConfig *Cnf = Owner->GetConfig(U.Access);
916 if (Cnf == 0)
917 return false;
918
919 Workers = new Worker(this,Cnf,Owner->Log);
920 Owner->Add(Workers);
921 if (Workers->Start() == false)
922 return false;
923
924 /* When pipelining we commit 10 items. This needs to change when we
925 added other source retry to have cycle maintain a pipeline depth
926 on its own. */
927 if (Cnf->Pipeline == true)
6ce72612 928 MaxPipeDepth = _config->FindI("Acquire::Max-Pipeline-Depth",10);
8e5fc8f5
AL
929 else
930 MaxPipeDepth = 1;
931 }
5cb5d8dc 932
93bf083d 933 return Cycle();
0a8a80e5
AL
934}
935 /*}}}*/
936// Queue::Shutdown - Shutdown the worker processes /*{{{*/
937// ---------------------------------------------------------------------
8e5fc8f5
AL
938/* If final is true then all workers are eliminated, otherwise only workers
939 that do not need cleanup are removed */
940bool pkgAcquire::Queue::Shutdown(bool Final)
0a8a80e5
AL
941{
942 // Delete all of the workers
8e5fc8f5
AL
943 pkgAcquire::Worker **Cur = &Workers;
944 while (*Cur != 0)
0a8a80e5 945 {
8e5fc8f5
AL
946 pkgAcquire::Worker *Jnk = *Cur;
947 if (Final == true || Jnk->GetConf()->NeedsCleanup == false)
948 {
949 *Cur = Jnk->NextQueue;
950 Owner->Remove(Jnk);
951 delete Jnk;
952 }
953 else
954 Cur = &(*Cur)->NextQueue;
0a8a80e5
AL
955 }
956
957 return true;
3b5421b4
AL
958}
959 /*}}}*/
7d8afa39 960// Queue::FindItem - Find a URI in the item list /*{{{*/
c88edf1d
AL
961// ---------------------------------------------------------------------
962/* */
963pkgAcquire::Queue::QItem *pkgAcquire::Queue::FindItem(string URI,pkgAcquire::Worker *Owner)
964{
965 for (QItem *I = Items; I != 0; I = I->Next)
966 if (I->URI == URI && I->Worker == Owner)
967 return I;
968 return 0;
969}
970 /*}}}*/
971// Queue::ItemDone - Item has been completed /*{{{*/
972// ---------------------------------------------------------------------
973/* The worker signals this which causes the item to be removed from the
93bf083d
AL
974 queue. If this is the last queue instance then it is removed from the
975 main queue too.*/
c88edf1d
AL
976bool pkgAcquire::Queue::ItemDone(QItem *Itm)
977{
b185acc2 978 PipeDepth--;
08ea7806
DK
979 for (QItem::owner_iterator O = Itm->Owners.begin(); O != Itm->Owners.end(); ++O)
980 {
981 if ((*O)->Status == pkgAcquire::Item::StatFetching)
982 (*O)->Status = pkgAcquire::Item::StatDone;
983 }
984
93bf083d
AL
985 if (Itm->Owner->QueueCounter <= 1)
986 Owner->Dequeue(Itm->Owner);
987 else
988 {
989 Dequeue(Itm->Owner);
990 Owner->Bump();
991 }
08ea7806 992
93bf083d
AL
993 return Cycle();
994}
995 /*}}}*/
996// Queue::Cycle - Queue new items into the method /*{{{*/
997// ---------------------------------------------------------------------
b185acc2
AL
998/* This locates a new idle item and sends it to the worker. If pipelining
999 is enabled then it keeps the pipe full. */
93bf083d
AL
1000bool pkgAcquire::Queue::Cycle()
1001{
1002 if (Items == 0 || Workers == 0)
c88edf1d
AL
1003 return true;
1004
e7432370
AL
1005 if (PipeDepth < 0)
1006 return _error->Error("Pipedepth failure");
08ea7806 1007
93bf083d
AL
1008 // Look for a queable item
1009 QItem *I = Items;
e7432370 1010 while (PipeDepth < (signed)MaxPipeDepth)
b185acc2
AL
1011 {
1012 for (; I != 0; I = I->Next)
1013 if (I->Owner->Status == pkgAcquire::Item::StatIdle)
1014 break;
08ea7806 1015
b185acc2
AL
1016 // Nothing to do, queue is idle.
1017 if (I == 0)
1018 return true;
08ea7806 1019
b185acc2 1020 I->Worker = Workers;
ae732225
DK
1021 for (auto const &O: I->Owners)
1022 O->Status = pkgAcquire::Item::StatFetching;
e7432370 1023 PipeDepth++;
b185acc2
AL
1024 if (Workers->QueueItem(I) == false)
1025 return false;
1026 }
08ea7806 1027
b185acc2 1028 return true;
c88edf1d
AL
1029}
1030 /*}}}*/
be4401bf
AL
1031// Queue::Bump - Fetch any pending objects if we are idle /*{{{*/
1032// ---------------------------------------------------------------------
b185acc2 1033/* This is called when an item in multiple queues is dequeued */
be4401bf
AL
1034void pkgAcquire::Queue::Bump()
1035{
b185acc2 1036 Cycle();
be4401bf
AL
1037}
1038 /*}}}*/
08ea7806
DK
1039HashStringList pkgAcquire::Queue::QItem::GetExpectedHashes() const /*{{{*/
1040{
1041 /* each Item can have multiple owners and each owner might have different
1042 hashes, even if that is unlikely in practice and if so at least some
1043 owners will later fail. There is one situation through which is not a
1044 failure and still needs this handling: Two owners who expect the same
1045 file, but one owner only knows the SHA1 while the other only knows SHA256. */
1046 HashStringList superhsl;
1047 for (pkgAcquire::Queue::QItem::owner_iterator O = Owners.begin(); O != Owners.end(); ++O)
1048 {
1049 HashStringList const hsl = (*O)->GetExpectedHashes();
1050 if (hsl.usable() == false)
1051 continue;
1052 if (superhsl.usable() == false)
1053 superhsl = hsl;
1054 else
1055 {
1056 // we merge both lists - if we find disagreement send no hashes
1057 HashStringList::const_iterator hs = hsl.begin();
1058 for (; hs != hsl.end(); ++hs)
1059 if (superhsl.push_back(*hs) == false)
1060 break;
1061 if (hs != hsl.end())
1062 {
1063 superhsl.clear();
1064 break;
1065 }
1066 }
1067 }
1068 return superhsl;
1069}
1070 /*}}}*/
1071APT_PURE unsigned long long pkgAcquire::Queue::QItem::GetMaximumSize() const /*{{{*/
1072{
1073 unsigned long long Maximum = std::numeric_limits<unsigned long long>::max();
ae732225 1074 for (auto const &O: Owners)
08ea7806 1075 {
ae732225 1076 if (O->FileSize == 0)
08ea7806 1077 continue;
ae732225 1078 Maximum = std::min(Maximum, O->FileSize);
08ea7806
DK
1079 }
1080 if (Maximum == std::numeric_limits<unsigned long long>::max())
1081 return 0;
1082 return Maximum;
1083}
1084 /*}}}*/
1085void pkgAcquire::Queue::QItem::SyncDestinationFiles() const /*{{{*/
1086{
1087 /* ensure that the first owner has the best partial file of all and
1088 the rest have (potentially dangling) symlinks to it so that
1089 everything (like progress reporting) finds it easily */
1090 std::string superfile = Owner->DestFile;
1091 off_t supersize = 0;
1092 for (pkgAcquire::Queue::QItem::owner_iterator O = Owners.begin(); O != Owners.end(); ++O)
1093 {
1094 if ((*O)->DestFile == superfile)
1095 continue;
1096 struct stat file;
1097 if (lstat((*O)->DestFile.c_str(),&file) == 0)
1098 {
1099 if ((file.st_mode & S_IFREG) == 0)
ce1f3a2c 1100 RemoveFile("SyncDestinationFiles", (*O)->DestFile);
08ea7806
DK
1101 else if (supersize < file.st_size)
1102 {
1103 supersize = file.st_size;
ce1f3a2c 1104 RemoveFile("SyncDestinationFiles", superfile);
08ea7806
DK
1105 rename((*O)->DestFile.c_str(), superfile.c_str());
1106 }
1107 else
ce1f3a2c 1108 RemoveFile("SyncDestinationFiles", (*O)->DestFile);
08ea7806
DK
1109 if (symlink(superfile.c_str(), (*O)->DestFile.c_str()) != 0)
1110 {
1111 ; // not a problem per-se and no real alternative
1112 }
1113 }
1114 }
1115}
1116 /*}}}*/
1117std::string pkgAcquire::Queue::QItem::Custom600Headers() const /*{{{*/
1118{
1119 /* The others are relatively easy to merge, but this one?
1120 Lets not merge and see how far we can run with it…
1121 Likely, nobody will ever notice as all the items will
1122 be of the same class and hence generate the same headers. */
1123 return Owner->Custom600Headers();
1124}
1125 /*}}}*/
1126
b98f2859
AL
1127// AcquireStatus::pkgAcquireStatus - Constructor /*{{{*/
1128// ---------------------------------------------------------------------
1129/* */
533fe3d1 1130pkgAcquireStatus::pkgAcquireStatus() : d(NULL), Percent(-1), Update(true), MorePulses(false)
b98f2859
AL
1131{
1132 Start();
1133}
1134 /*}}}*/
1135// AcquireStatus::Pulse - Called periodically /*{{{*/
1136// ---------------------------------------------------------------------
1137/* This computes some internal state variables for the derived classes to
1138 use. It generates the current downloaded bytes and total bytes to download
1139 as well as the current CPS estimate. */
024d1123 1140bool pkgAcquireStatus::Pulse(pkgAcquire *Owner)
b98f2859
AL
1141{
1142 TotalBytes = 0;
1143 CurrentBytes = 0;
d568ed2d
AL
1144 TotalItems = 0;
1145 CurrentItems = 0;
fb193b1c 1146
b98f2859
AL
1147 // Compute the total number of bytes to fetch
1148 unsigned int Unknown = 0;
1149 unsigned int Count = 0;
1eba782f 1150 bool ExpectAdditionalItems = false;
c6e9cc58
MV
1151 for (pkgAcquire::ItemCIterator I = Owner->ItemsBegin();
1152 I != Owner->ItemsEnd();
f7f0d6c7 1153 ++I, ++Count)
b98f2859 1154 {
d568ed2d
AL
1155 TotalItems++;
1156 if ((*I)->Status == pkgAcquire::Item::StatDone)
f7f0d6c7 1157 ++CurrentItems;
b2e465d6 1158
1eba782f
DK
1159 // do we expect to acquire more files than we know of yet?
1160 if ((*I)->ExpectedAdditionalItems > 0)
1161 ExpectAdditionalItems = true;
c6e9cc58 1162
b98f2859
AL
1163 TotalBytes += (*I)->FileSize;
1164 if ((*I)->Complete == true)
1165 CurrentBytes += (*I)->FileSize;
1166 if ((*I)->FileSize == 0 && (*I)->Complete == false)
f7f0d6c7 1167 ++Unknown;
b98f2859 1168 }
fb193b1c 1169
b98f2859 1170 // Compute the current completion
dbbc5494 1171 unsigned long long ResumeSize = 0;
b98f2859
AL
1172 for (pkgAcquire::Worker *I = Owner->WorkersBegin(); I != 0;
1173 I = Owner->WorkerStep(I))
c62f7898 1174 {
b98f2859 1175 if (I->CurrentItem != 0 && I->CurrentItem->Owner->Complete == false)
aa0e1101
AL
1176 {
1177 CurrentBytes += I->CurrentSize;
1178 ResumeSize += I->ResumePoint;
08ea7806 1179
aa0e1101 1180 // Files with unknown size always have 100% completion
08ea7806 1181 if (I->CurrentItem->Owner->FileSize == 0 &&
aa0e1101
AL
1182 I->CurrentItem->Owner->Complete == false)
1183 TotalBytes += I->CurrentSize;
1184 }
c62f7898 1185 }
aa0e1101 1186
b98f2859
AL
1187 // Normalize the figures and account for unknown size downloads
1188 if (TotalBytes <= 0)
1189 TotalBytes = 1;
1190 if (Unknown == Count)
1191 TotalBytes = Unknown;
18ef0a78
AL
1192
1193 // Wha?! Is not supposed to happen.
1194 if (CurrentBytes > TotalBytes)
1195 CurrentBytes = TotalBytes;
96c6cab1 1196
b98f2859
AL
1197 // Compute the CPS
1198 struct timeval NewTime;
1199 gettimeofday(&NewTime,0);
2ec1674d 1200 if ((NewTime.tv_sec - Time.tv_sec == 6 && NewTime.tv_usec > Time.tv_usec) ||
b98f2859
AL
1201 NewTime.tv_sec - Time.tv_sec > 6)
1202 {
f17ac097
AL
1203 double Delta = NewTime.tv_sec - Time.tv_sec +
1204 (NewTime.tv_usec - Time.tv_usec)/1000000.0;
b98f2859 1205
b98f2859 1206 // Compute the CPS value
f17ac097 1207 if (Delta < 0.01)
e331f6ed
AL
1208 CurrentCPS = 0;
1209 else
aa0e1101
AL
1210 CurrentCPS = ((CurrentBytes - ResumeSize) - LastBytes)/Delta;
1211 LastBytes = CurrentBytes - ResumeSize;
dbbc5494 1212 ElapsedTime = (unsigned long long)Delta;
b98f2859
AL
1213 Time = NewTime;
1214 }
024d1123 1215
533fe3d1 1216 double const OldPercent = Percent;
c6e9cc58 1217 // calculate the percentage, if we have too little data assume 1%
1eba782f 1218 if (ExpectAdditionalItems)
96c6cab1 1219 Percent = 0;
533fe3d1 1220 else
96c6cab1 1221 // use both files and bytes because bytes can be unreliable
533fe3d1 1222 Percent = (0.8 * (CurrentBytes/float(TotalBytes)*100.0) +
96c6cab1 1223 0.2 * (CurrentItems/float(TotalItems)*100.0));
fb193b1c
MV
1224
1225 // debug
1226 if (_config->FindB("Debug::acquire::progress", false) == true)
1227 {
1228 std::clog
1229 << "["
1230 << std::setw(5) << std::setprecision(4) << std::showpoint << Percent
1231 << "]"
1232 << " Bytes: "
1233 << SizeToStr(CurrentBytes) << " / " << SizeToStr(TotalBytes)
1234 << " # Files: "
1235 << CurrentItems << " / " << TotalItems
1236 << std::endl;
1237 }
1238
533fe3d1
DK
1239 double const DiffPercent = Percent - OldPercent;
1240 if (DiffPercent < 0.001 && _config->FindB("Acquire::Progress::Diffpercent", false) == true)
1241 return true;
96c6cab1 1242
75ef8f14
MV
1243 int fd = _config->FindI("APT::Status-Fd",-1);
1244 if(fd > 0)
1245 {
1246 ostringstream status;
1247
1248 char msg[200];
1249 long i = CurrentItems < TotalItems ? CurrentItems + 1 : CurrentItems;
c033d415
MV
1250 unsigned long long ETA = 0;
1251 if(CurrentCPS > 0)
1252 ETA = (TotalBytes - CurrentBytes) / CurrentCPS;
75ef8f14 1253
1e8b4c0f
MV
1254 // only show the ETA if it makes sense
1255 if (ETA > 0 && ETA < 172800 /* two days */ )
0c508b03 1256 snprintf(msg,sizeof(msg), _("Retrieving file %li of %li (%s remaining)"), i, TotalItems, TimeToStr(ETA).c_str());
1e8b4c0f 1257 else
0c508b03 1258 snprintf(msg,sizeof(msg), _("Retrieving file %li of %li"), i, TotalItems);
533fe3d1 1259
75ef8f14 1260 // build the status str
b58e2c7c
DK
1261 std::string dlstatus;
1262 strprintf(dlstatus, "dlstatus:%ld:%.4f:%s\n", i, Percent, msg);
1263 FileFd::Write(fd, dlstatus.data(), dlstatus.size());
75ef8f14
MV
1264 }
1265
024d1123 1266 return true;
b98f2859
AL
1267}
1268 /*}}}*/
1269// AcquireStatus::Start - Called when the download is started /*{{{*/
1270// ---------------------------------------------------------------------
1271/* We just reset the counters */
1272void pkgAcquireStatus::Start()
1273{
1274 gettimeofday(&Time,0);
1275 gettimeofday(&StartTime,0);
1276 LastBytes = 0;
1277 CurrentCPS = 0;
1278 CurrentBytes = 0;
1279 TotalBytes = 0;
1280 FetchedBytes = 0;
1281 ElapsedTime = 0;
d568ed2d
AL
1282 TotalItems = 0;
1283 CurrentItems = 0;
b98f2859
AL
1284}
1285 /*}}}*/
a6568219 1286// AcquireStatus::Stop - Finished downloading /*{{{*/
b98f2859
AL
1287// ---------------------------------------------------------------------
1288/* This accurately computes the elapsed time and the total overall CPS. */
1289void pkgAcquireStatus::Stop()
1290{
1291 // Compute the CPS and elapsed time
1292 struct timeval NewTime;
1293 gettimeofday(&NewTime,0);
1294
31a0531d
AL
1295 double Delta = NewTime.tv_sec - StartTime.tv_sec +
1296 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
b98f2859 1297
b98f2859 1298 // Compute the CPS value
31a0531d 1299 if (Delta < 0.01)
e331f6ed
AL
1300 CurrentCPS = 0;
1301 else
31a0531d 1302 CurrentCPS = FetchedBytes/Delta;
b98f2859 1303 LastBytes = CurrentBytes;
dbbc5494 1304 ElapsedTime = (unsigned long long)Delta;
b98f2859
AL
1305}
1306 /*}}}*/
1307// AcquireStatus::Fetched - Called when a byte set has been fetched /*{{{*/
1308// ---------------------------------------------------------------------
1309/* This is used to get accurate final transfer rate reporting. */
73da43e9 1310void pkgAcquireStatus::Fetched(unsigned long long Size,unsigned long long Resume)
93274b8d 1311{
b98f2859
AL
1312 FetchedBytes += Size - Resume;
1313}
1314 /*}}}*/
862bafea 1315
c8a4ce6c
DK
1316pkgAcquire::UriIterator::UriIterator(pkgAcquire::Queue *Q) : d(NULL), CurQ(Q), CurItem(0)
1317{
1318 while (CurItem == 0 && CurQ != 0)
1319 {
1320 CurItem = CurQ->Items;
1321 CurQ = CurQ->Next;
1322 }
1323}
1324
9d653a6d
DK
1325APT_CONST pkgAcquire::UriIterator::~UriIterator() {}
1326APT_CONST pkgAcquire::MethodConfig::~MethodConfig() {}
1327APT_CONST pkgAcquireStatus::~pkgAcquireStatus() {}