]> git.saurik.com Git - apt.git/blob - apt-pkg/acquire-item.cc
47cd9eab25a18a4a4e2f53077f5b49063fe8f04d
[apt.git] / apt-pkg / acquire-item.cc
1 // -*- mode: cpp; mode: fold -*-
2 // Description /*{{{*/
3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
4 /* ######################################################################
5
6 Acquire Item - Item to acquire
7
8 Each item can download to exactly one file at a time. This means you
9 cannot create an item that fetches two uri's to two files at the same
10 time. The pkgAcqIndex class creates a second class upon instantiation
11 to fetch the other index files because of this.
12
13 ##################################################################### */
14 /*}}}*/
15 // Include Files /*{{{*/
16 #include <config.h>
17
18 #include <apt-pkg/acquire-item.h>
19 #include <apt-pkg/configuration.h>
20 #include <apt-pkg/aptconfiguration.h>
21 #include <apt-pkg/sourcelist.h>
22 #include <apt-pkg/error.h>
23 #include <apt-pkg/strutl.h>
24 #include <apt-pkg/fileutl.h>
25 #include <apt-pkg/sha1.h>
26 #include <apt-pkg/tagfile.h>
27 #include <apt-pkg/indexrecords.h>
28 #include <apt-pkg/acquire.h>
29 #include <apt-pkg/hashes.h>
30 #include <apt-pkg/indexfile.h>
31 #include <apt-pkg/pkgcache.h>
32 #include <apt-pkg/cacheiterators.h>
33 #include <apt-pkg/pkgrecords.h>
34
35 #include <stddef.h>
36 #include <stdlib.h>
37 #include <string.h>
38 #include <iostream>
39 #include <vector>
40 #include <sys/stat.h>
41 #include <unistd.h>
42 #include <errno.h>
43 #include <string>
44 #include <sstream>
45 #include <stdio.h>
46 #include <ctime>
47
48 #include <apti18n.h>
49 /*}}}*/
50
51 using namespace std;
52
53 static void printHashSumComparision(std::string const &URI, HashStringList const &Expected, HashStringList const &Actual) /*{{{*/
54 {
55 if (_config->FindB("Debug::Acquire::HashSumMismatch", false) == false)
56 return;
57 std::cerr << std::endl << URI << ":" << std::endl << " Expected Hash: " << std::endl;
58 for (HashStringList::const_iterator hs = Expected.begin(); hs != Expected.end(); ++hs)
59 std::cerr << "\t- " << hs->toStr() << std::endl;
60 std::cerr << " Actual Hash: " << std::endl;
61 for (HashStringList::const_iterator hs = Actual.begin(); hs != Actual.end(); ++hs)
62 std::cerr << "\t- " << hs->toStr() << std::endl;
63 }
64 /*}}}*/
65
66 // Acquire::Item::Item - Constructor /*{{{*/
67 pkgAcquire::Item::Item(pkgAcquire *Owner,
68 HashStringList const &ExpectedHashes,
69 unsigned long TransactionID)
70 : Owner(Owner), FileSize(0), PartialSize(0), Mode(0), ID(0), Complete(false),
71 Local(false), QueueCounter(0), TransactionID(TransactionID),
72 ExpectedAdditionalItems(0), ExpectedHashes(ExpectedHashes)
73 {
74 Owner->Add(this);
75 Status = StatIdle;
76 }
77 /*}}}*/
78 // Acquire::Item::~Item - Destructor /*{{{*/
79 // ---------------------------------------------------------------------
80 /* */
81 pkgAcquire::Item::~Item()
82 {
83 Owner->Remove(this);
84 }
85 /*}}}*/
86 // Acquire::Item::Failed - Item failed to download /*{{{*/
87 // ---------------------------------------------------------------------
88 /* We return to an idle state if there are still other queues that could
89 fetch this object */
90 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
91 {
92 Status = StatIdle;
93 if(ErrorText == "")
94 ErrorText = LookupTag(Message,"Message");
95 UsedMirror = LookupTag(Message,"UsedMirror");
96 if (QueueCounter <= 1)
97 {
98 /* This indicates that the file is not available right now but might
99 be sometime later. If we do a retry cycle then this should be
100 retried [CDROMs] */
101 if (Cnf->LocalOnly == true &&
102 StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
103 {
104 Status = StatIdle;
105 Dequeue();
106 return;
107 }
108
109 Status = StatError;
110 Dequeue();
111 }
112
113 // report mirror failure back to LP if we actually use a mirror
114 string FailReason = LookupTag(Message, "FailReason");
115 if(FailReason.size() != 0)
116 ReportMirrorFailure(FailReason);
117 else
118 ReportMirrorFailure(ErrorText);
119 }
120 /*}}}*/
121 // Acquire::Item::Start - Item has begun to download /*{{{*/
122 // ---------------------------------------------------------------------
123 /* Stash status and the file size. Note that setting Complete means
124 sub-phases of the acquire process such as decompresion are operating */
125 void pkgAcquire::Item::Start(string /*Message*/,unsigned long long Size)
126 {
127 Status = StatFetching;
128 if (FileSize == 0 && Complete == false)
129 FileSize = Size;
130 }
131 /*}}}*/
132 // Acquire::Item::Done - Item downloaded OK /*{{{*/
133 // ---------------------------------------------------------------------
134 /* */
135 void pkgAcquire::Item::Done(string Message,unsigned long long Size,HashStringList const &/*Hash*/,
136 pkgAcquire::MethodConfig * /*Cnf*/)
137 {
138 // We just downloaded something..
139 string FileName = LookupTag(Message,"Filename");
140 UsedMirror = LookupTag(Message,"UsedMirror");
141 if (Complete == false && !Local && FileName == DestFile)
142 {
143 if (Owner->Log != 0)
144 Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
145 }
146
147 if (FileSize == 0)
148 FileSize= Size;
149 Status = StatDone;
150 ErrorText = string();
151 Owner->Dequeue(this);
152 }
153 /*}}}*/
154 // Acquire::Item::Rename - Rename a file /*{{{*/
155 // ---------------------------------------------------------------------
156 /* This helper function is used by a lot of item methods as their final
157 step */
158 void pkgAcquire::Item::Rename(string From,string To)
159 {
160 if (rename(From.c_str(),To.c_str()) != 0)
161 {
162 char S[300];
163 snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
164 From.c_str(),To.c_str());
165 Status = StatError;
166 ErrorText = S;
167 }
168 }
169 /*}}}*/
170 bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const error)/*{{{*/
171 {
172 if(FileExists(DestFile))
173 Rename(DestFile, DestFile + ".FAILED");
174
175 switch (error)
176 {
177 case HashSumMismatch:
178 ErrorText = _("Hash Sum mismatch");
179 Status = StatAuthError;
180 ReportMirrorFailure("HashChecksumFailure");
181 break;
182 case SizeMismatch:
183 ErrorText = _("Size mismatch");
184 Status = StatAuthError;
185 ReportMirrorFailure("SizeFailure");
186 break;
187 case InvalidFormat:
188 ErrorText = _("Invalid file format");
189 Status = StatError;
190 // do not report as usually its not the mirrors fault, but Portal/Proxy
191 break;
192 }
193 return false;
194 }
195 /*}}}*/
196 // Acquire::Item::ReportMirrorFailure /*{{{*/
197 // ---------------------------------------------------------------------
198 void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
199 {
200 // we only act if a mirror was used at all
201 if(UsedMirror.empty())
202 return;
203 #if 0
204 std::cerr << "\nReportMirrorFailure: "
205 << UsedMirror
206 << " Uri: " << DescURI()
207 << " FailCode: "
208 << FailCode << std::endl;
209 #endif
210 const char *Args[40];
211 unsigned int i = 0;
212 string report = _config->Find("Methods::Mirror::ProblemReporting",
213 "/usr/lib/apt/apt-report-mirror-failure");
214 if(!FileExists(report))
215 return;
216 Args[i++] = report.c_str();
217 Args[i++] = UsedMirror.c_str();
218 Args[i++] = DescURI().c_str();
219 Args[i++] = FailCode.c_str();
220 Args[i++] = NULL;
221 pid_t pid = ExecFork();
222 if(pid < 0)
223 {
224 _error->Error("ReportMirrorFailure Fork failed");
225 return;
226 }
227 else if(pid == 0)
228 {
229 execvp(Args[0], (char**)Args);
230 std::cerr << "Could not exec " << Args[0] << std::endl;
231 _exit(100);
232 }
233 if(!ExecWait(pid, "report-mirror-failure"))
234 {
235 _error->Warning("Couldn't report problem to '%s'",
236 _config->Find("Methods::Mirror::ProblemReporting").c_str());
237 }
238 }
239 /*}}}*/
240 // AcqSubIndex::AcqSubIndex - Constructor /*{{{*/
241 // ---------------------------------------------------------------------
242 /* Get a sub-index file based on checksums from a 'master' file and
243 possibly query additional files */
244 pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner,
245 unsigned long TransactionID,
246 string const &URI,
247 string const &URIDesc, string const &ShortDesc,
248 HashStringList const &ExpectedHashes)
249 : Item(Owner, ExpectedHashes, TransactionID)
250 {
251 /* XXX: Beware: Currently this class does nothing (of value) anymore ! */
252 Debug = _config->FindB("Debug::pkgAcquire::SubIndex",false);
253
254 DestFile = _config->FindDir("Dir::State::lists") + "partial/";
255 DestFile += URItoFileName(URI);
256
257 Desc.URI = URI;
258 Desc.Description = URIDesc;
259 Desc.Owner = this;
260 Desc.ShortDesc = ShortDesc;
261
262 QueueURI(Desc);
263
264 if(Debug)
265 std::clog << "pkgAcqSubIndex: " << Desc.URI << std::endl;
266 }
267 /*}}}*/
268 // AcqSubIndex::Custom600Headers - Insert custom request headers /*{{{*/
269 // ---------------------------------------------------------------------
270 /* The only header we use is the last-modified header. */
271 string pkgAcqSubIndex::Custom600Headers() const
272 {
273 string Final = _config->FindDir("Dir::State::lists");
274 Final += URItoFileName(Desc.URI);
275
276 struct stat Buf;
277 if (stat(Final.c_str(),&Buf) != 0)
278 return "\nIndex-File: true\nFail-Ignore: true\n";
279 return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
280 }
281 /*}}}*/
282 void pkgAcqSubIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
283 {
284 if(Debug)
285 std::clog << "pkgAcqSubIndex failed: " << Desc.URI << " with " << Message << std::endl;
286
287 Complete = false;
288 Status = StatDone;
289 Dequeue();
290
291 // No good Index is provided
292 }
293 /*}}}*/
294 void pkgAcqSubIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/
295 pkgAcquire::MethodConfig *Cnf)
296 {
297 if(Debug)
298 std::clog << "pkgAcqSubIndex::Done(): " << Desc.URI << std::endl;
299
300 string FileName = LookupTag(Message,"Filename");
301 if (FileName.empty() == true)
302 {
303 Status = StatError;
304 ErrorText = "Method gave a blank filename";
305 return;
306 }
307
308 if (FileName != DestFile)
309 {
310 Local = true;
311 Desc.URI = "copy:" + FileName;
312 QueueURI(Desc);
313 return;
314 }
315
316 Item::Done(Message, Size, Hashes, Cnf);
317
318 string FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(Desc.URI);
319
320 /* Downloaded invalid transindex => Error (LP: #346386) (Closes: #627642) */
321 indexRecords SubIndexParser;
322 if (FileExists(DestFile) == true && !SubIndexParser.Load(DestFile)) {
323 Status = StatError;
324 ErrorText = SubIndexParser.ErrorText;
325 return;
326 }
327
328 // success in downloading the index
329 // rename the index
330 if(Debug)
331 std::clog << "Renaming: " << DestFile << " -> " << FinalFile << std::endl;
332 Rename(DestFile,FinalFile);
333 chmod(FinalFile.c_str(),0644);
334 DestFile = FinalFile;
335
336 if(ParseIndex(DestFile) == false)
337 return Failed("", NULL);
338
339 Complete = true;
340 Status = StatDone;
341 Dequeue();
342 return;
343 }
344 /*}}}*/
345 bool pkgAcqSubIndex::ParseIndex(string const &IndexFile) /*{{{*/
346 {
347 indexRecords SubIndexParser;
348 if (FileExists(IndexFile) == false || SubIndexParser.Load(IndexFile) == false)
349 return false;
350 // so something with the downloaded index
351 return true;
352 }
353 /*}}}*/
354 // AcqDiffIndex::AcqDiffIndex - Constructor /*{{{*/
355 // ---------------------------------------------------------------------
356 /* Get the DiffIndex file first and see if there are patches available
357 * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
358 * patches. If anything goes wrong in that process, it will fall back to
359 * the original packages file
360 */
361 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
362 unsigned long TransactionID,
363 IndexTarget const * const Target,
364 HashStringList const &ExpectedHashes,
365 indexRecords *MetaIndexParser)
366 : pkgAcqBaseIndex(Owner, TransactionID, Target, ExpectedHashes,
367 MetaIndexParser)
368 {
369
370 Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
371
372 RealURI = Target->URI;
373 Desc.Owner = this;
374 Desc.Description = Target->Description + "/DiffIndex";
375 Desc.ShortDesc = Target->ShortDesc;
376 Desc.URI = Target->URI + ".diff/Index";
377
378 DestFile = _config->FindDir("Dir::State::lists") + "partial/";
379 DestFile += URItoFileName(Target->URI) + string(".DiffIndex");
380
381 if(Debug)
382 std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
383
384 // look for the current package file
385 CurrentPackagesFile = _config->FindDir("Dir::State::lists");
386 CurrentPackagesFile += URItoFileName(RealURI);
387
388 // FIXME: this file:/ check is a hack to prevent fetching
389 // from local sources. this is really silly, and
390 // should be fixed cleanly as soon as possible
391 if(!FileExists(CurrentPackagesFile) ||
392 Desc.URI.substr(0,strlen("file:/")) == "file:/")
393 {
394 // we don't have a pkg file or we don't want to queue
395 if(Debug)
396 std::clog << "No index file, local or canceld by user" << std::endl;
397 Failed("", NULL);
398 return;
399 }
400
401 if(Debug)
402 std::clog << "pkgAcqDiffIndex::pkgAcqDiffIndex(): "
403 << CurrentPackagesFile << std::endl;
404
405 QueueURI(Desc);
406
407 }
408 /*}}}*/
409 // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/
410 // ---------------------------------------------------------------------
411 /* The only header we use is the last-modified header. */
412 string pkgAcqDiffIndex::Custom600Headers() const
413 {
414 string Final = _config->FindDir("Dir::State::lists");
415 Final += URItoFileName(RealURI) + string(".IndexDiff");
416
417 if(Debug)
418 std::clog << "Custom600Header-IMS: " << Final << std::endl;
419
420 struct stat Buf;
421 if (stat(Final.c_str(),&Buf) != 0)
422 return "\nIndex-File: true";
423
424 return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
425 }
426 /*}}}*/
427 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/
428 {
429 if(Debug)
430 std::clog << "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile
431 << std::endl;
432
433 pkgTagSection Tags;
434 string ServerSha1;
435 vector<DiffInfo> available_patches;
436
437 FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
438 pkgTagFile TF(&Fd);
439 if (_error->PendingError() == true)
440 return false;
441
442 if(TF.Step(Tags) == true)
443 {
444 bool found = false;
445 DiffInfo d;
446 string size;
447
448 string const tmp = Tags.FindS("SHA1-Current");
449 std::stringstream ss(tmp);
450 ss >> ServerSha1 >> size;
451 unsigned long const ServerSize = atol(size.c_str());
452
453 FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
454 SHA1Summation SHA1;
455 SHA1.AddFD(fd);
456 string const local_sha1 = SHA1.Result();
457
458 if(local_sha1 == ServerSha1)
459 {
460 // we have the same sha1 as the server so we are done here
461 if(Debug)
462 std::clog << "Package file is up-to-date" << std::endl;
463 // list cleanup needs to know that this file as well as the already
464 // present index is ours, so we create an empty diff to save it for us
465 new pkgAcqIndexDiffs(Owner, TransactionID, Target,
466 ExpectedHashes, MetaIndexParser,
467 ServerSha1, available_patches);
468 return true;
469 }
470 else
471 {
472 if(Debug)
473 std::clog << "SHA1-Current: " << ServerSha1 << " and we start at "<< fd.Name() << " " << fd.Size() << " " << local_sha1 << std::endl;
474
475 // check the historie and see what patches we need
476 string const history = Tags.FindS("SHA1-History");
477 std::stringstream hist(history);
478 while(hist >> d.sha1 >> size >> d.file)
479 {
480 // read until the first match is found
481 // from that point on, we probably need all diffs
482 if(d.sha1 == local_sha1)
483 found=true;
484 else if (found == false)
485 continue;
486
487 if(Debug)
488 std::clog << "Need to get diff: " << d.file << std::endl;
489 available_patches.push_back(d);
490 }
491
492 if (available_patches.empty() == false)
493 {
494 // patching with too many files is rather slow compared to a fast download
495 unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
496 if (fileLimit != 0 && fileLimit < available_patches.size())
497 {
498 if (Debug)
499 std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
500 << ") so fallback to complete download" << std::endl;
501 return false;
502 }
503
504 // see if the patches are too big
505 found = false; // it was true and it will be true again at the end
506 d = *available_patches.begin();
507 string const firstPatch = d.file;
508 unsigned long patchesSize = 0;
509 std::stringstream patches(Tags.FindS("SHA1-Patches"));
510 while(patches >> d.sha1 >> size >> d.file)
511 {
512 if (firstPatch == d.file)
513 found = true;
514 else if (found == false)
515 continue;
516
517 patchesSize += atol(size.c_str());
518 }
519 unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
520 if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
521 {
522 if (Debug)
523 std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
524 << ") so fallback to complete download" << std::endl;
525 return false;
526 }
527 }
528 }
529
530 // we have something, queue the next diff
531 if(found)
532 {
533 // queue the diffs
534 string::size_type const last_space = Description.rfind(" ");
535 if(last_space != string::npos)
536 Description.erase(last_space, Description.size()-last_space);
537
538 /* decide if we should download patches one by one or in one go:
539 The first is good if the server merges patches, but many don't so client
540 based merging can be attempt in which case the second is better.
541 "bad things" will happen if patches are merged on the server,
542 but client side merging is attempt as well */
543 bool pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true);
544 if (pdiff_merge == true)
545 {
546 // reprepro adds this flag if it has merged patches on the server
547 std::string const precedence = Tags.FindS("X-Patch-Precedence");
548 pdiff_merge = (precedence != "merged");
549 }
550
551 if (pdiff_merge == false)
552 {
553 new pkgAcqIndexDiffs(Owner, TransactionID, Target, ExpectedHashes,
554 MetaIndexParser,
555 ServerSha1, available_patches);
556 }
557 else
558 {
559 std::vector<pkgAcqIndexMergeDiffs*> *diffs = new std::vector<pkgAcqIndexMergeDiffs*>(available_patches.size());
560 for(size_t i = 0; i < available_patches.size(); ++i)
561 (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner,
562 TransactionID,
563 Target,
564 ExpectedHashes,
565 MetaIndexParser,
566 available_patches[i],
567 diffs);
568 }
569
570 Complete = false;
571 Status = StatDone;
572 Dequeue();
573 return true;
574 }
575 }
576
577 // Nothing found, report and return false
578 // Failing here is ok, if we return false later, the full
579 // IndexFile is queued
580 if(Debug)
581 std::clog << "Can't find a patch in the index file" << std::endl;
582 return false;
583 }
584 /*}}}*/
585 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
586 {
587 if(Debug)
588 std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl
589 << "Falling back to normal index file acquire" << std::endl;
590
591 new pkgAcqIndex(Owner, TransactionID, Target, ExpectedHashes, MetaIndexParser);
592
593 Complete = false;
594 Status = StatDone;
595 Dequeue();
596 }
597 /*}}}*/
598 void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/
599 pkgAcquire::MethodConfig *Cnf)
600 {
601 if(Debug)
602 std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
603
604 Item::Done(Message, Size, Hashes, Cnf);
605
606 string FinalFile;
607 FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
608
609 // success in downloading the index
610 // rename the index
611 FinalFile += string(".IndexDiff");
612 if(Debug)
613 std::clog << "Renaming: " << DestFile << " -> " << FinalFile
614 << std::endl;
615 Rename(DestFile,FinalFile);
616 chmod(FinalFile.c_str(),0644);
617 DestFile = FinalFile;
618
619 if(!ParseDiffIndex(DestFile))
620 return Failed("", NULL);
621
622 Complete = true;
623 Status = StatDone;
624 Dequeue();
625 return;
626 }
627 /*}}}*/
628 // AcqIndexDiffs::AcqIndexDiffs - Constructor /*{{{*/
629 // ---------------------------------------------------------------------
630 /* The package diff is added to the queue. one object is constructed
631 * for each diff and the index
632 */
633 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
634 unsigned long TransactionID,
635 struct IndexTarget const * const Target,
636 HashStringList const &ExpectedHashes,
637 indexRecords *MetaIndexParser,
638 string ServerSha1,
639 vector<DiffInfo> diffs)
640 : pkgAcqBaseIndex(Owner, TransactionID, Target, ExpectedHashes, MetaIndexParser),
641 available_patches(diffs), ServerSha1(ServerSha1)
642 {
643
644 DestFile = _config->FindDir("Dir::State::lists") + "partial/";
645 DestFile += URItoFileName(Target->URI);
646
647 Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
648
649 RealURI = Target->URI;
650 Desc.Owner = this;
651 Description = Target->Description;
652 Desc.ShortDesc = Target->ShortDesc;
653
654 if(available_patches.empty() == true)
655 {
656 // we are done (yeah!)
657 Finish(true);
658 }
659 else
660 {
661 // get the next diff
662 State = StateFetchDiff;
663 QueueNextDiff();
664 }
665 }
666 /*}}}*/
667 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
668 {
669 if(Debug)
670 std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl
671 << "Falling back to normal index file acquire" << std::endl;
672 new pkgAcqIndex(Owner, TransactionID, Target, ExpectedHashes, MetaIndexParser);
673 Finish();
674 }
675 /*}}}*/
676 // Finish - helper that cleans the item out of the fetcher queue /*{{{*/
677 void pkgAcqIndexDiffs::Finish(bool allDone)
678 {
679 // we restore the original name, this is required, otherwise
680 // the file will be cleaned
681 if(allDone)
682 {
683 DestFile = _config->FindDir("Dir::State::lists");
684 DestFile += URItoFileName(RealURI);
685
686 if(HashSums().usable() && !HashSums().VerifyFile(DestFile))
687 {
688 RenameOnError(HashSumMismatch);
689 Dequeue();
690 return;
691 }
692
693 // this is for the "real" finish
694 Complete = true;
695 Status = StatDone;
696 Dequeue();
697 if(Debug)
698 std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
699 return;
700 }
701
702 if(Debug)
703 std::clog << "Finishing: " << Desc.URI << std::endl;
704 Complete = false;
705 Status = StatDone;
706 Dequeue();
707 return;
708 }
709 /*}}}*/
710 bool pkgAcqIndexDiffs::QueueNextDiff() /*{{{*/
711 {
712
713 // calc sha1 of the just patched file
714 string FinalFile = _config->FindDir("Dir::State::lists");
715 FinalFile += URItoFileName(RealURI);
716
717 FileFd fd(FinalFile, FileFd::ReadOnly);
718 SHA1Summation SHA1;
719 SHA1.AddFD(fd);
720 string local_sha1 = string(SHA1.Result());
721 if(Debug)
722 std::clog << "QueueNextDiff: "
723 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
724
725 // final file reached before all patches are applied
726 if(local_sha1 == ServerSha1)
727 {
728 Finish(true);
729 return true;
730 }
731
732 // remove all patches until the next matching patch is found
733 // this requires the Index file to be ordered
734 for(vector<DiffInfo>::iterator I=available_patches.begin();
735 available_patches.empty() == false &&
736 I != available_patches.end() &&
737 I->sha1 != local_sha1;
738 ++I)
739 {
740 available_patches.erase(I);
741 }
742
743 // error checking and falling back if no patch was found
744 if(available_patches.empty() == true)
745 {
746 Failed("", NULL);
747 return false;
748 }
749
750 // queue the right diff
751 Desc.URI = RealURI + ".diff/" + available_patches[0].file + ".gz";
752 Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
753 DestFile = _config->FindDir("Dir::State::lists") + "partial/";
754 DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
755
756 if(Debug)
757 std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
758
759 QueueURI(Desc);
760
761 return true;
762 }
763 /*}}}*/
764 void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size, HashStringList const &Hashes, /*{{{*/
765 pkgAcquire::MethodConfig *Cnf)
766 {
767 if(Debug)
768 std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
769
770 Item::Done(Message, Size, Hashes, Cnf);
771
772 string FinalFile;
773 FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
774
775 // success in downloading a diff, enter ApplyDiff state
776 if(State == StateFetchDiff)
777 {
778
779 // rred excepts the patch as $FinalFile.ed
780 Rename(DestFile,FinalFile+".ed");
781
782 if(Debug)
783 std::clog << "Sending to rred method: " << FinalFile << std::endl;
784
785 State = StateApplyDiff;
786 Local = true;
787 Desc.URI = "rred:" + FinalFile;
788 QueueURI(Desc);
789 Mode = "rred";
790 return;
791 }
792
793
794 // success in download/apply a diff, queue next (if needed)
795 if(State == StateApplyDiff)
796 {
797 // remove the just applied patch
798 available_patches.erase(available_patches.begin());
799 unlink((FinalFile + ".ed").c_str());
800
801 // move into place
802 if(Debug)
803 {
804 std::clog << "Moving patched file in place: " << std::endl
805 << DestFile << " -> " << FinalFile << std::endl;
806 }
807 Rename(DestFile,FinalFile);
808 chmod(FinalFile.c_str(),0644);
809
810 // see if there is more to download
811 if(available_patches.empty() == false) {
812 new pkgAcqIndexDiffs(Owner, TransactionID, Target,
813 ExpectedHashes, MetaIndexParser,
814 ServerSha1, available_patches);
815 return Finish();
816 } else
817 return Finish(true);
818 }
819 }
820 /*}}}*/
821 // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor /*{{{*/
822 pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner,
823 unsigned long TransactionID,
824 struct IndexTarget const * const Target,
825 HashStringList const &ExpectedHashes,
826 indexRecords *MetaIndexParser,
827 DiffInfo const &patch,
828 std::vector<pkgAcqIndexMergeDiffs*> const * const allPatches)
829 : pkgAcqBaseIndex(Owner, TransactionID, Target, ExpectedHashes, MetaIndexParser),
830 patch(patch), allPatches(allPatches), State(StateFetchDiff)
831 {
832
833 DestFile = _config->FindDir("Dir::State::lists") + "partial/";
834 DestFile += URItoFileName(Target->URI);
835
836 Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
837
838 RealURI = Target->URI;
839 Desc.Owner = this;
840 Description = Target->Description;
841 Desc.ShortDesc = Target->ShortDesc;
842
843 Desc.URI = RealURI + ".diff/" + patch.file + ".gz";
844 Desc.Description = Description + " " + patch.file + string(".pdiff");
845 DestFile = _config->FindDir("Dir::State::lists") + "partial/";
846 DestFile += URItoFileName(RealURI + ".diff/" + patch.file);
847
848 if(Debug)
849 std::clog << "pkgAcqIndexMergeDiffs: " << Desc.URI << std::endl;
850
851 QueueURI(Desc);
852 }
853 /*}}}*/
854 void pkgAcqIndexMergeDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
855 {
856 if(Debug)
857 std::clog << "pkgAcqIndexMergeDiffs failed: " << Desc.URI << " with " << Message << std::endl;
858 Complete = false;
859 Status = StatDone;
860 Dequeue();
861
862 // check if we are the first to fail, otherwise we are done here
863 State = StateDoneDiff;
864 for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
865 I != allPatches->end(); ++I)
866 if ((*I)->State == StateErrorDiff)
867 return;
868
869 // first failure means we should fallback
870 State = StateErrorDiff;
871 std::clog << "Falling back to normal index file acquire" << std::endl;
872 new pkgAcqIndex(Owner, TransactionID, Target, ExpectedHashes, MetaIndexParser);
873 }
874 /*}}}*/
875 void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/
876 pkgAcquire::MethodConfig *Cnf)
877 {
878 if(Debug)
879 std::clog << "pkgAcqIndexMergeDiffs::Done(): " << Desc.URI << std::endl;
880
881 Item::Done(Message,Size,Hashes,Cnf);
882
883 string const FinalFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
884
885 if (State == StateFetchDiff)
886 {
887 // rred expects the patch as $FinalFile.ed.$patchname.gz
888 Rename(DestFile, FinalFile + ".ed." + patch.file + ".gz");
889
890 // check if this is the last completed diff
891 State = StateDoneDiff;
892 for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
893 I != allPatches->end(); ++I)
894 if ((*I)->State != StateDoneDiff)
895 {
896 if(Debug)
897 std::clog << "Not the last done diff in the batch: " << Desc.URI << std::endl;
898 return;
899 }
900
901 // this is the last completed diff, so we are ready to apply now
902 State = StateApplyDiff;
903
904 if(Debug)
905 std::clog << "Sending to rred method: " << FinalFile << std::endl;
906
907 Local = true;
908 Desc.URI = "rred:" + FinalFile;
909 QueueURI(Desc);
910 Mode = "rred";
911 return;
912 }
913 // success in download/apply all diffs, clean up
914 else if (State == StateApplyDiff)
915 {
916 // see if we really got the expected file
917 if(ExpectedHashes.usable() && !ExpectedHashes.VerifyFile(DestFile))
918 {
919 RenameOnError(HashSumMismatch);
920 return;
921 }
922
923 // move the result into place
924 if(Debug)
925 std::clog << "Moving patched file in place: " << std::endl
926 << DestFile << " -> " << FinalFile << std::endl;
927 Rename(DestFile, FinalFile);
928 chmod(FinalFile.c_str(), 0644);
929
930 // otherwise lists cleanup will eat the file
931 DestFile = FinalFile;
932
933 // ensure the ed's are gone regardless of list-cleanup
934 for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
935 I != allPatches->end(); ++I)
936 {
937 std::string patch = FinalFile + ".ed." + (*I)->patch.file + ".gz";
938 unlink(patch.c_str());
939 }
940
941 // all set and done
942 Complete = true;
943 if(Debug)
944 std::clog << "allDone: " << DestFile << "\n" << std::endl;
945 }
946 }
947 /*}}}*/
948 // AcqIndex::AcqIndex - Constructor /*{{{*/
949 // ---------------------------------------------------------------------
950 /* The package file is added to the queue and a second class is
951 instantiated to fetch the revision file */
952 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
953 string URI,string URIDesc,string ShortDesc,
954 HashStringList const &ExpectedHash, string comprExt)
955 : pkgAcqBaseIndex(Owner, 0, NULL, ExpectedHash, NULL), RealURI(URI)
956 {
957 if(comprExt.empty() == true)
958 {
959 // autoselect the compression method
960 std::vector<std::string> types = APT::Configuration::getCompressionTypes();
961 for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
962 comprExt.append(*t).append(" ");
963 if (comprExt.empty() == false)
964 comprExt.erase(comprExt.end()-1);
965 }
966 CompressionExtension = comprExt;
967
968 Init(URI, URIDesc, ShortDesc);
969 }
970 #if 0
971 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target,
972 HashStringList const &ExpectedHash,
973 indexRecords *MetaIndexParser)
974 : pkgAcqBaseIndex(Owner, Target, ExpectedHash, MetaIndexParser),
975 RealURI(Target->URI)
976 {
977 // autoselect the compression method
978 AutoSelectCompression();
979 Init(Target->URI, Target->Description, Target->ShortDesc);
980 }
981 #endif
982 /*}}}*/
983 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
984 unsigned long TransactionID,
985 IndexTarget const *Target,
986 HashStringList const &ExpectedHash,
987 indexRecords *MetaIndexParser)
988 : pkgAcqBaseIndex(Owner, TransactionID, Target, ExpectedHash,
989 MetaIndexParser), RealURI(Target->URI)
990 {
991 // autoselect the compression method
992 AutoSelectCompression();
993 Init(Target->URI, Target->Description, Target->ShortDesc);
994
995 if(_config->FindB("Debug::Acquire::Transaction", false) == true)
996 std::clog << "New pkgIndex with TransactionID "
997 << TransactionID << std::endl;
998 }
999 /*}}}*/
1000 void pkgAcqIndex::AutoSelectCompression()
1001 {
1002 std::vector<std::string> types = APT::Configuration::getCompressionTypes();
1003 CompressionExtension = "";
1004 if (ExpectedHashes.usable())
1005 {
1006 for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
1007 if (*t == "uncompressed" || MetaIndexParser->Exists(string(Target->MetaKey).append(".").append(*t)) == true)
1008 CompressionExtension.append(*t).append(" ");
1009 }
1010 else
1011 {
1012 for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
1013 CompressionExtension.append(*t).append(" ");
1014 }
1015 if (CompressionExtension.empty() == false)
1016 CompressionExtension.erase(CompressionExtension.end()-1);
1017 }
1018 // AcqIndex::Init - defered Constructor /*{{{*/
1019 void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &ShortDesc) {
1020 Decompression = false;
1021 Erase = false;
1022
1023 DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1024 DestFile += URItoFileName(URI);
1025
1026 std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
1027 std::string MetaKey;
1028 if (comprExt == "uncompressed")
1029 {
1030 Desc.URI = URI;
1031 if(Target)
1032 MetaKey = string(Target->MetaKey);
1033 }
1034 else
1035 {
1036 Desc.URI = URI + '.' + comprExt;
1037 if(Target)
1038 MetaKey = string(Target->MetaKey) + '.' + comprExt;
1039 }
1040
1041 // load the filesize
1042 if(MetaIndexParser)
1043 {
1044 indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey);
1045 if(Record)
1046 FileSize = Record->Size;
1047
1048 InitByHashIfNeeded(MetaKey);
1049 }
1050
1051 Desc.Description = URIDesc;
1052 Desc.Owner = this;
1053 Desc.ShortDesc = ShortDesc;
1054
1055 QueueURI(Desc);
1056 }
1057 /*}}}*/
1058 // AcqIndex::AdjustForByHash - modify URI for by-hash support /*{{{*/
1059 // ---------------------------------------------------------------------
1060 /* */
1061 void pkgAcqIndex::InitByHashIfNeeded(const std::string MetaKey)
1062 {
1063 // TODO:
1064 // - (maybe?) add support for by-hash into the sources.list as flag
1065 // - make apt-ftparchive generate the hashes (and expire?)
1066 std::string HostKnob = "APT::Acquire::" + ::URI(Desc.URI).Host + "::By-Hash";
1067 if(_config->FindB("APT::Acquire::By-Hash", false) == true ||
1068 _config->FindB(HostKnob, false) == true ||
1069 MetaIndexParser->GetSupportsAcquireByHash())
1070 {
1071 indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey);
1072 if(Record)
1073 {
1074 // FIXME: should we really use the best hash here? or a fixed one?
1075 const HashString *TargetHash = Record->Hashes.find("");
1076 std::string ByHash = "/by-hash/" + TargetHash->HashType() + "/" + TargetHash->HashValue();
1077 size_t trailing_slash = Desc.URI.find_last_of("/");
1078 Desc.URI = Desc.URI.replace(
1079 trailing_slash,
1080 Desc.URI.substr(trailing_slash+1).size()+1,
1081 ByHash);
1082 } else {
1083 _error->Warning(
1084 "Fetching ByHash requested but can not find record for %s",
1085 MetaKey.c_str());
1086 }
1087 }
1088 }
1089 /*}}}*/
1090 // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/
1091 // ---------------------------------------------------------------------
1092 /* The only header we use is the last-modified header. */
1093 string pkgAcqIndex::Custom600Headers() const
1094 {
1095 string Final = _config->FindDir("Dir::State::lists");
1096 Final += URItoFileName(RealURI);
1097 if (_config->FindB("Acquire::GzipIndexes",false))
1098 Final += ".gz";
1099
1100 string msg = "\nIndex-File: true";
1101 // FIXME: this really should use "IndexTarget::IsOptional()" but that
1102 // seems to be difficult without breaking ABI
1103 if (ShortDesc().find("Translation") != 0)
1104 msg += "\nFail-Ignore: true";
1105 struct stat Buf;
1106 if (stat(Final.c_str(),&Buf) == 0)
1107 msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1108
1109 return msg;
1110 }
1111 /*}}}*/
1112 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/
1113 {
1114 size_t const nextExt = CompressionExtension.find(' ');
1115 if (nextExt != std::string::npos)
1116 {
1117 CompressionExtension = CompressionExtension.substr(nextExt+1);
1118 Init(RealURI, Desc.Description, Desc.ShortDesc);
1119 return;
1120 }
1121
1122 // on decompression failure, remove bad versions in partial/
1123 if (Decompression && Erase) {
1124 string s = _config->FindDir("Dir::State::lists") + "partial/";
1125 s.append(URItoFileName(RealURI));
1126 unlink(s.c_str());
1127 }
1128
1129 Item::Failed(Message,Cnf);
1130
1131 /// cancel the entire transaction
1132 Owner->AbortTransaction(TransactionID);
1133 }
1134 /*}}}*/
1135 // AcqIndex::Done - Finished a fetch /*{{{*/
1136 // ---------------------------------------------------------------------
1137 /* This goes through a number of states.. On the initial fetch the
1138 method could possibly return an alternate filename which points
1139 to the uncompressed version of the file. If this is so the file
1140 is copied into the partial directory. In all other cases the file
1141 is decompressed with a gzip uri. */
1142 void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes,
1143 pkgAcquire::MethodConfig *Cfg)
1144 {
1145 Item::Done(Message,Size,Hashes,Cfg);
1146
1147 if (Decompression == true)
1148 {
1149 if (ExpectedHashes.usable() && ExpectedHashes != Hashes)
1150 {
1151 RenameOnError(HashSumMismatch);
1152 printHashSumComparision(RealURI, ExpectedHashes, Hashes);
1153 Failed(Message, Cfg);
1154 return;
1155 }
1156
1157 // FIXME: this can go away once we only ever download stuff that
1158 // has a valid hash and we never do GET based probing
1159 //
1160 /* Always verify the index file for correctness (all indexes must
1161 * have a Package field) (LP: #346386) (Closes: #627642)
1162 */
1163 FileFd fd(DestFile, FileFd::ReadOnly);
1164 // Only test for correctness if the file is not empty (empty is ok)
1165 if (fd.FileSize() > 0)
1166 {
1167 pkgTagSection sec;
1168 pkgTagFile tag(&fd);
1169
1170 // all our current indexes have a field 'Package' in each section
1171 if (_error->PendingError() == true || tag.Step(sec) == false || sec.Exists("Package") == false)
1172 {
1173 RenameOnError(InvalidFormat);
1174 Failed(Message, Cfg);
1175 return;
1176 }
1177 }
1178
1179 // Done, queue for rename on transaction finished
1180 PartialFile = DestFile;
1181
1182 string FinalFile = _config->FindDir("Dir::State::lists");
1183 FinalFile += URItoFileName(RealURI);
1184 DestFile = FinalFile;
1185 #if 0
1186 /* We restore the original name to DestFile so that the clean operation
1187 will work OK */
1188 DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1189 DestFile += URItoFileName(RealURI);
1190
1191 // Remove the compressed version.
1192 if (Erase == true)
1193 unlink(DestFile.c_str());
1194 #endif
1195 return;
1196 }
1197
1198 Erase = false;
1199 Complete = true;
1200
1201 // Handle the unzipd case
1202 string FileName = LookupTag(Message,"Alt-Filename");
1203 if (FileName.empty() == false)
1204 {
1205 // The files timestamp matches
1206 if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
1207 return;
1208 Decompression = true;
1209 Local = true;
1210 DestFile += ".decomp";
1211 Desc.URI = "copy:" + FileName;
1212 QueueURI(Desc);
1213 Mode = "copy";
1214 return;
1215 }
1216
1217 FileName = LookupTag(Message,"Filename");
1218 if (FileName.empty() == true)
1219 {
1220 Status = StatError;
1221 ErrorText = "Method gave a blank filename";
1222 }
1223
1224 std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
1225
1226 // The files timestamp matches
1227 if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) {
1228 if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
1229 // Update DestFile for .gz suffix so that the clean operation keeps it
1230 DestFile += ".gz";
1231 return;
1232 }
1233
1234 if (FileName == DestFile)
1235 Erase = true;
1236 else
1237 Local = true;
1238
1239 string decompProg;
1240
1241 // If we enable compressed indexes and already have gzip, keep it
1242 if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) {
1243 string FinalFile = _config->FindDir("Dir::State::lists");
1244 FinalFile += URItoFileName(RealURI) + ".gz";
1245 Rename(DestFile,FinalFile);
1246 chmod(FinalFile.c_str(),0644);
1247
1248 // Update DestFile for .gz suffix so that the clean operation keeps it
1249 DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1250 DestFile += URItoFileName(RealURI) + ".gz";
1251 return;
1252 }
1253
1254 // get the binary name for your used compression type
1255 decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
1256 if(decompProg.empty() == false);
1257 else if(compExt == "uncompressed")
1258 decompProg = "copy";
1259 else {
1260 _error->Error("Unsupported extension: %s", compExt.c_str());
1261 return;
1262 }
1263
1264 Decompression = true;
1265 DestFile += ".decomp";
1266 Desc.URI = decompProg + ":" + FileName;
1267 QueueURI(Desc);
1268
1269 // FIXME: this points to a c++ string that goes out of scope
1270 Mode = decompProg.c_str();
1271 }
1272 /*}}}*/
1273 // AcqIndexTrans::pkgAcqIndexTrans - Constructor /*{{{*/
1274 // ---------------------------------------------------------------------
1275 /* The Translation file is added to the queue */
1276 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
1277 string URI,string URIDesc,string ShortDesc)
1278 : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashStringList(), "")
1279 {
1280 }
1281 /*}}}*/
1282 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
1283 unsigned long TransactionID,
1284 IndexTarget const * const Target,
1285 HashStringList const &ExpectedHashes,
1286 indexRecords *MetaIndexParser)
1287 : pkgAcqIndex(Owner, TransactionID, Target, ExpectedHashes, MetaIndexParser)
1288 {
1289 // load the filesize
1290 indexRecords::checkSum *Record = MetaIndexParser->Lookup(string(Target->MetaKey));
1291 if(Record)
1292 FileSize = Record->Size;
1293 }
1294 /*}}}*/
1295 // AcqIndexTrans::Custom600Headers - Insert custom request headers /*{{{*/
1296 // ---------------------------------------------------------------------
1297 string pkgAcqIndexTrans::Custom600Headers() const
1298 {
1299 string Final = _config->FindDir("Dir::State::lists");
1300 Final += URItoFileName(RealURI);
1301
1302 struct stat Buf;
1303 if (stat(Final.c_str(),&Buf) != 0)
1304 return "\nFail-Ignore: true\nIndex-File: true";
1305 return "\nFail-Ignore: true\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1306 }
1307 /*}}}*/
1308 // AcqIndexTrans::Failed - Silence failure messages for missing files /*{{{*/
1309 // ---------------------------------------------------------------------
1310 /* */
1311 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1312 {
1313 size_t const nextExt = CompressionExtension.find(' ');
1314 if (nextExt != std::string::npos)
1315 {
1316 CompressionExtension = CompressionExtension.substr(nextExt+1);
1317 Init(RealURI, Desc.Description, Desc.ShortDesc);
1318 Status = StatIdle;
1319 return;
1320 }
1321
1322 // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor
1323 if (Cnf->LocalOnly == true ||
1324 StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1325 {
1326 // Ignore this
1327 Status = StatDone;
1328 Complete = false;
1329 Dequeue();
1330 return;
1331 }
1332
1333 Item::Failed(Message,Cnf);
1334 }
1335 /*}}}*/
1336 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner, /*{{{*/
1337 unsigned long TransactionID,
1338 string URI,string URIDesc,string ShortDesc,
1339 string MetaIndexFile,
1340 const vector<IndexTarget*>* IndexTargets,
1341 indexRecords* MetaIndexParser) :
1342 Item(Owner, HashStringList(), TransactionID), RealURI(URI),
1343 MetaIndexParser(MetaIndexParser), MetaIndexFile(MetaIndexFile),
1344 IndexTargets(IndexTargets), AuthPass(false), IMSHit(false)
1345 {
1346 DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1347 DestFile += URItoFileName(URI);
1348
1349 // remove any partial downloaded sig-file in partial/.
1350 // it may confuse proxies and is too small to warrant a
1351 // partial download anyway
1352 unlink(DestFile.c_str());
1353
1354 // set the TransactionID
1355 if(_config->FindB("Debug::Acquire::Transaction", false) == true)
1356 std::clog << "New pkgAcqMetaSig with TransactionID "
1357 << TransactionID << std::endl;
1358
1359 // Create the item
1360 Desc.Description = URIDesc;
1361 Desc.Owner = this;
1362 Desc.ShortDesc = ShortDesc;
1363 Desc.URI = URI;
1364
1365 QueueURI(Desc);
1366 }
1367 /*}}}*/
1368 pkgAcqMetaSig::~pkgAcqMetaSig() /*{{{*/
1369 {
1370 }
1371 /*}}}*/
1372 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers /*{{{*/
1373 // ---------------------------------------------------------------------
1374 /* The only header we use is the last-modified header. */
1375 string pkgAcqMetaSig::Custom600Headers() const
1376 {
1377 string FinalFile = _config->FindDir("Dir::State::lists");
1378 FinalFile += URItoFileName(RealURI);
1379
1380 struct stat Buf;
1381 if (stat(FinalFile.c_str(),&Buf) != 0)
1382 return "\nIndex-File: true";
1383
1384 return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1385 }
1386
1387 void pkgAcqMetaSig::Done(string Message,unsigned long long Size, HashStringList const &Hashes,
1388 pkgAcquire::MethodConfig *Cfg)
1389 {
1390 Item::Done(Message, Size, Hashes, Cfg);
1391
1392 string FileName = LookupTag(Message,"Filename");
1393 if (FileName.empty() == true)
1394 {
1395 Status = StatError;
1396 ErrorText = "Method gave a blank filename";
1397 return;
1398 }
1399
1400 if (FileName != DestFile)
1401 {
1402 // We have to copy it into place
1403 Local = true;
1404 Desc.URI = "copy:" + FileName;
1405 QueueURI(Desc);
1406 return;
1407 }
1408
1409 if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1410 IMSHit = true;
1411
1412 // adjust paths if its a ims-hit
1413 if(IMSHit)
1414 {
1415 string FinalFile = _config->FindDir("Dir::State::lists");
1416 FinalFile += URItoFileName(RealURI);
1417
1418 DestFile = PartialFile = FinalFile;
1419 }
1420
1421 // queue for verify
1422 if(AuthPass == false)
1423 {
1424 AuthPass = true;
1425 Desc.URI = "gpgv:" + DestFile;
1426 DestFile = MetaIndexFile;
1427 QueueURI(Desc);
1428 return;
1429 }
1430
1431 // queue to copy the file in place if it was not a ims hit, on ims
1432 // hit the file is already at the right place
1433 if(IMSHit == false)
1434 {
1435 PartialFile = _config->FindDir("Dir::State::lists") + "partial/";
1436 PartialFile += URItoFileName(RealURI);
1437
1438 DestFile = _config->FindDir("Dir::State::lists");
1439 DestFile += URItoFileName(RealURI);
1440 }
1441
1442 Complete = true;
1443
1444 }
1445 /*}}}*/
1446 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
1447 {
1448 string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1449
1450 // this ensures that any file in the lists/ dir is removed by the
1451 // transaction
1452 DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1453 DestFile += URItoFileName(RealURI);
1454 PartialFile = "";
1455
1456 // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor
1457 if (Cnf->LocalOnly == true ||
1458 StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1459 {
1460 // Ignore this
1461 Status = StatDone;
1462 Complete = false;
1463 Dequeue();
1464 return;
1465 }
1466 Item::Failed(Message,Cnf);
1467 }
1468 /*}}}*/
1469 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner, /*{{{*/
1470 unsigned long TransactionID,
1471 string URI,string URIDesc,string ShortDesc,
1472 string MetaIndexSigURI,string MetaIndexSigURIDesc, string MetaIndexSigShortDesc,
1473 const vector<IndexTarget*>* IndexTargets,
1474 indexRecords* MetaIndexParser) :
1475 Item(Owner, HashStringList(), TransactionID), RealURI(URI), IndexTargets(IndexTargets),
1476 MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false),
1477 MetaIndexSigURI(MetaIndexSigURI), MetaIndexSigURIDesc(MetaIndexSigURIDesc),
1478 MetaIndexSigShortDesc(MetaIndexSigShortDesc)
1479 {
1480 if(TransactionID == 0)
1481 this->TransactionID = (unsigned long)this;
1482
1483 if(_config->FindB("Debug::Acquire::Transaction", false) == true)
1484 std::clog << "New pkgAcqMetaIndex with TransactionID "
1485 << TransactionID << std::endl;
1486
1487 Init(URIDesc, ShortDesc);
1488 }
1489 /*}}}*/
1490 // pkgAcqMetaIndex::Init - Delayed constructor /*{{{*/
1491 void pkgAcqMetaIndex::Init(std::string URIDesc, std::string ShortDesc)
1492 {
1493 DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1494 DestFile += URItoFileName(RealURI);
1495
1496 // Create the item
1497 Desc.Description = URIDesc;
1498 Desc.Owner = this;
1499 Desc.ShortDesc = ShortDesc;
1500 Desc.URI = RealURI;
1501
1502 // we expect more item
1503 ExpectedAdditionalItems = IndexTargets->size();
1504 QueueURI(Desc);
1505 }
1506 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers /*{{{*/
1507 // ---------------------------------------------------------------------
1508 /* The only header we use is the last-modified header. */
1509 string pkgAcqMetaIndex::Custom600Headers() const
1510 {
1511 string Final = _config->FindDir("Dir::State::lists");
1512 Final += URItoFileName(RealURI);
1513
1514 struct stat Buf;
1515 if (stat(Final.c_str(),&Buf) != 0)
1516 return "\nIndex-File: true";
1517
1518 return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1519 }
1520 /*}}}*/
1521 void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/
1522 pkgAcquire::MethodConfig *Cfg)
1523 {
1524 Item::Done(Message,Size,Hashes,Cfg);
1525
1526 // MetaIndexes are done in two passes: one to download the
1527 // metaindex with an appropriate method, and a second to verify it
1528 // with the gpgv method
1529
1530 if (AuthPass == true)
1531 {
1532 AuthDone(Message);
1533
1534 // all cool, move Release file into place
1535 Complete = true;
1536 }
1537 else
1538 {
1539 RetrievalDone(Message);
1540 if (!Complete)
1541 // Still more retrieving to do
1542 return;
1543
1544 if (SigFile == "")
1545 {
1546 // load indexes, the signature will downloaded afterwards
1547 MetaIndexParser->Load(DestFile);
1548 QueueIndexes(true);
1549 }
1550 else
1551 {
1552 // FIXME: move this into pkgAcqMetaClearSig::Done on the next
1553 // ABI break
1554
1555 // if we expect a ClearTextSignature (InRelase), ensure that
1556 // this is what we get and if not fail to queue a
1557 // Release/Release.gpg, see #346386
1558 if (SigFile == DestFile && !StartsWithGPGClearTextSignature(DestFile))
1559 {
1560 Failed(Message, Cfg);
1561 return;
1562 }
1563
1564 // There was a signature file, so pass it to gpgv for
1565 // verification
1566 if (_config->FindB("Debug::pkgAcquire::Auth", false))
1567 std::cerr << "Metaindex acquired, queueing gpg verification ("
1568 << SigFile << "," << DestFile << ")\n";
1569 AuthPass = true;
1570 Desc.URI = "gpgv:" + SigFile;
1571 QueueURI(Desc);
1572 Mode = "gpgv";
1573 return;
1574 }
1575 }
1576
1577 if (Complete == true)
1578 {
1579 string FinalFile = _config->FindDir("Dir::State::lists");
1580 FinalFile += URItoFileName(RealURI);
1581 if (SigFile == DestFile)
1582 SigFile = FinalFile;
1583 // queue for copy in place
1584 PartialFile = DestFile;
1585 DestFile = FinalFile;
1586 }
1587 }
1588 /*}}}*/
1589 void pkgAcqMetaIndex::RetrievalDone(string Message) /*{{{*/
1590 {
1591 // We have just finished downloading a Release file (it is not
1592 // verified yet)
1593
1594 string FileName = LookupTag(Message,"Filename");
1595 if (FileName.empty() == true)
1596 {
1597 Status = StatError;
1598 ErrorText = "Method gave a blank filename";
1599 return;
1600 }
1601
1602 if (FileName != DestFile)
1603 {
1604 Local = true;
1605 Desc.URI = "copy:" + FileName;
1606 QueueURI(Desc);
1607 return;
1608 }
1609
1610 // make sure to verify against the right file on I-M-S hit
1611 IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1612 if(IMSHit)
1613 {
1614 string FinalFile = _config->FindDir("Dir::State::lists");
1615 FinalFile += URItoFileName(RealURI);
1616 if (SigFile == DestFile)
1617 {
1618 SigFile = FinalFile;
1619 // constructor of pkgAcqMetaClearSig moved it out of the way,
1620 // now move it back in on IMS hit for the 'old' file
1621 string const OldClearSig = DestFile + ".reverify";
1622 if (RealFileExists(OldClearSig) == true)
1623 Rename(OldClearSig, FinalFile);
1624 }
1625 DestFile = FinalFile;
1626 }
1627
1628 // queue a signature
1629 if(SigFile != DestFile)
1630 new pkgAcqMetaSig(Owner, TransactionID,
1631 MetaIndexSigURI, MetaIndexSigURIDesc,
1632 MetaIndexSigShortDesc, DestFile, IndexTargets,
1633 MetaIndexParser);
1634
1635 Complete = true;
1636 }
1637 /*}}}*/
1638 void pkgAcqMetaIndex::AuthDone(string Message) /*{{{*/
1639 {
1640 // At this point, the gpgv method has succeeded, so there is a
1641 // valid signature from a key in the trusted keyring. We
1642 // perform additional verification of its contents, and use them
1643 // to verify the indexes we are about to download
1644
1645 if (!MetaIndexParser->Load(DestFile))
1646 {
1647 Status = StatAuthError;
1648 ErrorText = MetaIndexParser->ErrorText;
1649 return;
1650 }
1651
1652 if (!VerifyVendor(Message))
1653 {
1654 return;
1655 }
1656
1657 if (_config->FindB("Debug::pkgAcquire::Auth", false))
1658 std::cerr << "Signature verification succeeded: "
1659 << DestFile << std::endl;
1660
1661 // Download further indexes with verification
1662 QueueIndexes(true);
1663
1664 #if 0
1665 // is it a clearsigned MetaIndex file?
1666 if (DestFile == SigFile)
1667 return;
1668
1669 // Done, move signature file into position
1670 string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1671 URItoFileName(RealURI) + ".gpg";
1672 Rename(SigFile,VerifiedSigFile);
1673 chmod(VerifiedSigFile.c_str(),0644);
1674 #endif
1675 }
1676 /*}}}*/
1677 void pkgAcqMetaIndex::QueueIndexes(bool verify) /*{{{*/
1678 {
1679 bool transInRelease = false;
1680 {
1681 std::vector<std::string> const keys = MetaIndexParser->MetaKeys();
1682 for (std::vector<std::string>::const_iterator k = keys.begin(); k != keys.end(); ++k)
1683 // FIXME: Feels wrong to check for hardcoded string here, but what should we do else…
1684 if (k->find("Translation-") != std::string::npos)
1685 {
1686 transInRelease = true;
1687 break;
1688 }
1689 }
1690
1691 // at this point the real Items are loaded in the fetcher
1692 ExpectedAdditionalItems = 0;
1693 for (vector <IndexTarget*>::const_iterator Target = IndexTargets->begin();
1694 Target != IndexTargets->end();
1695 ++Target)
1696 {
1697 HashStringList ExpectedIndexHashes;
1698 const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1699 bool compressedAvailable = false;
1700 if (Record == NULL)
1701 {
1702 if ((*Target)->IsOptional() == true)
1703 {
1704 std::vector<std::string> types = APT::Configuration::getCompressionTypes();
1705 for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
1706 if (MetaIndexParser->Exists((*Target)->MetaKey + "." + *t) == true)
1707 {
1708 compressedAvailable = true;
1709 break;
1710 }
1711 }
1712 else if (verify == true)
1713 {
1714 Status = StatAuthError;
1715 strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), (*Target)->MetaKey.c_str());
1716 return;
1717 }
1718 }
1719 else
1720 {
1721 ExpectedIndexHashes = Record->Hashes;
1722 if (_config->FindB("Debug::pkgAcquire::Auth", false))
1723 {
1724 std::cerr << "Queueing: " << (*Target)->URI << std::endl
1725 << "Expected Hash:" << std::endl;
1726 for (HashStringList::const_iterator hs = ExpectedIndexHashes.begin(); hs != ExpectedIndexHashes.end(); ++hs)
1727 std::cerr << "\t- " << hs->toStr() << std::endl;
1728 std::cerr << "For: " << Record->MetaKeyFilename << std::endl;
1729 }
1730 if (verify == true && ExpectedIndexHashes.empty() == true && (*Target)->IsOptional() == false)
1731 {
1732 Status = StatAuthError;
1733 strprintf(ErrorText, _("Unable to find hash sum for '%s' in Release file"), (*Target)->MetaKey.c_str());
1734 return;
1735 }
1736 }
1737
1738 if ((*Target)->IsOptional() == true)
1739 {
1740 if ((*Target)->IsSubIndex() == true)
1741 new pkgAcqSubIndex(Owner, TransactionID,
1742 (*Target)->URI, (*Target)->Description,
1743 (*Target)->ShortDesc, ExpectedIndexHashes);
1744 else if (transInRelease == false || Record != NULL || compressedAvailable == true)
1745 {
1746 if (_config->FindB("Acquire::PDiffs",true) == true && transInRelease == true &&
1747 MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true)
1748 new pkgAcqDiffIndex(Owner, TransactionID, *Target, ExpectedIndexHashes, MetaIndexParser);
1749 else
1750 new pkgAcqIndexTrans(Owner, TransactionID, *Target, ExpectedIndexHashes, MetaIndexParser);
1751 }
1752 continue;
1753 }
1754
1755 /* Queue Packages file (either diff or full packages files, depending
1756 on the users option) - we also check if the PDiff Index file is listed
1757 in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
1758 instead, but passing the required info to it is to much hassle */
1759 if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
1760 MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true))
1761 new pkgAcqDiffIndex(Owner, TransactionID, *Target, ExpectedIndexHashes, MetaIndexParser);
1762 else
1763 new pkgAcqIndex(Owner, TransactionID, *Target, ExpectedIndexHashes, MetaIndexParser);
1764 }
1765 }
1766 /*}}}*/
1767 bool pkgAcqMetaIndex::VerifyVendor(string Message) /*{{{*/
1768 {
1769 string::size_type pos;
1770
1771 // check for missing sigs (that where not fatal because otherwise we had
1772 // bombed earlier)
1773 string missingkeys;
1774 string msg = _("There is no public key available for the "
1775 "following key IDs:\n");
1776 pos = Message.find("NO_PUBKEY ");
1777 if (pos != std::string::npos)
1778 {
1779 string::size_type start = pos+strlen("NO_PUBKEY ");
1780 string Fingerprint = Message.substr(start, Message.find("\n")-start);
1781 missingkeys += (Fingerprint);
1782 }
1783 if(!missingkeys.empty())
1784 _error->Warning("%s", (msg + missingkeys).c_str());
1785
1786 string Transformed = MetaIndexParser->GetExpectedDist();
1787
1788 if (Transformed == "../project/experimental")
1789 {
1790 Transformed = "experimental";
1791 }
1792
1793 pos = Transformed.rfind('/');
1794 if (pos != string::npos)
1795 {
1796 Transformed = Transformed.substr(0, pos);
1797 }
1798
1799 if (Transformed == ".")
1800 {
1801 Transformed = "";
1802 }
1803
1804 if (_config->FindB("Acquire::Check-Valid-Until", true) == true &&
1805 MetaIndexParser->GetValidUntil() > 0) {
1806 time_t const invalid_since = time(NULL) - MetaIndexParser->GetValidUntil();
1807 if (invalid_since > 0)
1808 // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1809 // the time since then the file is invalid - formated in the same way as in
1810 // the download progress display (e.g. 7d 3h 42min 1s)
1811 return _error->Error(
1812 _("Release file for %s is expired (invalid since %s). "
1813 "Updates for this repository will not be applied."),
1814 RealURI.c_str(), TimeToStr(invalid_since).c_str());
1815 }
1816
1817 if (_config->FindB("Debug::pkgAcquire::Auth", false))
1818 {
1819 std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1820 std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1821 std::cerr << "Transformed Dist: " << Transformed << std::endl;
1822 }
1823
1824 if (MetaIndexParser->CheckDist(Transformed) == false)
1825 {
1826 // This might become fatal one day
1827 // Status = StatAuthError;
1828 // ErrorText = "Conflicting distribution; expected "
1829 // + MetaIndexParser->GetExpectedDist() + " but got "
1830 // + MetaIndexParser->GetDist();
1831 // return false;
1832 if (!Transformed.empty())
1833 {
1834 _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1835 Desc.Description.c_str(),
1836 Transformed.c_str(),
1837 MetaIndexParser->GetDist().c_str());
1838 }
1839 }
1840
1841 return true;
1842 }
1843 /*}}}*/
1844 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present /*{{{*/
1845 // ---------------------------------------------------------------------
1846 /* */
1847 void pkgAcqMetaIndex::Failed(string /*Message*/,
1848 pkgAcquire::MethodConfig * /*Cnf*/)
1849 {
1850 #if 0
1851 if (AuthPass == true)
1852 {
1853 // gpgv method failed, if we have a good signature
1854 string LastGoodSigFile = _config->FindDir("Dir::State::lists").append("partial/").append(URItoFileName(RealURI));
1855 if (DestFile != SigFile)
1856 LastGoodSigFile.append(".gpg");
1857 LastGoodSigFile.append(".reverify");
1858
1859 if(FileExists(LastGoodSigFile))
1860 {
1861 string VerifiedSigFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1862 if (DestFile != SigFile)
1863 VerifiedSigFile.append(".gpg");
1864 Rename(LastGoodSigFile, VerifiedSigFile);
1865 Status = StatTransientNetworkError;
1866 _error->Warning(_("An error occurred during the signature "
1867 "verification. The repository is not updated "
1868 "and the previous index files will be used. "
1869 "GPG error: %s: %s\n"),
1870 Desc.Description.c_str(),
1871 LookupTag(Message,"Message").c_str());
1872 RunScripts("APT::Update::Auth-Failure");
1873 return;
1874 } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) {
1875 /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */
1876 _error->Error(_("GPG error: %s: %s"),
1877 Desc.Description.c_str(),
1878 LookupTag(Message,"Message").c_str());
1879 return;
1880 } else {
1881 _error->Warning(_("GPG error: %s: %s"),
1882 Desc.Description.c_str(),
1883 LookupTag(Message,"Message").c_str());
1884 }
1885 // gpgv method failed
1886 ReportMirrorFailure("GPGFailure");
1887 }
1888 #endif
1889 /* Always move the meta index, even if gpgv failed. This ensures
1890 * that PackageFile objects are correctly filled in */
1891 if (FileExists(DestFile)) {
1892 string FinalFile = _config->FindDir("Dir::State::lists");
1893 FinalFile += URItoFileName(RealURI);
1894 /* InRelease files become Release files, otherwise
1895 * they would be considered as trusted later on */
1896 if (SigFile == DestFile) {
1897 RealURI = RealURI.replace(RealURI.rfind("InRelease"), 9,
1898 "Release");
1899 FinalFile = FinalFile.replace(FinalFile.rfind("InRelease"), 9,
1900 "Release");
1901 SigFile = FinalFile;
1902 }
1903 Rename(DestFile,FinalFile);
1904 chmod(FinalFile.c_str(),0644);
1905
1906 DestFile = FinalFile;
1907 }
1908
1909 // No Release file was present, or verification failed, so fall
1910 // back to queueing Packages files without verification
1911 QueueIndexes(false);
1912 }
1913 /*}}}*/
1914
1915 void pkgAcqMetaIndex::Finished()
1916 {
1917 if(_config->FindB("Debug::Acquire::Transaction", false) == true)
1918 std::clog << "Finished: " << DestFile <<std::endl;
1919 if(Owner->TransactionHasError(TransactionID) == false &&
1920 TransactionID > 0)
1921 Owner->CommitTransaction(TransactionID);
1922 }
1923
1924
1925 pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner, /*{{{*/
1926 string const &URI, string const &URIDesc, string const &ShortDesc,
1927 string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc,
1928 string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc,
1929 const vector<IndexTarget*>* IndexTargets,
1930 indexRecords* MetaIndexParser) :
1931 pkgAcqMetaIndex(Owner, (unsigned long)this, URI, URIDesc, ShortDesc, MetaSigURI, MetaSigURIDesc,MetaSigShortDesc, IndexTargets, MetaIndexParser),
1932 MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1933 MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc)
1934 {
1935 SigFile = DestFile;
1936
1937 // index targets + (worst case:) Release/Release.gpg
1938 ExpectedAdditionalItems = IndexTargets->size() + 2;
1939
1940 // keep the old InRelease around in case of transistent network errors
1941 string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1942 if (RealFileExists(Final) == true)
1943 {
1944 string const LastGoodSig = DestFile + ".reverify";
1945 Rename(Final,LastGoodSig);
1946 }
1947 }
1948 /*}}}*/
1949 pkgAcqMetaClearSig::~pkgAcqMetaClearSig() /*{{{*/
1950 {
1951 // if the file was never queued undo file-changes done in the constructor
1952 if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false)
1953 {
1954 string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1955 string const LastGoodSig = DestFile + ".reverify";
1956 if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true)
1957 Rename(LastGoodSig, Final);
1958 }
1959 }
1960 /*}}}*/
1961 // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/
1962 // ---------------------------------------------------------------------
1963 // FIXME: this can go away once the InRelease file is used widely
1964 string pkgAcqMetaClearSig::Custom600Headers() const
1965 {
1966 string Final = _config->FindDir("Dir::State::lists");
1967 Final += URItoFileName(RealURI);
1968
1969 struct stat Buf;
1970 if (stat(Final.c_str(),&Buf) != 0)
1971 {
1972 Final = DestFile + ".reverify";
1973 if (stat(Final.c_str(),&Buf) != 0)
1974 return "\nIndex-File: true\nFail-Ignore: true\n";
1975 }
1976
1977 return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1978 }
1979 /*}}}*/
1980 void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/
1981 {
1982 // we failed, we will not get additional items from this method
1983 ExpectedAdditionalItems = 0;
1984
1985 if (AuthPass == false)
1986 {
1987 // Remove the 'old' InRelease file if we try Release.gpg now as otherwise
1988 // the file will stay around and gives a false-auth impression (CVE-2012-0214)
1989 string FinalFile = _config->FindDir("Dir::State::lists");
1990 FinalFile.append(URItoFileName(RealURI));
1991 if (FileExists(FinalFile))
1992 unlink(FinalFile.c_str());
1993
1994 new pkgAcqMetaIndex(Owner, TransactionID,
1995 MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1996 MetaSigURI, MetaSigURIDesc, MetaSigShortDesc,
1997 IndexTargets, MetaIndexParser);
1998 if (Cnf->LocalOnly == true ||
1999 StringToBool(LookupTag(Message, "Transient-Failure"), false) == false)
2000 Dequeue();
2001 }
2002 else
2003 pkgAcqMetaIndex::Failed(Message, Cnf);
2004 }
2005 /*}}}*/
2006 // AcqArchive::AcqArchive - Constructor /*{{{*/
2007 // ---------------------------------------------------------------------
2008 /* This just sets up the initial fetch environment and queues the first
2009 possibilitiy */
2010 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
2011 pkgRecords *Recs,pkgCache::VerIterator const &Version,
2012 string &StoreFilename) :
2013 Item(Owner, HashStringList()), Version(Version), Sources(Sources), Recs(Recs),
2014 StoreFilename(StoreFilename), Vf(Version.FileList()),
2015 Trusted(false)
2016 {
2017 Retries = _config->FindI("Acquire::Retries",0);
2018
2019 if (Version.Arch() == 0)
2020 {
2021 _error->Error(_("I wasn't able to locate a file for the %s package. "
2022 "This might mean you need to manually fix this package. "
2023 "(due to missing arch)"),
2024 Version.ParentPkg().FullName().c_str());
2025 return;
2026 }
2027
2028 /* We need to find a filename to determine the extension. We make the
2029 assumption here that all the available sources for this version share
2030 the same extension.. */
2031 // Skip not source sources, they do not have file fields.
2032 for (; Vf.end() == false; ++Vf)
2033 {
2034 if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
2035 continue;
2036 break;
2037 }
2038
2039 // Does not really matter here.. we are going to fail out below
2040 if (Vf.end() != true)
2041 {
2042 // If this fails to get a file name we will bomb out below.
2043 pkgRecords::Parser &Parse = Recs->Lookup(Vf);
2044 if (_error->PendingError() == true)
2045 return;
2046
2047 // Generate the final file name as: package_version_arch.foo
2048 StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
2049 QuoteString(Version.VerStr(),"_:") + '_' +
2050 QuoteString(Version.Arch(),"_:.") +
2051 "." + flExtension(Parse.FileName());
2052 }
2053
2054 // check if we have one trusted source for the package. if so, switch
2055 // to "TrustedOnly" mode - but only if not in AllowUnauthenticated mode
2056 bool const allowUnauth = _config->FindB("APT::Get::AllowUnauthenticated", false);
2057 bool const debugAuth = _config->FindB("Debug::pkgAcquire::Auth", false);
2058 bool seenUntrusted = false;
2059 for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; ++i)
2060 {
2061 pkgIndexFile *Index;
2062 if (Sources->FindIndex(i.File(),Index) == false)
2063 continue;
2064
2065 if (debugAuth == true)
2066 std::cerr << "Checking index: " << Index->Describe()
2067 << "(Trusted=" << Index->IsTrusted() << ")" << std::endl;
2068
2069 if (Index->IsTrusted() == true)
2070 {
2071 Trusted = true;
2072 if (allowUnauth == false)
2073 break;
2074 }
2075 else
2076 seenUntrusted = true;
2077 }
2078
2079 // "allow-unauthenticated" restores apts old fetching behaviour
2080 // that means that e.g. unauthenticated file:// uris are higher
2081 // priority than authenticated http:// uris
2082 if (allowUnauth == true && seenUntrusted == true)
2083 Trusted = false;
2084
2085 // Select a source
2086 if (QueueNext() == false && _error->PendingError() == false)
2087 _error->Error(_("Can't find a source to download version '%s' of '%s'"),
2088 Version.VerStr(), Version.ParentPkg().FullName(false).c_str());
2089 }
2090 /*}}}*/
2091 // AcqArchive::QueueNext - Queue the next file source /*{{{*/
2092 // ---------------------------------------------------------------------
2093 /* This queues the next available file version for download. It checks if
2094 the archive is already available in the cache and stashs the MD5 for
2095 checking later. */
2096 bool pkgAcqArchive::QueueNext()
2097 {
2098 for (; Vf.end() == false; ++Vf)
2099 {
2100 // Ignore not source sources
2101 if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
2102 continue;
2103
2104 // Try to cross match against the source list
2105 pkgIndexFile *Index;
2106 if (Sources->FindIndex(Vf.File(),Index) == false)
2107 continue;
2108
2109 // only try to get a trusted package from another source if that source
2110 // is also trusted
2111 if(Trusted && !Index->IsTrusted())
2112 continue;
2113
2114 // Grab the text package record
2115 pkgRecords::Parser &Parse = Recs->Lookup(Vf);
2116 if (_error->PendingError() == true)
2117 return false;
2118
2119 string PkgFile = Parse.FileName();
2120 ExpectedHashes = Parse.Hashes();
2121
2122 if (PkgFile.empty() == true)
2123 return _error->Error(_("The package index files are corrupted. No Filename: "
2124 "field for package %s."),
2125 Version.ParentPkg().Name());
2126
2127 Desc.URI = Index->ArchiveURI(PkgFile);
2128 Desc.Description = Index->ArchiveInfo(Version);
2129 Desc.Owner = this;
2130 Desc.ShortDesc = Version.ParentPkg().FullName(true);
2131
2132 // See if we already have the file. (Legacy filenames)
2133 FileSize = Version->Size;
2134 string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
2135 struct stat Buf;
2136 if (stat(FinalFile.c_str(),&Buf) == 0)
2137 {
2138 // Make sure the size matches
2139 if ((unsigned long long)Buf.st_size == Version->Size)
2140 {
2141 Complete = true;
2142 Local = true;
2143 Status = StatDone;
2144 StoreFilename = DestFile = FinalFile;
2145 return true;
2146 }
2147
2148 /* Hmm, we have a file and its size does not match, this means it is
2149 an old style mismatched arch */
2150 unlink(FinalFile.c_str());
2151 }
2152
2153 // Check it again using the new style output filenames
2154 FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
2155 if (stat(FinalFile.c_str(),&Buf) == 0)
2156 {
2157 // Make sure the size matches
2158 if ((unsigned long long)Buf.st_size == Version->Size)
2159 {
2160 Complete = true;
2161 Local = true;
2162 Status = StatDone;
2163 StoreFilename = DestFile = FinalFile;
2164 return true;
2165 }
2166
2167 /* Hmm, we have a file and its size does not match, this shouldn't
2168 happen.. */
2169 unlink(FinalFile.c_str());
2170 }
2171
2172 DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
2173
2174 // Check the destination file
2175 if (stat(DestFile.c_str(),&Buf) == 0)
2176 {
2177 // Hmm, the partial file is too big, erase it
2178 if ((unsigned long long)Buf.st_size > Version->Size)
2179 unlink(DestFile.c_str());
2180 else
2181 PartialSize = Buf.st_size;
2182 }
2183
2184 // Disables download of archives - useful if no real installation follows,
2185 // e.g. if we are just interested in proposed installation order
2186 if (_config->FindB("Debug::pkgAcqArchive::NoQueue", false) == true)
2187 {
2188 Complete = true;
2189 Local = true;
2190 Status = StatDone;
2191 StoreFilename = DestFile = FinalFile;
2192 return true;
2193 }
2194
2195 // Create the item
2196 Local = false;
2197 QueueURI(Desc);
2198
2199 ++Vf;
2200 return true;
2201 }
2202 return false;
2203 }
2204 /*}}}*/
2205 // AcqArchive::Done - Finished fetching /*{{{*/
2206 // ---------------------------------------------------------------------
2207 /* */
2208 void pkgAcqArchive::Done(string Message,unsigned long long Size, HashStringList const &CalcHashes,
2209 pkgAcquire::MethodConfig *Cfg)
2210 {
2211 Item::Done(Message, Size, CalcHashes, Cfg);
2212
2213 // Check the size
2214 if (Size != Version->Size)
2215 {
2216 RenameOnError(SizeMismatch);
2217 return;
2218 }
2219
2220 // FIXME: could this empty() check impose *any* sort of security issue?
2221 if(ExpectedHashes.usable() && ExpectedHashes != CalcHashes)
2222 {
2223 RenameOnError(HashSumMismatch);
2224 printHashSumComparision(DestFile, ExpectedHashes, CalcHashes);
2225 return;
2226 }
2227
2228 // Grab the output filename
2229 string FileName = LookupTag(Message,"Filename");
2230 if (FileName.empty() == true)
2231 {
2232 Status = StatError;
2233 ErrorText = "Method gave a blank filename";
2234 return;
2235 }
2236
2237 Complete = true;
2238
2239 // Reference filename
2240 if (FileName != DestFile)
2241 {
2242 StoreFilename = DestFile = FileName;
2243 Local = true;
2244 return;
2245 }
2246
2247 // Done, move it into position
2248 string FinalFile = _config->FindDir("Dir::Cache::Archives");
2249 FinalFile += flNotDir(StoreFilename);
2250 Rename(DestFile,FinalFile);
2251
2252 StoreFilename = DestFile = FinalFile;
2253 Complete = true;
2254 }
2255 /*}}}*/
2256 // AcqArchive::Failed - Failure handler /*{{{*/
2257 // ---------------------------------------------------------------------
2258 /* Here we try other sources */
2259 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2260 {
2261 ErrorText = LookupTag(Message,"Message");
2262
2263 /* We don't really want to retry on failed media swaps, this prevents
2264 that. An interesting observation is that permanent failures are not
2265 recorded. */
2266 if (Cnf->Removable == true &&
2267 StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2268 {
2269 // Vf = Version.FileList();
2270 while (Vf.end() == false) ++Vf;
2271 StoreFilename = string();
2272 Item::Failed(Message,Cnf);
2273 return;
2274 }
2275
2276 if (QueueNext() == false)
2277 {
2278 // This is the retry counter
2279 if (Retries != 0 &&
2280 Cnf->LocalOnly == false &&
2281 StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2282 {
2283 Retries--;
2284 Vf = Version.FileList();
2285 if (QueueNext() == true)
2286 return;
2287 }
2288
2289 StoreFilename = string();
2290 Item::Failed(Message,Cnf);
2291 }
2292 }
2293 /*}}}*/
2294 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
2295 // ---------------------------------------------------------------------
2296 APT_PURE bool pkgAcqArchive::IsTrusted() const
2297 {
2298 return Trusted;
2299 }
2300 /*}}}*/
2301 // AcqArchive::Finished - Fetching has finished, tidy up /*{{{*/
2302 // ---------------------------------------------------------------------
2303 /* */
2304 void pkgAcqArchive::Finished()
2305 {
2306 if (Status == pkgAcquire::Item::StatDone &&
2307 Complete == true)
2308 return;
2309 StoreFilename = string();
2310 }
2311 /*}}}*/
2312 // AcqFile::pkgAcqFile - Constructor /*{{{*/
2313 // ---------------------------------------------------------------------
2314 /* The file is added to the queue */
2315 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI, HashStringList const &Hashes,
2316 unsigned long long Size,string Dsc,string ShortDesc,
2317 const string &DestDir, const string &DestFilename,
2318 bool IsIndexFile) :
2319 Item(Owner, Hashes), IsIndexFile(IsIndexFile)
2320 {
2321 Retries = _config->FindI("Acquire::Retries",0);
2322
2323 if(!DestFilename.empty())
2324 DestFile = DestFilename;
2325 else if(!DestDir.empty())
2326 DestFile = DestDir + "/" + flNotDir(URI);
2327 else
2328 DestFile = flNotDir(URI);
2329
2330 // Create the item
2331 Desc.URI = URI;
2332 Desc.Description = Dsc;
2333 Desc.Owner = this;
2334
2335 // Set the short description to the archive component
2336 Desc.ShortDesc = ShortDesc;
2337
2338 // Get the transfer sizes
2339 FileSize = Size;
2340 struct stat Buf;
2341 if (stat(DestFile.c_str(),&Buf) == 0)
2342 {
2343 // Hmm, the partial file is too big, erase it
2344 if ((Size > 0) && (unsigned long long)Buf.st_size > Size)
2345 unlink(DestFile.c_str());
2346 else
2347 PartialSize = Buf.st_size;
2348 }
2349
2350 QueueURI(Desc);
2351 }
2352 /*}}}*/
2353 // AcqFile::Done - Item downloaded OK /*{{{*/
2354 // ---------------------------------------------------------------------
2355 /* */
2356 void pkgAcqFile::Done(string Message,unsigned long long Size,HashStringList const &CalcHashes,
2357 pkgAcquire::MethodConfig *Cnf)
2358 {
2359 Item::Done(Message,Size,CalcHashes,Cnf);
2360
2361 // Check the hash
2362 if(ExpectedHashes.usable() && ExpectedHashes != CalcHashes)
2363 {
2364 RenameOnError(HashSumMismatch);
2365 printHashSumComparision(DestFile, ExpectedHashes, CalcHashes);
2366 return;
2367 }
2368
2369 string FileName = LookupTag(Message,"Filename");
2370 if (FileName.empty() == true)
2371 {
2372 Status = StatError;
2373 ErrorText = "Method gave a blank filename";
2374 return;
2375 }
2376
2377 Complete = true;
2378
2379 // The files timestamp matches
2380 if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
2381 return;
2382
2383 // We have to copy it into place
2384 if (FileName != DestFile)
2385 {
2386 Local = true;
2387 if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
2388 Cnf->Removable == true)
2389 {
2390 Desc.URI = "copy:" + FileName;
2391 QueueURI(Desc);
2392 return;
2393 }
2394
2395 // Erase the file if it is a symlink so we can overwrite it
2396 struct stat St;
2397 if (lstat(DestFile.c_str(),&St) == 0)
2398 {
2399 if (S_ISLNK(St.st_mode) != 0)
2400 unlink(DestFile.c_str());
2401 }
2402
2403 // Symlink the file
2404 if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
2405 {
2406 ErrorText = "Link to " + DestFile + " failure ";
2407 Status = StatError;
2408 Complete = false;
2409 }
2410 }
2411 }
2412 /*}}}*/
2413 // AcqFile::Failed - Failure handler /*{{{*/
2414 // ---------------------------------------------------------------------
2415 /* Here we try other sources */
2416 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2417 {
2418 ErrorText = LookupTag(Message,"Message");
2419
2420 // This is the retry counter
2421 if (Retries != 0 &&
2422 Cnf->LocalOnly == false &&
2423 StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2424 {
2425 Retries--;
2426 QueueURI(Desc);
2427 return;
2428 }
2429
2430 Item::Failed(Message,Cnf);
2431 }
2432 /*}}}*/
2433 // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/
2434 // ---------------------------------------------------------------------
2435 /* The only header we use is the last-modified header. */
2436 string pkgAcqFile::Custom600Headers() const
2437 {
2438 if (IsIndexFile)
2439 return "\nIndex-File: true";
2440 return "";
2441 }
2442 /*}}}*/