]> git.saurik.com Git - apt.git/blob - apt-pkg/acquire-item.cc
fix Alt-Filename handling of file method
[apt.git] / apt-pkg / acquire-item.cc
1 // -*- mode: cpp; mode: fold -*-
2 // Description /*{{{*/
3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
4 /* ######################################################################
5
6 Acquire Item - Item to acquire
7
8 Each item can download to exactly one file at a time. This means you
9 cannot create an item that fetches two uri's to two files at the same
10 time. The pkgAcqIndex class creates a second class upon instantiation
11 to fetch the other index files because of this.
12
13 ##################################################################### */
14 /*}}}*/
15 // Include Files /*{{{*/
16 #include <config.h>
17
18 #include <apt-pkg/acquire-item.h>
19 #include <apt-pkg/configuration.h>
20 #include <apt-pkg/aptconfiguration.h>
21 #include <apt-pkg/sourcelist.h>
22 #include <apt-pkg/error.h>
23 #include <apt-pkg/strutl.h>
24 #include <apt-pkg/fileutl.h>
25 #include <apt-pkg/tagfile.h>
26 #include <apt-pkg/metaindex.h>
27 #include <apt-pkg/acquire.h>
28 #include <apt-pkg/hashes.h>
29 #include <apt-pkg/indexfile.h>
30 #include <apt-pkg/pkgcache.h>
31 #include <apt-pkg/cacheiterators.h>
32 #include <apt-pkg/pkgrecords.h>
33 #include <apt-pkg/gpgv.h>
34
35 #include <algorithm>
36 #include <stddef.h>
37 #include <stdlib.h>
38 #include <string.h>
39 #include <iostream>
40 #include <vector>
41 #include <sys/stat.h>
42 #include <unistd.h>
43 #include <errno.h>
44 #include <string>
45 #include <stdio.h>
46 #include <ctime>
47 #include <sstream>
48 #include <numeric>
49
50 #include <apti18n.h>
51 /*}}}*/
52
53 using namespace std;
54
55 static void printHashSumComparison(std::string const &URI, HashStringList const &Expected, HashStringList const &Actual) /*{{{*/
56 {
57 if (_config->FindB("Debug::Acquire::HashSumMismatch", false) == false)
58 return;
59 std::cerr << std::endl << URI << ":" << std::endl << " Expected Hash: " << std::endl;
60 for (HashStringList::const_iterator hs = Expected.begin(); hs != Expected.end(); ++hs)
61 std::cerr << "\t- " << hs->toStr() << std::endl;
62 std::cerr << " Actual Hash: " << std::endl;
63 for (HashStringList::const_iterator hs = Actual.begin(); hs != Actual.end(); ++hs)
64 std::cerr << "\t- " << hs->toStr() << std::endl;
65 }
66 /*}}}*/
67 static std::string GetPartialFileName(std::string const &file) /*{{{*/
68 {
69 std::string DestFile = _config->FindDir("Dir::State::lists") + "partial/";
70 DestFile += file;
71 return DestFile;
72 }
73 /*}}}*/
74 static std::string GetPartialFileNameFromURI(std::string const &uri) /*{{{*/
75 {
76 return GetPartialFileName(URItoFileName(uri));
77 }
78 /*}}}*/
79 static std::string GetFinalFileNameFromURI(std::string const &uri) /*{{{*/
80 {
81 return _config->FindDir("Dir::State::lists") + URItoFileName(uri);
82 }
83 /*}}}*/
84 static std::string GetKeepCompressedFileName(std::string file, IndexTarget const &Target)/*{{{*/
85 {
86 if (Target.KeepCompressed == false)
87 return file;
88
89 std::string const KeepCompressedAs = Target.Option(IndexTarget::KEEPCOMPRESSEDAS);
90 if (KeepCompressedAs.empty() == false)
91 {
92 std::string const ext = KeepCompressedAs.substr(0, KeepCompressedAs.find(' '));
93 if (ext != "uncompressed")
94 file.append(".").append(ext);
95 }
96 return file;
97 }
98 /*}}}*/
99 static std::string GetMergeDiffsPatchFileName(std::string const &Final, std::string const &Patch)/*{{{*/
100 {
101 // rred expects the patch as $FinalFile.ed.$patchname.gz
102 return Final + ".ed." + Patch + ".gz";
103 }
104 /*}}}*/
105 static std::string GetDiffsPatchFileName(std::string const &Final) /*{{{*/
106 {
107 // rred expects the patch as $FinalFile.ed
108 return Final + ".ed";
109 }
110 /*}}}*/
111 static std::string GetExistingFilename(std::string const &File) /*{{{*/
112 {
113 if (RealFileExists(File))
114 return File;
115 for (auto const &type : APT::Configuration::getCompressorExtensions())
116 {
117 std::string const Final = File + type;
118 if (RealFileExists(Final))
119 return Final;
120 }
121 return "";
122 }
123 /*}}}*/
124 static std::string GetDiffIndexFileName(std::string const &Name) /*{{{*/
125 {
126 return Name + ".diff/Index";
127 }
128 /*}}}*/
129 static std::string GetDiffIndexURI(IndexTarget const &Target) /*{{{*/
130 {
131 return Target.URI + ".diff/Index";
132 }
133 /*}}}*/
134
135 static bool MessageInsecureRepository(bool const isError, std::string const &msg)/*{{{*/
136 {
137 if (isError)
138 {
139 _error->Error("%s", msg.c_str());
140 _error->Notice("%s", _("Updating from such a repository can't be done securely, and is therefore disabled by default."));
141 }
142 else
143 {
144 _error->Warning("%s", msg.c_str());
145 _error->Notice("%s", _("Data from such a repository can't be authenticated and is therefore potentially dangerous to use."));
146 }
147 _error->Notice("%s", _("See apt-secure(8) manpage for repository creation and user configuration details."));
148 return false;
149 }
150 static bool MessageInsecureRepository(bool const isError, char const * const msg, std::string const &repo)
151 {
152 std::string m;
153 strprintf(m, msg, repo.c_str());
154 return MessageInsecureRepository(isError, m);
155 }
156 /*}}}*/
157 static bool AllowInsecureRepositories(char const * const msg, std::string const &repo,/*{{{*/
158 metaIndex const * const MetaIndexParser, pkgAcqMetaClearSig * const TransactionManager, pkgAcquire::Item * const I)
159 {
160 if(MetaIndexParser->GetTrusted() == metaIndex::TRI_YES)
161 return true;
162
163 if (_config->FindB("Acquire::AllowInsecureRepositories") == true)
164 {
165 MessageInsecureRepository(false, msg, repo);
166 return true;
167 }
168
169 MessageInsecureRepository(true, msg, repo);
170 TransactionManager->AbortTransaction();
171 I->Status = pkgAcquire::Item::StatError;
172 return false;
173 }
174 /*}}}*/
175 static HashStringList GetExpectedHashesFromFor(metaIndex * const Parser, std::string const &MetaKey)/*{{{*/
176 {
177 if (Parser == NULL)
178 return HashStringList();
179 metaIndex::checkSum * const R = Parser->Lookup(MetaKey);
180 if (R == NULL)
181 return HashStringList();
182 return R->Hashes;
183 }
184 /*}}}*/
185
186 // all ::HashesRequired and ::GetExpectedHashes implementations /*{{{*/
187 /* ::GetExpectedHashes is abstract and has to be implemented by all subclasses.
188 It is best to implement it as broadly as possible, while ::HashesRequired defaults
189 to true and should be as restrictive as possible for false cases. Note that if
190 a hash is returned by ::GetExpectedHashes it must match. Only if it doesn't
191 ::HashesRequired is called to evaluate if its okay to have no hashes. */
192 APT_CONST bool pkgAcqTransactionItem::HashesRequired() const
193 {
194 /* signed repositories obviously have a parser and good hashes.
195 unsigned repositories, too, as even if we can't trust them for security,
196 we can at least trust them for integrity of the download itself.
197 Only repositories without a Release file can (obviously) not have
198 hashes – and they are very uncommon and strongly discouraged */
199 return TransactionManager->MetaIndexParser != NULL &&
200 TransactionManager->MetaIndexParser->GetLoadedSuccessfully() == metaIndex::TRI_YES;
201 }
202 HashStringList pkgAcqTransactionItem::GetExpectedHashes() const
203 {
204 return GetExpectedHashesFor(GetMetaKey());
205 }
206
207 APT_CONST bool pkgAcqMetaBase::HashesRequired() const
208 {
209 // Release and co have no hashes 'by design'.
210 return false;
211 }
212 HashStringList pkgAcqMetaBase::GetExpectedHashes() const
213 {
214 return HashStringList();
215 }
216
217 APT_CONST bool pkgAcqIndexDiffs::HashesRequired() const
218 {
219 /* We can't check hashes of rred result as we don't know what the
220 hash of the file will be. We just know the hash of the patch(es),
221 the hash of the file they will apply on and the hash of the resulting
222 file. */
223 if (State == StateFetchDiff)
224 return true;
225 return false;
226 }
227 HashStringList pkgAcqIndexDiffs::GetExpectedHashes() const
228 {
229 if (State == StateFetchDiff)
230 return available_patches[0].download_hashes;
231 return HashStringList();
232 }
233
234 APT_CONST bool pkgAcqIndexMergeDiffs::HashesRequired() const
235 {
236 /* @see #pkgAcqIndexDiffs::HashesRequired, with the difference that
237 we can check the rred result after all patches are applied as
238 we know the expected result rather than potentially apply more patches */
239 if (State == StateFetchDiff)
240 return true;
241 return State == StateApplyDiff;
242 }
243 HashStringList pkgAcqIndexMergeDiffs::GetExpectedHashes() const
244 {
245 if (State == StateFetchDiff)
246 return patch.download_hashes;
247 else if (State == StateApplyDiff)
248 return GetExpectedHashesFor(Target.MetaKey);
249 return HashStringList();
250 }
251
252 APT_CONST bool pkgAcqArchive::HashesRequired() const
253 {
254 return LocalSource == false;
255 }
256 HashStringList pkgAcqArchive::GetExpectedHashes() const
257 {
258 // figured out while parsing the records
259 return ExpectedHashes;
260 }
261
262 APT_CONST bool pkgAcqFile::HashesRequired() const
263 {
264 // supplied as parameter at creation time, so the caller decides
265 return ExpectedHashes.usable();
266 }
267 HashStringList pkgAcqFile::GetExpectedHashes() const
268 {
269 return ExpectedHashes;
270 }
271 /*}}}*/
272 // Acquire::Item::QueueURI and specialisations from child classes /*{{{*/
273 bool pkgAcquire::Item::QueueURI(pkgAcquire::ItemDesc &Item)
274 {
275 Owner->Enqueue(Item);
276 return true;
277 }
278 /* The idea here is that an item isn't queued if it exists on disk and the
279 transition manager was a hit as this means that the files it contains
280 the checksums for can't be updated either (or they are and we are asking
281 for a hashsum mismatch to happen which helps nobody) */
282 bool pkgAcqTransactionItem::QueueURI(pkgAcquire::ItemDesc &Item)
283 {
284 if (TransactionManager->State != TransactionStarted)
285 {
286 if (_config->FindB("Debug::Acquire::Transaction", false))
287 std::clog << "Skip " << Target.URI << " as transaction was already dealt with!" << std::endl;
288 return false;
289 }
290 std::string const FinalFile = GetFinalFilename();
291 if (TransactionManager != NULL && TransactionManager->IMSHit == true &&
292 FileExists(FinalFile) == true)
293 {
294 PartialFile = DestFile = FinalFile;
295 Status = StatDone;
296 return false;
297 }
298 return pkgAcquire::Item::QueueURI(Item);
299 }
300 /* The transition manager InRelease itself (or its older sisters-in-law
301 Release & Release.gpg) is always queued as this allows us to rerun gpgv
302 on it to verify that we aren't stalled with old files */
303 bool pkgAcqMetaBase::QueueURI(pkgAcquire::ItemDesc &Item)
304 {
305 return pkgAcquire::Item::QueueURI(Item);
306 }
307 /* the Diff/Index needs to queue also the up-to-date complete index file
308 to ensure that the list cleaner isn't eating it */
309 bool pkgAcqDiffIndex::QueueURI(pkgAcquire::ItemDesc &Item)
310 {
311 if (pkgAcqTransactionItem::QueueURI(Item) == true)
312 return true;
313 QueueOnIMSHit();
314 return false;
315 }
316 /*}}}*/
317 // Acquire::Item::GetFinalFilename and specialisations for child classes /*{{{*/
318 std::string pkgAcquire::Item::GetFinalFilename() const
319 {
320 // Beware: Desc.URI is modified by redirections
321 return GetFinalFileNameFromURI(Desc.URI);
322 }
323 std::string pkgAcqDiffIndex::GetFinalFilename() const
324 {
325 return GetFinalFileNameFromURI(GetDiffIndexURI(Target));
326 }
327 std::string pkgAcqIndex::GetFinalFilename() const
328 {
329 std::string const FinalFile = GetFinalFileNameFromURI(Target.URI);
330 return GetKeepCompressedFileName(FinalFile, Target);
331 }
332 std::string pkgAcqMetaSig::GetFinalFilename() const
333 {
334 return GetFinalFileNameFromURI(Target.URI);
335 }
336 std::string pkgAcqBaseIndex::GetFinalFilename() const
337 {
338 return GetFinalFileNameFromURI(Target.URI);
339 }
340 std::string pkgAcqMetaBase::GetFinalFilename() const
341 {
342 return GetFinalFileNameFromURI(Target.URI);
343 }
344 std::string pkgAcqArchive::GetFinalFilename() const
345 {
346 return _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
347 }
348 /*}}}*/
349 // pkgAcqTransactionItem::GetMetaKey and specialisations for child classes /*{{{*/
350 std::string pkgAcqTransactionItem::GetMetaKey() const
351 {
352 return Target.MetaKey;
353 }
354 std::string pkgAcqIndex::GetMetaKey() const
355 {
356 if (Stage == STAGE_DECOMPRESS_AND_VERIFY || CurrentCompressionExtension == "uncompressed")
357 return Target.MetaKey;
358 return Target.MetaKey + "." + CurrentCompressionExtension;
359 }
360 std::string pkgAcqDiffIndex::GetMetaKey() const
361 {
362 return GetDiffIndexFileName(Target.MetaKey);
363 }
364 /*}}}*/
365 //pkgAcqTransactionItem::TransactionState and specialisations for child classes /*{{{*/
366 bool pkgAcqTransactionItem::TransactionState(TransactionStates const state)
367 {
368 bool const Debug = _config->FindB("Debug::Acquire::Transaction", false);
369 switch(state)
370 {
371 case TransactionStarted: _error->Fatal("Item %s changed to invalid transaction start state!", Target.URI.c_str()); break;
372 case TransactionAbort:
373 if(Debug == true)
374 std::clog << " Cancel: " << DestFile << std::endl;
375 if (Status == pkgAcquire::Item::StatIdle)
376 {
377 Status = pkgAcquire::Item::StatDone;
378 Dequeue();
379 }
380 break;
381 case TransactionCommit:
382 if(PartialFile.empty() == false)
383 {
384 bool sameFile = (PartialFile == DestFile);
385 // we use symlinks on IMS-Hit to avoid copies
386 if (RealFileExists(DestFile))
387 {
388 struct stat Buf;
389 if (lstat(PartialFile.c_str(), &Buf) != -1)
390 {
391 if (S_ISLNK(Buf.st_mode) && Buf.st_size > 0)
392 {
393 char partial[Buf.st_size + 1];
394 ssize_t const sp = readlink(PartialFile.c_str(), partial, Buf.st_size);
395 if (sp == -1)
396 _error->Errno("pkgAcqTransactionItem::TransactionState-sp", _("Failed to readlink %s"), PartialFile.c_str());
397 else
398 {
399 partial[sp] = '\0';
400 sameFile = (DestFile == partial);
401 }
402 }
403 }
404 else
405 _error->Errno("pkgAcqTransactionItem::TransactionState-stat", _("Failed to stat %s"), PartialFile.c_str());
406 }
407 if (sameFile == false)
408 {
409 // ensure that even without lists-cleanup all compressions are nuked
410 std::string FinalFile = GetFinalFileNameFromURI(Target.URI);
411 if (FileExists(FinalFile))
412 {
413 if(Debug == true)
414 std::clog << "rm " << FinalFile << " # " << DescURI() << std::endl;
415 if (RemoveFile("TransactionStates-Cleanup", FinalFile) == false)
416 return false;
417 }
418 for (auto const &ext: APT::Configuration::getCompressorExtensions())
419 {
420 auto const Final = FinalFile + ext;
421 if (FileExists(Final))
422 {
423 if(Debug == true)
424 std::clog << "rm " << Final << " # " << DescURI() << std::endl;
425 if (RemoveFile("TransactionStates-Cleanup", Final) == false)
426 return false;
427 }
428 }
429 if(Debug == true)
430 std::clog << "mv " << PartialFile << " -> "<< DestFile << " # " << DescURI() << std::endl;
431 if (Rename(PartialFile, DestFile) == false)
432 return false;
433 }
434 else if(Debug == true)
435 std::clog << "keep " << PartialFile << " # " << DescURI() << std::endl;
436
437 } else {
438 if(Debug == true)
439 std::clog << "rm " << DestFile << " # " << DescURI() << std::endl;
440 if (RemoveFile("TransItem::TransactionCommit", DestFile) == false)
441 return false;
442 }
443 break;
444 }
445 return true;
446 }
447 bool pkgAcqMetaBase::TransactionState(TransactionStates const state)
448 {
449 // Do not remove InRelease on IMSHit of Release.gpg [yes, this is very edgecasey]
450 if (TransactionManager->IMSHit == false)
451 return pkgAcqTransactionItem::TransactionState(state);
452 return true;
453 }
454 bool pkgAcqIndex::TransactionState(TransactionStates const state)
455 {
456 if (pkgAcqTransactionItem::TransactionState(state) == false)
457 return false;
458
459 switch (state)
460 {
461 case TransactionStarted: _error->Fatal("AcqIndex %s changed to invalid transaction start state!", Target.URI.c_str()); break;
462 case TransactionAbort:
463 if (Stage == STAGE_DECOMPRESS_AND_VERIFY)
464 {
465 // keep the compressed file, but drop the decompressed
466 EraseFileName.clear();
467 if (PartialFile.empty() == false && flExtension(PartialFile) != CurrentCompressionExtension)
468 RemoveFile("TransactionAbort", PartialFile);
469 }
470 break;
471 case TransactionCommit:
472 if (EraseFileName.empty() == false)
473 RemoveFile("AcqIndex::TransactionCommit", EraseFileName);
474 break;
475 }
476 return true;
477 }
478 bool pkgAcqDiffIndex::TransactionState(TransactionStates const state)
479 {
480 if (pkgAcqTransactionItem::TransactionState(state) == false)
481 return false;
482
483 switch (state)
484 {
485 case TransactionStarted: _error->Fatal("Item %s changed to invalid transaction start state!", Target.URI.c_str()); break;
486 case TransactionCommit:
487 break;
488 case TransactionAbort:
489 std::string const Partial = GetPartialFileNameFromURI(Target.URI);
490 RemoveFile("TransactionAbort", Partial);
491 break;
492 }
493
494 return true;
495 }
496 /*}}}*/
497
498 class APT_HIDDEN NoActionItem : public pkgAcquire::Item /*{{{*/
499 /* The sole purpose of this class is having an item which does nothing to
500 reach its done state to prevent cleanup deleting the mentioned file.
501 Handy in cases in which we know we have the file already, like IMS-Hits. */
502 {
503 IndexTarget const Target;
504 public:
505 virtual std::string DescURI() const APT_OVERRIDE {return Target.URI;};
506 virtual HashStringList GetExpectedHashes() const APT_OVERRIDE {return HashStringList();};
507
508 NoActionItem(pkgAcquire * const Owner, IndexTarget const &Target) :
509 pkgAcquire::Item(Owner), Target(Target)
510 {
511 Status = StatDone;
512 DestFile = GetFinalFileNameFromURI(Target.URI);
513 }
514 NoActionItem(pkgAcquire * const Owner, IndexTarget const &Target, std::string const &FinalFile) :
515 pkgAcquire::Item(Owner), Target(Target)
516 {
517 Status = StatDone;
518 DestFile = FinalFile;
519 }
520 };
521 /*}}}*/
522
523 // Acquire::Item::Item - Constructor /*{{{*/
524 APT_IGNORE_DEPRECATED_PUSH
525 pkgAcquire::Item::Item(pkgAcquire * const owner) :
526 FileSize(0), PartialSize(0), Mode(0), ID(0), Complete(false), Local(false),
527 QueueCounter(0), ExpectedAdditionalItems(0), Owner(owner), d(NULL)
528 {
529 Owner->Add(this);
530 Status = StatIdle;
531 }
532 APT_IGNORE_DEPRECATED_POP
533 /*}}}*/
534 // Acquire::Item::~Item - Destructor /*{{{*/
535 pkgAcquire::Item::~Item()
536 {
537 Owner->Remove(this);
538 }
539 /*}}}*/
540 std::string pkgAcquire::Item::Custom600Headers() const /*{{{*/
541 {
542 return std::string();
543 }
544 /*}}}*/
545 std::string pkgAcquire::Item::ShortDesc() const /*{{{*/
546 {
547 return DescURI();
548 }
549 /*}}}*/
550 APT_CONST void pkgAcquire::Item::Finished() /*{{{*/
551 {
552 }
553 /*}}}*/
554 APT_PURE pkgAcquire * pkgAcquire::Item::GetOwner() const /*{{{*/
555 {
556 return Owner;
557 }
558 /*}}}*/
559 APT_CONST pkgAcquire::ItemDesc &pkgAcquire::Item::GetItemDesc() /*{{{*/
560 {
561 return Desc;
562 }
563 /*}}}*/
564 APT_CONST bool pkgAcquire::Item::IsTrusted() const /*{{{*/
565 {
566 return false;
567 }
568 /*}}}*/
569 // Acquire::Item::Failed - Item failed to download /*{{{*/
570 // ---------------------------------------------------------------------
571 /* We return to an idle state if there are still other queues that could
572 fetch this object */
573 void pkgAcquire::Item::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)
574 {
575 if(ErrorText.empty())
576 ErrorText = LookupTag(Message,"Message");
577 if (QueueCounter <= 1)
578 {
579 /* This indicates that the file is not available right now but might
580 be sometime later. If we do a retry cycle then this should be
581 retried [CDROMs] */
582 if (Cnf != NULL && Cnf->LocalOnly == true &&
583 StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
584 {
585 Status = StatIdle;
586 Dequeue();
587 return;
588 }
589
590 switch (Status)
591 {
592 case StatIdle:
593 case StatFetching:
594 case StatDone:
595 Status = StatError;
596 break;
597 case StatAuthError:
598 case StatError:
599 case StatTransientNetworkError:
600 break;
601 }
602 Complete = false;
603 Dequeue();
604 }
605
606 string const FailReason = LookupTag(Message, "FailReason");
607 if (FailReason == "MaximumSizeExceeded")
608 RenameOnError(MaximumSizeExceeded);
609 else if (Status == StatAuthError)
610 RenameOnError(HashSumMismatch);
611
612 // report mirror failure back to LP if we actually use a mirror
613 if (FailReason.empty() == false)
614 ReportMirrorFailure(FailReason);
615 else
616 ReportMirrorFailure(ErrorText);
617
618 if (QueueCounter > 1)
619 Status = StatIdle;
620 }
621 /*}}}*/
622 // Acquire::Item::Start - Item has begun to download /*{{{*/
623 // ---------------------------------------------------------------------
624 /* Stash status and the file size. Note that setting Complete means
625 sub-phases of the acquire process such as decompresion are operating */
626 void pkgAcquire::Item::Start(string const &/*Message*/, unsigned long long const Size)
627 {
628 Status = StatFetching;
629 ErrorText.clear();
630 if (FileSize == 0 && Complete == false)
631 FileSize = Size;
632 }
633 /*}}}*/
634 // Acquire::Item::VerifyDone - check if Item was downloaded OK /*{{{*/
635 /* Note that hash-verification is 'hardcoded' in acquire-worker and has
636 * already passed if this method is called. */
637 bool pkgAcquire::Item::VerifyDone(std::string const &Message,
638 pkgAcquire::MethodConfig const * const /*Cnf*/)
639 {
640 std::string const FileName = LookupTag(Message,"Filename");
641 if (FileName.empty() == true)
642 {
643 Status = StatError;
644 ErrorText = "Method gave a blank filename";
645 return false;
646 }
647
648 return true;
649 }
650 /*}}}*/
651 // Acquire::Item::Done - Item downloaded OK /*{{{*/
652 void pkgAcquire::Item::Done(string const &/*Message*/, HashStringList const &Hashes,
653 pkgAcquire::MethodConfig const * const /*Cnf*/)
654 {
655 // We just downloaded something..
656 if (FileSize == 0)
657 {
658 unsigned long long const downloadedSize = Hashes.FileSize();
659 if (downloadedSize != 0)
660 {
661 FileSize = downloadedSize;
662 }
663 }
664 Status = StatDone;
665 ErrorText = string();
666 Owner->Dequeue(this);
667 }
668 /*}}}*/
669 // Acquire::Item::Rename - Rename a file /*{{{*/
670 // ---------------------------------------------------------------------
671 /* This helper function is used by a lot of item methods as their final
672 step */
673 bool pkgAcquire::Item::Rename(string const &From,string const &To)
674 {
675 if (From == To || rename(From.c_str(),To.c_str()) == 0)
676 return true;
677
678 std::string S;
679 strprintf(S, _("rename failed, %s (%s -> %s)."), strerror(errno),
680 From.c_str(),To.c_str());
681 Status = StatError;
682 if (ErrorText.empty())
683 ErrorText = S;
684 else
685 ErrorText = ErrorText + ": " + S;
686 return false;
687 }
688 /*}}}*/
689 void pkgAcquire::Item::Dequeue() /*{{{*/
690 {
691 Owner->Dequeue(this);
692 }
693 /*}}}*/
694 bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const error)/*{{{*/
695 {
696 if (RealFileExists(DestFile))
697 Rename(DestFile, DestFile + ".FAILED");
698
699 std::string errtext;
700 switch (error)
701 {
702 case HashSumMismatch:
703 errtext = _("Hash Sum mismatch");
704 Status = StatAuthError;
705 ReportMirrorFailure("HashChecksumFailure");
706 break;
707 case SizeMismatch:
708 errtext = _("Size mismatch");
709 Status = StatAuthError;
710 ReportMirrorFailure("SizeFailure");
711 break;
712 case InvalidFormat:
713 errtext = _("Invalid file format");
714 Status = StatError;
715 // do not report as usually its not the mirrors fault, but Portal/Proxy
716 break;
717 case SignatureError:
718 errtext = _("Signature error");
719 Status = StatError;
720 break;
721 case NotClearsigned:
722 strprintf(errtext, _("Clearsigned file isn't valid, got '%s' (does the network require authentication?)"), "NOSPLIT");
723 Status = StatAuthError;
724 break;
725 case MaximumSizeExceeded:
726 // the method is expected to report a good error for this
727 Status = StatError;
728 break;
729 case PDiffError:
730 // no handling here, done by callers
731 break;
732 }
733 if (ErrorText.empty())
734 ErrorText = errtext;
735 return false;
736 }
737 /*}}}*/
738 void pkgAcquire::Item::SetActiveSubprocess(const std::string &subprocess)/*{{{*/
739 {
740 ActiveSubprocess = subprocess;
741 APT_IGNORE_DEPRECATED(Mode = ActiveSubprocess.c_str();)
742 }
743 /*}}}*/
744 // Acquire::Item::ReportMirrorFailure /*{{{*/
745 void pkgAcquire::Item::ReportMirrorFailure(string const &FailCode)
746 {
747 // we only act if a mirror was used at all
748 if(UsedMirror.empty())
749 return;
750 #if 0
751 std::cerr << "\nReportMirrorFailure: "
752 << UsedMirror
753 << " Uri: " << DescURI()
754 << " FailCode: "
755 << FailCode << std::endl;
756 #endif
757 string report = _config->Find("Methods::Mirror::ProblemReporting",
758 "/usr/lib/apt/apt-report-mirror-failure");
759 if(!FileExists(report))
760 return;
761
762 std::vector<char const*> Args;
763 Args.push_back(report.c_str());
764 Args.push_back(UsedMirror.c_str());
765 Args.push_back(DescURI().c_str());
766 Args.push_back(FailCode.c_str());
767 Args.push_back(NULL);
768
769 pid_t pid = ExecFork();
770 if(pid < 0)
771 {
772 _error->Error("ReportMirrorFailure Fork failed");
773 return;
774 }
775 else if(pid == 0)
776 {
777 execvp(Args[0], (char**)Args.data());
778 std::cerr << "Could not exec " << Args[0] << std::endl;
779 _exit(100);
780 }
781 if(!ExecWait(pid, "report-mirror-failure"))
782 {
783 _error->Warning("Couldn't report problem to '%s'",
784 _config->Find("Methods::Mirror::ProblemReporting").c_str());
785 }
786 }
787 /*}}}*/
788 std::string pkgAcquire::Item::HashSum() const /*{{{*/
789 {
790 HashStringList const hashes = GetExpectedHashes();
791 HashString const * const hs = hashes.find(NULL);
792 return hs != NULL ? hs->toStr() : "";
793 }
794 /*}}}*/
795
796 pkgAcqTransactionItem::pkgAcqTransactionItem(pkgAcquire * const Owner, /*{{{*/
797 pkgAcqMetaClearSig * const transactionManager, IndexTarget const &target) :
798 pkgAcquire::Item(Owner), d(NULL), Target(target), TransactionManager(transactionManager)
799 {
800 if (TransactionManager != this)
801 TransactionManager->Add(this);
802 }
803 /*}}}*/
804 pkgAcqTransactionItem::~pkgAcqTransactionItem() /*{{{*/
805 {
806 }
807 /*}}}*/
808 HashStringList pkgAcqTransactionItem::GetExpectedHashesFor(std::string const &MetaKey) const /*{{{*/
809 {
810 return GetExpectedHashesFromFor(TransactionManager->MetaIndexParser, MetaKey);
811 }
812 /*}}}*/
813
814 static void LoadLastMetaIndexParser(pkgAcqMetaClearSig * const TransactionManager, std::string const &FinalRelease, std::string const &FinalInRelease)/*{{{*/
815 {
816 if (TransactionManager->IMSHit == true)
817 return;
818 if (RealFileExists(FinalInRelease) || RealFileExists(FinalRelease))
819 {
820 TransactionManager->LastMetaIndexParser = TransactionManager->MetaIndexParser->UnloadedClone();
821 if (TransactionManager->LastMetaIndexParser != NULL)
822 {
823 _error->PushToStack();
824 if (RealFileExists(FinalInRelease))
825 TransactionManager->LastMetaIndexParser->Load(FinalInRelease, NULL);
826 else
827 TransactionManager->LastMetaIndexParser->Load(FinalRelease, NULL);
828 // its unlikely to happen, but if what we have is bad ignore it
829 if (_error->PendingError())
830 {
831 delete TransactionManager->LastMetaIndexParser;
832 TransactionManager->LastMetaIndexParser = NULL;
833 }
834 _error->RevertToStack();
835 }
836 }
837 }
838 /*}}}*/
839
840 // AcqMetaBase - Constructor /*{{{*/
841 pkgAcqMetaBase::pkgAcqMetaBase(pkgAcquire * const Owner,
842 pkgAcqMetaClearSig * const TransactionManager,
843 std::vector<IndexTarget> const &IndexTargets,
844 IndexTarget const &DataTarget)
845 : pkgAcqTransactionItem(Owner, TransactionManager, DataTarget), d(NULL),
846 IndexTargets(IndexTargets),
847 AuthPass(false), IMSHit(false), State(TransactionStarted)
848 {
849 }
850 /*}}}*/
851 // AcqMetaBase::Add - Add a item to the current Transaction /*{{{*/
852 void pkgAcqMetaBase::Add(pkgAcqTransactionItem * const I)
853 {
854 Transaction.push_back(I);
855 }
856 /*}}}*/
857 // AcqMetaBase::AbortTransaction - Abort the current Transaction /*{{{*/
858 void pkgAcqMetaBase::AbortTransaction()
859 {
860 if(_config->FindB("Debug::Acquire::Transaction", false) == true)
861 std::clog << "AbortTransaction: " << TransactionManager << std::endl;
862
863 switch (TransactionManager->State)
864 {
865 case TransactionStarted: break;
866 case TransactionAbort: _error->Fatal("Transaction %s was already aborted and is aborted again", TransactionManager->Target.URI.c_str()); return;
867 case TransactionCommit: _error->Fatal("Transaction %s was already aborted and is now commited", TransactionManager->Target.URI.c_str()); return;
868 }
869 TransactionManager->State = TransactionAbort;
870
871 // ensure the toplevel is in error state too
872 for (std::vector<pkgAcqTransactionItem*>::iterator I = Transaction.begin();
873 I != Transaction.end(); ++I)
874 {
875 if ((*I)->Status != pkgAcquire::Item::StatFetching)
876 Owner->Dequeue(*I);
877 (*I)->TransactionState(TransactionAbort);
878 }
879 Transaction.clear();
880 }
881 /*}}}*/
882 // AcqMetaBase::TransactionHasError - Check for errors in Transaction /*{{{*/
883 APT_PURE bool pkgAcqMetaBase::TransactionHasError() const
884 {
885 for (std::vector<pkgAcqTransactionItem*>::const_iterator I = Transaction.begin();
886 I != Transaction.end(); ++I)
887 {
888 switch((*I)->Status) {
889 case StatDone: break;
890 case StatIdle: break;
891 case StatAuthError: return true;
892 case StatError: return true;
893 case StatTransientNetworkError: return true;
894 case StatFetching: break;
895 }
896 }
897 return false;
898 }
899 /*}}}*/
900 // AcqMetaBase::CommitTransaction - Commit a transaction /*{{{*/
901 void pkgAcqMetaBase::CommitTransaction()
902 {
903 if(_config->FindB("Debug::Acquire::Transaction", false) == true)
904 std::clog << "CommitTransaction: " << this << std::endl;
905
906 switch (TransactionManager->State)
907 {
908 case TransactionStarted: break;
909 case TransactionAbort: _error->Fatal("Transaction %s was already commited and is now aborted", TransactionManager->Target.URI.c_str()); return;
910 case TransactionCommit: _error->Fatal("Transaction %s was already commited and is again commited", TransactionManager->Target.URI.c_str()); return;
911 }
912 TransactionManager->State = TransactionCommit;
913
914 // move new files into place *and* remove files that are not
915 // part of the transaction but are still on disk
916 for (std::vector<pkgAcqTransactionItem*>::iterator I = Transaction.begin();
917 I != Transaction.end(); ++I)
918 {
919 (*I)->TransactionState(TransactionCommit);
920 }
921 Transaction.clear();
922 }
923 /*}}}*/
924 // AcqMetaBase::TransactionStageCopy - Stage a file for copying /*{{{*/
925 void pkgAcqMetaBase::TransactionStageCopy(pkgAcqTransactionItem * const I,
926 const std::string &From,
927 const std::string &To)
928 {
929 I->PartialFile = From;
930 I->DestFile = To;
931 }
932 /*}}}*/
933 // AcqMetaBase::TransactionStageRemoval - Stage a file for removal /*{{{*/
934 void pkgAcqMetaBase::TransactionStageRemoval(pkgAcqTransactionItem * const I,
935 const std::string &FinalFile)
936 {
937 I->PartialFile = "";
938 I->DestFile = FinalFile;
939 }
940 /*}}}*/
941 // AcqMetaBase::GenerateAuthWarning - Check gpg authentication error /*{{{*/
942 bool pkgAcqMetaBase::CheckStopAuthentication(pkgAcquire::Item * const I, const std::string &Message)
943 {
944 // FIXME: this entire function can do now that we disallow going to
945 // a unauthenticated state and can cleanly rollback
946
947 string const Final = I->GetFinalFilename();
948 if(FileExists(Final))
949 {
950 I->Status = StatTransientNetworkError;
951 _error->Warning(_("An error occurred during the signature "
952 "verification. The repository is not updated "
953 "and the previous index files will be used. "
954 "GPG error: %s: %s"),
955 Desc.Description.c_str(),
956 LookupTag(Message,"Message").c_str());
957 RunScripts("APT::Update::Auth-Failure");
958 return true;
959 } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) {
960 /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */
961 _error->Error(_("GPG error: %s: %s"),
962 Desc.Description.c_str(),
963 LookupTag(Message,"Message").c_str());
964 I->Status = StatAuthError;
965 return true;
966 } else {
967 _error->Warning(_("GPG error: %s: %s"),
968 Desc.Description.c_str(),
969 LookupTag(Message,"Message").c_str());
970 }
971 // gpgv method failed
972 ReportMirrorFailure("GPGFailure");
973 return false;
974 }
975 /*}}}*/
976 // AcqMetaBase::Custom600Headers - Get header for AcqMetaBase /*{{{*/
977 // ---------------------------------------------------------------------
978 string pkgAcqMetaBase::Custom600Headers() const
979 {
980 std::string Header = "\nIndex-File: true";
981 std::string MaximumSize;
982 strprintf(MaximumSize, "\nMaximum-Size: %i",
983 _config->FindI("Acquire::MaxReleaseFileSize", 10*1000*1000));
984 Header += MaximumSize;
985
986 string const FinalFile = GetFinalFilename();
987 struct stat Buf;
988 if (stat(FinalFile.c_str(),&Buf) == 0)
989 Header += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
990
991 return Header;
992 }
993 /*}}}*/
994 // AcqMetaBase::QueueForSignatureVerify /*{{{*/
995 void pkgAcqMetaBase::QueueForSignatureVerify(pkgAcqTransactionItem * const I, std::string const &File, std::string const &Signature)
996 {
997 AuthPass = true;
998 I->Desc.URI = "gpgv:" + Signature;
999 I->DestFile = File;
1000 QueueURI(I->Desc);
1001 I->SetActiveSubprocess("gpgv");
1002 }
1003 /*}}}*/
1004 // AcqMetaBase::CheckDownloadDone /*{{{*/
1005 bool pkgAcqMetaBase::CheckDownloadDone(pkgAcqTransactionItem * const I, const std::string &Message, HashStringList const &Hashes) const
1006 {
1007 // We have just finished downloading a Release file (it is not
1008 // verified yet)
1009
1010 std::string const FileName = LookupTag(Message,"Filename");
1011 if (FileName != I->DestFile && RealFileExists(I->DestFile) == false)
1012 {
1013 I->Local = true;
1014 I->Desc.URI = "copy:" + FileName;
1015 I->QueueURI(I->Desc);
1016 return false;
1017 }
1018
1019 // make sure to verify against the right file on I-M-S hit
1020 bool IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"), false);
1021 if (IMSHit == false && Hashes.usable())
1022 {
1023 // detect IMS-Hits servers haven't detected by Hash comparison
1024 std::string const FinalFile = I->GetFinalFilename();
1025 if (RealFileExists(FinalFile) && Hashes.VerifyFile(FinalFile) == true)
1026 {
1027 IMSHit = true;
1028 RemoveFile("CheckDownloadDone", I->DestFile);
1029 }
1030 }
1031
1032 if(IMSHit == true)
1033 {
1034 // for simplicity, the transaction manager is always InRelease
1035 // even if it doesn't exist.
1036 if (TransactionManager != NULL)
1037 TransactionManager->IMSHit = true;
1038 I->PartialFile = I->DestFile = I->GetFinalFilename();
1039 }
1040
1041 // set Item to complete as the remaining work is all local (verify etc)
1042 I->Complete = true;
1043
1044 return true;
1045 }
1046 /*}}}*/
1047 bool pkgAcqMetaBase::CheckAuthDone(string const &Message) /*{{{*/
1048 {
1049 // At this point, the gpgv method has succeeded, so there is a
1050 // valid signature from a key in the trusted keyring. We
1051 // perform additional verification of its contents, and use them
1052 // to verify the indexes we are about to download
1053
1054 if (TransactionManager->IMSHit == false)
1055 {
1056 // open the last (In)Release if we have it
1057 std::string const FinalFile = GetFinalFilename();
1058 std::string FinalRelease;
1059 std::string FinalInRelease;
1060 if (APT::String::Endswith(FinalFile, "InRelease"))
1061 {
1062 FinalInRelease = FinalFile;
1063 FinalRelease = FinalFile.substr(0, FinalFile.length() - strlen("InRelease")) + "Release";
1064 }
1065 else
1066 {
1067 FinalInRelease = FinalFile.substr(0, FinalFile.length() - strlen("Release")) + "InRelease";
1068 FinalRelease = FinalFile;
1069 }
1070 LoadLastMetaIndexParser(TransactionManager, FinalRelease, FinalInRelease);
1071 }
1072
1073 if (TransactionManager->MetaIndexParser->Load(DestFile, &ErrorText) == false)
1074 {
1075 Status = StatAuthError;
1076 return false;
1077 }
1078
1079 if (!VerifyVendor(Message))
1080 {
1081 Status = StatAuthError;
1082 return false;
1083 }
1084
1085 if (_config->FindB("Debug::pkgAcquire::Auth", false))
1086 std::cerr << "Signature verification succeeded: "
1087 << DestFile << std::endl;
1088
1089 // Download further indexes with verification
1090 QueueIndexes(true);
1091
1092 return true;
1093 }
1094 /*}}}*/
1095 void pkgAcqMetaBase::QueueIndexes(bool const verify) /*{{{*/
1096 {
1097 // at this point the real Items are loaded in the fetcher
1098 ExpectedAdditionalItems = 0;
1099
1100 bool metaBaseSupportsByHash = false;
1101 if (TransactionManager != NULL && TransactionManager->MetaIndexParser != NULL)
1102 metaBaseSupportsByHash = TransactionManager->MetaIndexParser->GetSupportsAcquireByHash();
1103
1104 for (std::vector <IndexTarget>::iterator Target = IndexTargets.begin();
1105 Target != IndexTargets.end();
1106 ++Target)
1107 {
1108 // all is an implementation detail. Users shouldn't use this as arch
1109 // We need this support trickery here as e.g. Debian has binary-all files already,
1110 // but arch:all packages are still in the arch:any files, so we would waste precious
1111 // download time, bandwidth and diskspace for nothing, BUT Debian doesn't feature all
1112 // in the set of supported architectures, so we can filter based on this property rather
1113 // than invent an entirely new flag we would need to carry for all of eternity.
1114 if (Target->Option(IndexTarget::ARCHITECTURE) == "all")
1115 {
1116 if (TransactionManager->MetaIndexParser->IsArchitectureSupported("all") == false)
1117 continue;
1118 if (TransactionManager->MetaIndexParser->IsArchitectureAllSupportedFor(*Target) == false)
1119 continue;
1120 }
1121
1122 bool trypdiff = Target->OptionBool(IndexTarget::PDIFFS);
1123 if (verify == true)
1124 {
1125 if (TransactionManager->MetaIndexParser->Exists(Target->MetaKey) == false)
1126 {
1127 // optional targets that we do not have in the Release file are skipped
1128 if (Target->IsOptional)
1129 continue;
1130
1131 std::string const &arch = Target->Option(IndexTarget::ARCHITECTURE);
1132 if (arch.empty() == false)
1133 {
1134 if (TransactionManager->MetaIndexParser->IsArchitectureSupported(arch) == false)
1135 {
1136 _error->Notice(_("Skipping acquire of configured file '%s' as repository '%s' doesn't support architecture '%s'"),
1137 Target->MetaKey.c_str(), TransactionManager->Target.Description.c_str(), arch.c_str());
1138 continue;
1139 }
1140 // if the architecture is officially supported but currently no packages for it available,
1141 // ignore silently as this is pretty much the same as just shipping an empty file.
1142 // if we don't know which architectures are supported, we do NOT ignore it to notify user about this
1143 if (TransactionManager->MetaIndexParser->IsArchitectureSupported("*undefined*") == false)
1144 continue;
1145 }
1146
1147 Status = StatAuthError;
1148 strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), Target->MetaKey.c_str());
1149 return;
1150 }
1151 else
1152 {
1153 auto const hashes = GetExpectedHashesFor(Target->MetaKey);
1154 if (hashes.usable() == false && hashes.empty() == false)
1155 {
1156 _error->Warning(_("Skipping acquire of configured file '%s' as repository '%s' provides only weak security information for it"),
1157 Target->MetaKey.c_str(), TransactionManager->Target.Description.c_str());
1158 continue;
1159 }
1160 }
1161
1162 // autoselect the compression method
1163 std::vector<std::string> types = VectorizeString(Target->Option(IndexTarget::COMPRESSIONTYPES), ' ');
1164 types.erase(std::remove_if(types.begin(), types.end(), [&](std::string const &t) {
1165 if (t == "uncompressed")
1166 return TransactionManager->MetaIndexParser->Exists(Target->MetaKey) == false;
1167 std::string const MetaKey = Target->MetaKey + "." + t;
1168 return TransactionManager->MetaIndexParser->Exists(MetaKey) == false;
1169 }), types.end());
1170 if (types.empty() == false)
1171 {
1172 std::ostringstream os;
1173 // add the special compressiontype byhash first if supported
1174 std::string const useByHashConf = Target->Option(IndexTarget::BY_HASH);
1175 bool useByHash = false;
1176 if(useByHashConf == "force")
1177 useByHash = true;
1178 else
1179 useByHash = StringToBool(useByHashConf) == true && metaBaseSupportsByHash;
1180 if (useByHash == true)
1181 os << "by-hash ";
1182 std::copy(types.begin(), types.end()-1, std::ostream_iterator<std::string>(os, " "));
1183 os << *types.rbegin();
1184 Target->Options["COMPRESSIONTYPES"] = os.str();
1185 }
1186 else
1187 Target->Options["COMPRESSIONTYPES"].clear();
1188
1189 std::string filename = GetExistingFilename(GetFinalFileNameFromURI(Target->URI));
1190 if (filename.empty() == false)
1191 {
1192 // if the Release file is a hit and we have an index it must be the current one
1193 if (TransactionManager->IMSHit == true)
1194 ;
1195 else if (TransactionManager->LastMetaIndexParser != NULL)
1196 {
1197 // see if the file changed since the last Release file
1198 // we use the uncompressed files as we might compress differently compared to the server,
1199 // so the hashes might not match, even if they contain the same data.
1200 HashStringList const newFile = GetExpectedHashesFromFor(TransactionManager->MetaIndexParser, Target->MetaKey);
1201 HashStringList const oldFile = GetExpectedHashesFromFor(TransactionManager->LastMetaIndexParser, Target->MetaKey);
1202 if (newFile != oldFile)
1203 filename.clear();
1204 }
1205 else
1206 filename.clear();
1207 }
1208 else
1209 trypdiff = false; // no file to patch
1210
1211 if (filename.empty() == false)
1212 {
1213 new NoActionItem(Owner, *Target, filename);
1214 std::string const idxfilename = GetFinalFileNameFromURI(GetDiffIndexURI(*Target));
1215 if (FileExists(idxfilename))
1216 new NoActionItem(Owner, *Target, idxfilename);
1217 continue;
1218 }
1219
1220 // check if we have patches available
1221 trypdiff &= TransactionManager->MetaIndexParser->Exists(GetDiffIndexFileName(Target->MetaKey));
1222 }
1223 else
1224 {
1225 // if we have no file to patch, no point in trying
1226 trypdiff &= (GetExistingFilename(GetFinalFileNameFromURI(Target->URI)).empty() == false);
1227 }
1228
1229 // no point in patching from local sources
1230 if (trypdiff)
1231 {
1232 std::string const proto = Target->URI.substr(0, strlen("file:/"));
1233 if (proto == "file:/" || proto == "copy:/" || proto == "cdrom:")
1234 trypdiff = false;
1235 }
1236
1237 // Queue the Index file (Packages, Sources, Translation-$foo, …)
1238 if (trypdiff)
1239 new pkgAcqDiffIndex(Owner, TransactionManager, *Target);
1240 else
1241 new pkgAcqIndex(Owner, TransactionManager, *Target);
1242 }
1243 }
1244 /*}}}*/
1245 bool pkgAcqMetaBase::VerifyVendor(string const &Message) /*{{{*/
1246 {
1247 string::size_type pos;
1248
1249 // check for missing sigs (that where not fatal because otherwise we had
1250 // bombed earlier)
1251 string missingkeys;
1252 string msg = _("There is no public key available for the "
1253 "following key IDs:\n");
1254 pos = Message.find("NO_PUBKEY ");
1255 if (pos != std::string::npos)
1256 {
1257 string::size_type start = pos+strlen("NO_PUBKEY ");
1258 string Fingerprint = Message.substr(start, Message.find("\n")-start);
1259 missingkeys += (Fingerprint);
1260 }
1261 if(!missingkeys.empty())
1262 _error->Warning("%s", (msg + missingkeys).c_str());
1263
1264 string Transformed = TransactionManager->MetaIndexParser->GetExpectedDist();
1265
1266 if (Transformed == "../project/experimental")
1267 {
1268 Transformed = "experimental";
1269 }
1270
1271 pos = Transformed.rfind('/');
1272 if (pos != string::npos)
1273 {
1274 Transformed = Transformed.substr(0, pos);
1275 }
1276
1277 if (Transformed == ".")
1278 {
1279 Transformed = "";
1280 }
1281
1282 if (TransactionManager->MetaIndexParser->GetValidUntil() > 0)
1283 {
1284 time_t const invalid_since = time(NULL) - TransactionManager->MetaIndexParser->GetValidUntil();
1285 if (invalid_since > 0)
1286 {
1287 std::string errmsg;
1288 strprintf(errmsg,
1289 // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1290 // the time since then the file is invalid - formatted in the same way as in
1291 // the download progress display (e.g. 7d 3h 42min 1s)
1292 _("Release file for %s is expired (invalid since %s). "
1293 "Updates for this repository will not be applied."),
1294 Target.URI.c_str(), TimeToStr(invalid_since).c_str());
1295 if (ErrorText.empty())
1296 ErrorText = errmsg;
1297 return _error->Error("%s", errmsg.c_str());
1298 }
1299 }
1300
1301 /* Did we get a file older than what we have? This is a last minute IMS hit and doubles
1302 as a prevention of downgrading us to older (still valid) files */
1303 if (TransactionManager->IMSHit == false && TransactionManager->LastMetaIndexParser != NULL &&
1304 TransactionManager->LastMetaIndexParser->GetDate() > TransactionManager->MetaIndexParser->GetDate())
1305 {
1306 TransactionManager->IMSHit = true;
1307 RemoveFile("VerifyVendor", DestFile);
1308 PartialFile = DestFile = GetFinalFilename();
1309 // load the 'old' file in the 'new' one instead of flipping pointers as
1310 // the new one isn't owned by us, while the old one is so cleanup would be confused.
1311 TransactionManager->MetaIndexParser->swapLoad(TransactionManager->LastMetaIndexParser);
1312 delete TransactionManager->LastMetaIndexParser;
1313 TransactionManager->LastMetaIndexParser = NULL;
1314 }
1315
1316 if (_config->FindB("Debug::pkgAcquire::Auth", false))
1317 {
1318 std::cerr << "Got Codename: " << TransactionManager->MetaIndexParser->GetCodename() << std::endl;
1319 std::cerr << "Expecting Dist: " << TransactionManager->MetaIndexParser->GetExpectedDist() << std::endl;
1320 std::cerr << "Transformed Dist: " << Transformed << std::endl;
1321 }
1322
1323 if (TransactionManager->MetaIndexParser->CheckDist(Transformed) == false)
1324 {
1325 // This might become fatal one day
1326 // Status = StatAuthError;
1327 // ErrorText = "Conflicting distribution; expected "
1328 // + MetaIndexParser->GetExpectedDist() + " but got "
1329 // + MetaIndexParser->GetCodename();
1330 // return false;
1331 if (!Transformed.empty())
1332 {
1333 _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1334 Desc.Description.c_str(),
1335 Transformed.c_str(),
1336 TransactionManager->MetaIndexParser->GetCodename().c_str());
1337 }
1338 }
1339
1340 return true;
1341 }
1342 /*}}}*/
1343 pkgAcqMetaBase::~pkgAcqMetaBase()
1344 {
1345 }
1346
1347 pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire * const Owner, /*{{{*/
1348 IndexTarget const &ClearsignedTarget,
1349 IndexTarget const &DetachedDataTarget, IndexTarget const &DetachedSigTarget,
1350 std::vector<IndexTarget> const &IndexTargets,
1351 metaIndex * const MetaIndexParser) :
1352 pkgAcqMetaIndex(Owner, this, ClearsignedTarget, DetachedSigTarget, IndexTargets),
1353 d(NULL), ClearsignedTarget(ClearsignedTarget),
1354 DetachedDataTarget(DetachedDataTarget),
1355 MetaIndexParser(MetaIndexParser), LastMetaIndexParser(NULL)
1356 {
1357 // index targets + (worst case:) Release/Release.gpg
1358 ExpectedAdditionalItems = IndexTargets.size() + 2;
1359 TransactionManager->Add(this);
1360 }
1361 /*}}}*/
1362 pkgAcqMetaClearSig::~pkgAcqMetaClearSig() /*{{{*/
1363 {
1364 if (LastMetaIndexParser != NULL)
1365 delete LastMetaIndexParser;
1366 }
1367 /*}}}*/
1368 // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/
1369 string pkgAcqMetaClearSig::Custom600Headers() const
1370 {
1371 string Header = pkgAcqMetaBase::Custom600Headers();
1372 Header += "\nFail-Ignore: true";
1373 std::string const key = TransactionManager->MetaIndexParser->GetSignedBy();
1374 if (key.empty() == false)
1375 Header += "\nSigned-By: " + key;
1376
1377 return Header;
1378 }
1379 /*}}}*/
1380 void pkgAcqMetaClearSig::Finished() /*{{{*/
1381 {
1382 if(_config->FindB("Debug::Acquire::Transaction", false) == true)
1383 std::clog << "Finished: " << DestFile <<std::endl;
1384 if(TransactionManager != NULL && TransactionManager->State == TransactionStarted &&
1385 TransactionManager->TransactionHasError() == false)
1386 TransactionManager->CommitTransaction();
1387 }
1388 /*}}}*/
1389 bool pkgAcqMetaClearSig::VerifyDone(std::string const &Message, /*{{{*/
1390 pkgAcquire::MethodConfig const * const Cnf)
1391 {
1392 Item::VerifyDone(Message, Cnf);
1393
1394 if (FileExists(DestFile) && !StartsWithGPGClearTextSignature(DestFile))
1395 return RenameOnError(NotClearsigned);
1396
1397 return true;
1398 }
1399 /*}}}*/
1400 // pkgAcqMetaClearSig::Done - We got a file /*{{{*/
1401 void pkgAcqMetaClearSig::Done(std::string const &Message,
1402 HashStringList const &Hashes,
1403 pkgAcquire::MethodConfig const * const Cnf)
1404 {
1405 Item::Done(Message, Hashes, Cnf);
1406
1407 if(AuthPass == false)
1408 {
1409 if(CheckDownloadDone(this, Message, Hashes) == true)
1410 QueueForSignatureVerify(this, DestFile, DestFile);
1411 return;
1412 }
1413 else if(CheckAuthDone(Message) == true)
1414 {
1415 if (TransactionManager->IMSHit == false)
1416 TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename());
1417 else if (RealFileExists(GetFinalFilename()) == false)
1418 {
1419 // We got an InRelease file IMSHit, but we haven't one, which means
1420 // we had a valid Release/Release.gpg combo stepping in, which we have
1421 // to 'acquire' now to ensure list cleanup isn't removing them
1422 new NoActionItem(Owner, DetachedDataTarget);
1423 new NoActionItem(Owner, DetachedSigTarget);
1424 }
1425 }
1426 }
1427 /*}}}*/
1428 void pkgAcqMetaClearSig::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf) /*{{{*/
1429 {
1430 Item::Failed(Message, Cnf);
1431
1432 // we failed, we will not get additional items from this method
1433 ExpectedAdditionalItems = 0;
1434
1435 if (AuthPass == false)
1436 {
1437 if (Status == StatAuthError || Status == StatTransientNetworkError)
1438 {
1439 // if we expected a ClearTextSignature (InRelease) but got a network
1440 // error or got a file, but it wasn't valid, we end up here (see VerifyDone).
1441 // As these is usually called by web-portals we do not try Release/Release.gpg
1442 // as this is gonna fail anyway and instead abort our try (LP#346386)
1443 TransactionManager->AbortTransaction();
1444 return;
1445 }
1446
1447 // Queue the 'old' InRelease file for removal if we try Release.gpg
1448 // as otherwise the file will stay around and gives a false-auth
1449 // impression (CVE-2012-0214)
1450 TransactionManager->TransactionStageRemoval(this, GetFinalFilename());
1451 Status = StatDone;
1452
1453 new pkgAcqMetaIndex(Owner, TransactionManager, DetachedDataTarget, DetachedSigTarget, IndexTargets);
1454 }
1455 else
1456 {
1457 if(CheckStopAuthentication(this, Message))
1458 return;
1459
1460 // No Release file was present, or verification failed, so fall
1461 // back to queueing Packages files without verification
1462 // only allow going further if the user explicitly wants it
1463 if(AllowInsecureRepositories(_("The repository '%s' is not signed."), ClearsignedTarget.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true)
1464 {
1465 Status = StatDone;
1466
1467 /* InRelease files become Release files, otherwise
1468 * they would be considered as trusted later on */
1469 string const FinalRelease = GetFinalFileNameFromURI(DetachedDataTarget.URI);
1470 string const PartialRelease = GetPartialFileNameFromURI(DetachedDataTarget.URI);
1471 string const FinalReleasegpg = GetFinalFileNameFromURI(DetachedSigTarget.URI);
1472 string const FinalInRelease = GetFinalFilename();
1473 Rename(DestFile, PartialRelease);
1474 TransactionManager->TransactionStageCopy(this, PartialRelease, FinalRelease);
1475 LoadLastMetaIndexParser(TransactionManager, FinalRelease, FinalInRelease);
1476
1477 // we parse the indexes here because at this point the user wanted
1478 // a repository that may potentially harm him
1479 if (TransactionManager->MetaIndexParser->Load(PartialRelease, &ErrorText) == false || VerifyVendor(Message) == false)
1480 /* expired Release files are still a problem you need extra force for */;
1481 else
1482 QueueIndexes(true);
1483 }
1484 }
1485 }
1486 /*}}}*/
1487
1488 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire * const Owner, /*{{{*/
1489 pkgAcqMetaClearSig * const TransactionManager,
1490 IndexTarget const &DataTarget,
1491 IndexTarget const &DetachedSigTarget,
1492 vector<IndexTarget> const &IndexTargets) :
1493 pkgAcqMetaBase(Owner, TransactionManager, IndexTargets, DataTarget), d(NULL),
1494 DetachedSigTarget(DetachedSigTarget)
1495 {
1496 if(_config->FindB("Debug::Acquire::Transaction", false) == true)
1497 std::clog << "New pkgAcqMetaIndex with TransactionManager "
1498 << this->TransactionManager << std::endl;
1499
1500 DestFile = GetPartialFileNameFromURI(DataTarget.URI);
1501
1502 // Create the item
1503 Desc.Description = DataTarget.Description;
1504 Desc.Owner = this;
1505 Desc.ShortDesc = DataTarget.ShortDesc;
1506 Desc.URI = DataTarget.URI;
1507
1508 // we expect more item
1509 ExpectedAdditionalItems = IndexTargets.size();
1510 QueueURI(Desc);
1511 }
1512 /*}}}*/
1513 void pkgAcqMetaIndex::Done(string const &Message, /*{{{*/
1514 HashStringList const &Hashes,
1515 pkgAcquire::MethodConfig const * const Cfg)
1516 {
1517 Item::Done(Message,Hashes,Cfg);
1518
1519 if(CheckDownloadDone(this, Message, Hashes))
1520 {
1521 // we have a Release file, now download the Signature, all further
1522 // verify/queue for additional downloads will be done in the
1523 // pkgAcqMetaSig::Done() code
1524 new pkgAcqMetaSig(Owner, TransactionManager, DetachedSigTarget, this);
1525 }
1526 }
1527 /*}}}*/
1528 // pkgAcqMetaIndex::Failed - no Release file present /*{{{*/
1529 void pkgAcqMetaIndex::Failed(string const &Message,
1530 pkgAcquire::MethodConfig const * const Cnf)
1531 {
1532 pkgAcquire::Item::Failed(Message, Cnf);
1533 Status = StatDone;
1534
1535 // No Release file was present so fall
1536 // back to queueing Packages files without verification
1537 // only allow going further if the user explicitly wants it
1538 if(AllowInsecureRepositories(_("The repository '%s' does not have a Release file."), Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true)
1539 {
1540 // ensure old Release files are removed
1541 TransactionManager->TransactionStageRemoval(this, GetFinalFilename());
1542
1543 // queue without any kind of hashsum support
1544 QueueIndexes(false);
1545 }
1546 }
1547 /*}}}*/
1548 std::string pkgAcqMetaIndex::DescURI() const /*{{{*/
1549 {
1550 return Target.URI;
1551 }
1552 /*}}}*/
1553 pkgAcqMetaIndex::~pkgAcqMetaIndex() {}
1554
1555 // AcqMetaSig::AcqMetaSig - Constructor /*{{{*/
1556 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire * const Owner,
1557 pkgAcqMetaClearSig * const TransactionManager,
1558 IndexTarget const &Target,
1559 pkgAcqMetaIndex * const MetaIndex) :
1560 pkgAcqTransactionItem(Owner, TransactionManager, Target), d(NULL), MetaIndex(MetaIndex)
1561 {
1562 DestFile = GetPartialFileNameFromURI(Target.URI);
1563
1564 // remove any partial downloaded sig-file in partial/.
1565 // it may confuse proxies and is too small to warrant a
1566 // partial download anyway
1567 RemoveFile("pkgAcqMetaSig", DestFile);
1568
1569 // set the TransactionManager
1570 if(_config->FindB("Debug::Acquire::Transaction", false) == true)
1571 std::clog << "New pkgAcqMetaSig with TransactionManager "
1572 << TransactionManager << std::endl;
1573
1574 // Create the item
1575 Desc.Description = Target.Description;
1576 Desc.Owner = this;
1577 Desc.ShortDesc = Target.ShortDesc;
1578 Desc.URI = Target.URI;
1579
1580 // If we got a hit for Release, we will get one for Release.gpg too (or obscure errors),
1581 // so we skip the download step and go instantly to verification
1582 if (TransactionManager->IMSHit == true && RealFileExists(GetFinalFilename()))
1583 {
1584 Complete = true;
1585 Status = StatDone;
1586 PartialFile = DestFile = GetFinalFilename();
1587 MetaIndexFileSignature = DestFile;
1588 MetaIndex->QueueForSignatureVerify(this, MetaIndex->DestFile, DestFile);
1589 }
1590 else
1591 QueueURI(Desc);
1592 }
1593 /*}}}*/
1594 pkgAcqMetaSig::~pkgAcqMetaSig() /*{{{*/
1595 {
1596 }
1597 /*}}}*/
1598 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers /*{{{*/
1599 std::string pkgAcqMetaSig::Custom600Headers() const
1600 {
1601 std::string Header = pkgAcqTransactionItem::Custom600Headers();
1602 std::string const key = TransactionManager->MetaIndexParser->GetSignedBy();
1603 if (key.empty() == false)
1604 Header += "\nSigned-By: " + key;
1605 return Header;
1606 }
1607 /*}}}*/
1608 // AcqMetaSig::Done - The signature was downloaded/verified /*{{{*/
1609 void pkgAcqMetaSig::Done(string const &Message, HashStringList const &Hashes,
1610 pkgAcquire::MethodConfig const * const Cfg)
1611 {
1612 if (MetaIndexFileSignature.empty() == false)
1613 {
1614 DestFile = MetaIndexFileSignature;
1615 MetaIndexFileSignature.clear();
1616 }
1617 Item::Done(Message, Hashes, Cfg);
1618
1619 if(MetaIndex->AuthPass == false)
1620 {
1621 if(MetaIndex->CheckDownloadDone(this, Message, Hashes) == true)
1622 {
1623 // destfile will be modified to point to MetaIndexFile for the
1624 // gpgv method, so we need to save it here
1625 MetaIndexFileSignature = DestFile;
1626 MetaIndex->QueueForSignatureVerify(this, MetaIndex->DestFile, DestFile);
1627 }
1628 return;
1629 }
1630 else if(MetaIndex->CheckAuthDone(Message) == true)
1631 {
1632 if (TransactionManager->IMSHit == false)
1633 {
1634 TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename());
1635 TransactionManager->TransactionStageCopy(MetaIndex, MetaIndex->DestFile, MetaIndex->GetFinalFilename());
1636 }
1637 }
1638 }
1639 /*}}}*/
1640 void pkgAcqMetaSig::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
1641 {
1642 Item::Failed(Message,Cnf);
1643
1644 // check if we need to fail at this point
1645 if (MetaIndex->AuthPass == true && MetaIndex->CheckStopAuthentication(this, Message))
1646 return;
1647
1648 string const FinalRelease = MetaIndex->GetFinalFilename();
1649 string const FinalReleasegpg = GetFinalFilename();
1650 string const FinalInRelease = TransactionManager->GetFinalFilename();
1651
1652 if (RealFileExists(FinalReleasegpg) || RealFileExists(FinalInRelease))
1653 {
1654 std::string downgrade_msg;
1655 strprintf(downgrade_msg, _("The repository '%s' is no longer signed."),
1656 MetaIndex->Target.Description.c_str());
1657 if(_config->FindB("Acquire::AllowDowngradeToInsecureRepositories"))
1658 {
1659 // meh, the users wants to take risks (we still mark the packages
1660 // from this repository as unauthenticated)
1661 _error->Warning("%s", downgrade_msg.c_str());
1662 _error->Warning(_("This is normally not allowed, but the option "
1663 "Acquire::AllowDowngradeToInsecureRepositories was "
1664 "given to override it."));
1665 Status = StatDone;
1666 } else {
1667 MessageInsecureRepository(true, downgrade_msg);
1668 if (TransactionManager->IMSHit == false)
1669 Rename(MetaIndex->DestFile, MetaIndex->DestFile + ".FAILED");
1670 Item::Failed("Message: " + downgrade_msg, Cnf);
1671 TransactionManager->AbortTransaction();
1672 return;
1673 }
1674 }
1675
1676 // ensures that a Release.gpg file in the lists/ is removed by the transaction
1677 TransactionManager->TransactionStageRemoval(this, DestFile);
1678
1679 // only allow going further if the user explicitly wants it
1680 if (AllowInsecureRepositories(_("The repository '%s' is not signed."), MetaIndex->Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true)
1681 {
1682 LoadLastMetaIndexParser(TransactionManager, FinalRelease, FinalInRelease);
1683
1684 // we parse the indexes here because at this point the user wanted
1685 // a repository that may potentially harm him
1686 bool const GoodLoad = TransactionManager->MetaIndexParser->Load(MetaIndex->DestFile, &ErrorText);
1687 if (MetaIndex->VerifyVendor(Message) == false)
1688 /* expired Release files are still a problem you need extra force for */;
1689 else
1690 MetaIndex->QueueIndexes(GoodLoad);
1691
1692 TransactionManager->TransactionStageCopy(MetaIndex, MetaIndex->DestFile, MetaIndex->GetFinalFilename());
1693 }
1694
1695 // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor
1696 if (Cnf->LocalOnly == true ||
1697 StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1698 {
1699 // Ignore this
1700 Status = StatDone;
1701 }
1702 }
1703 /*}}}*/
1704
1705
1706 // AcqBaseIndex - Constructor /*{{{*/
1707 pkgAcqBaseIndex::pkgAcqBaseIndex(pkgAcquire * const Owner,
1708 pkgAcqMetaClearSig * const TransactionManager,
1709 IndexTarget const &Target)
1710 : pkgAcqTransactionItem(Owner, TransactionManager, Target), d(NULL)
1711 {
1712 }
1713 /*}}}*/
1714 pkgAcqBaseIndex::~pkgAcqBaseIndex() {}
1715
1716 // AcqDiffIndex::AcqDiffIndex - Constructor /*{{{*/
1717 // ---------------------------------------------------------------------
1718 /* Get the DiffIndex file first and see if there are patches available
1719 * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
1720 * patches. If anything goes wrong in that process, it will fall back to
1721 * the original packages file
1722 */
1723 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire * const Owner,
1724 pkgAcqMetaClearSig * const TransactionManager,
1725 IndexTarget const &Target)
1726 : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL), diffs(NULL)
1727 {
1728 Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
1729
1730 Desc.Owner = this;
1731 Desc.Description = GetDiffIndexFileName(Target.Description);
1732 Desc.ShortDesc = Target.ShortDesc;
1733 Desc.URI = GetDiffIndexURI(Target);
1734
1735 DestFile = GetPartialFileNameFromURI(Desc.URI);
1736
1737 if(Debug)
1738 std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
1739
1740 QueueURI(Desc);
1741 }
1742 /*}}}*/
1743 // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/
1744 // ---------------------------------------------------------------------
1745 /* The only header we use is the last-modified header. */
1746 string pkgAcqDiffIndex::Custom600Headers() const
1747 {
1748 if (TransactionManager->LastMetaIndexParser != NULL)
1749 return "\nIndex-File: true";
1750
1751 string const Final = GetFinalFilename();
1752
1753 if(Debug)
1754 std::clog << "Custom600Header-IMS: " << Final << std::endl;
1755
1756 struct stat Buf;
1757 if (stat(Final.c_str(),&Buf) != 0)
1758 return "\nIndex-File: true";
1759
1760 return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1761 }
1762 /*}}}*/
1763 void pkgAcqDiffIndex::QueueOnIMSHit() const /*{{{*/
1764 {
1765 // list cleanup needs to know that this file as well as the already
1766 // present index is ours, so we create an empty diff to save it for us
1767 new pkgAcqIndexDiffs(Owner, TransactionManager, Target);
1768 }
1769 /*}}}*/
1770 bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
1771 {
1772 // failing here is fine: our caller will take care of trying to
1773 // get the complete file if patching fails
1774 if(Debug)
1775 std::clog << "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile
1776 << std::endl;
1777
1778 FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
1779 pkgTagFile TF(&Fd);
1780 if (Fd.IsOpen() == false || Fd.Failed())
1781 return false;
1782
1783 pkgTagSection Tags;
1784 if(unlikely(TF.Step(Tags) == false))
1785 return false;
1786
1787 HashStringList ServerHashes;
1788 unsigned long long ServerSize = 0;
1789
1790 for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
1791 {
1792 std::string tagname = *type;
1793 tagname.append("-Current");
1794 std::string const tmp = Tags.FindS(tagname.c_str());
1795 if (tmp.empty() == true)
1796 continue;
1797
1798 string hash;
1799 unsigned long long size;
1800 std::stringstream ss(tmp);
1801 ss >> hash >> size;
1802 if (unlikely(hash.empty() == true))
1803 continue;
1804 if (unlikely(ServerSize != 0 && ServerSize != size))
1805 continue;
1806 ServerHashes.push_back(HashString(*type, hash));
1807 ServerSize = size;
1808 }
1809
1810 if (ServerHashes.usable() == false)
1811 {
1812 if (Debug == true)
1813 std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": Did not find a good hashsum in the index" << std::endl;
1814 return false;
1815 }
1816
1817 std::string const CurrentPackagesFile = GetFinalFileNameFromURI(Target.URI);
1818 HashStringList const TargetFileHashes = GetExpectedHashesFor(Target.MetaKey);
1819 if (TargetFileHashes.usable() == false || ServerHashes != TargetFileHashes)
1820 {
1821 if (Debug == true)
1822 {
1823 std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": Index has different hashes than parser, probably older, so fail pdiffing" << std::endl;
1824 printHashSumComparison(CurrentPackagesFile, ServerHashes, TargetFileHashes);
1825 }
1826 return false;
1827 }
1828
1829 HashStringList LocalHashes;
1830 // try avoiding calculating the hash here as this is costly
1831 if (TransactionManager->LastMetaIndexParser != NULL)
1832 LocalHashes = GetExpectedHashesFromFor(TransactionManager->LastMetaIndexParser, Target.MetaKey);
1833 if (LocalHashes.usable() == false)
1834 {
1835 FileFd fd(CurrentPackagesFile, FileFd::ReadOnly, FileFd::Auto);
1836 Hashes LocalHashesCalc(ServerHashes);
1837 LocalHashesCalc.AddFD(fd);
1838 LocalHashes = LocalHashesCalc.GetHashStringList();
1839 }
1840
1841 if (ServerHashes == LocalHashes)
1842 {
1843 // we have the same sha1 as the server so we are done here
1844 if(Debug)
1845 std::clog << "pkgAcqDiffIndex: Package file " << CurrentPackagesFile << " is up-to-date" << std::endl;
1846 QueueOnIMSHit();
1847 return true;
1848 }
1849
1850 if(Debug)
1851 std::clog << "Server-Current: " << ServerHashes.find(NULL)->toStr() << " and we start at "
1852 << CurrentPackagesFile << " " << LocalHashes.FileSize() << " " << LocalHashes.find(NULL)->toStr() << std::endl;
1853
1854 // historically, older hashes have more info than newer ones, so start
1855 // collecting with older ones first to avoid implementing complicated
1856 // information merging techniques… a failure is after all always
1857 // recoverable with a complete file and hashes aren't changed that often.
1858 std::vector<char const *> types;
1859 for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
1860 types.push_back(*type);
1861
1862 // parse all of (provided) history
1863 vector<DiffInfo> available_patches;
1864 bool firstAcceptedHashes = true;
1865 for (auto type = types.crbegin(); type != types.crend(); ++type)
1866 {
1867 if (LocalHashes.find(*type) == NULL)
1868 continue;
1869
1870 std::string tagname = *type;
1871 tagname.append("-History");
1872 std::string const tmp = Tags.FindS(tagname.c_str());
1873 if (tmp.empty() == true)
1874 continue;
1875
1876 string hash, filename;
1877 unsigned long long size;
1878 std::stringstream ss(tmp);
1879
1880 while (ss >> hash >> size >> filename)
1881 {
1882 if (unlikely(hash.empty() == true || filename.empty() == true))
1883 continue;
1884
1885 // see if we have a record for this file already
1886 std::vector<DiffInfo>::iterator cur = available_patches.begin();
1887 for (; cur != available_patches.end(); ++cur)
1888 {
1889 if (cur->file != filename)
1890 continue;
1891 cur->result_hashes.push_back(HashString(*type, hash));
1892 break;
1893 }
1894 if (cur != available_patches.end())
1895 continue;
1896 if (firstAcceptedHashes == true)
1897 {
1898 DiffInfo next;
1899 next.file = filename;
1900 next.result_hashes.push_back(HashString(*type, hash));
1901 next.result_hashes.FileSize(size);
1902 available_patches.push_back(next);
1903 }
1904 else
1905 {
1906 if (Debug == true)
1907 std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename
1908 << " wasn't in the list for the first parsed hash! (history)" << std::endl;
1909 break;
1910 }
1911 }
1912 firstAcceptedHashes = false;
1913 }
1914
1915 if (unlikely(available_patches.empty() == true))
1916 {
1917 if (Debug)
1918 std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": "
1919 << "Couldn't find any patches for the patch series." << std::endl;
1920 return false;
1921 }
1922
1923 for (auto type = types.crbegin(); type != types.crend(); ++type)
1924 {
1925 if (LocalHashes.find(*type) == NULL)
1926 continue;
1927
1928 std::string tagname = *type;
1929 tagname.append("-Patches");
1930 std::string const tmp = Tags.FindS(tagname.c_str());
1931 if (tmp.empty() == true)
1932 continue;
1933
1934 string hash, filename;
1935 unsigned long long size;
1936 std::stringstream ss(tmp);
1937
1938 while (ss >> hash >> size >> filename)
1939 {
1940 if (unlikely(hash.empty() == true || filename.empty() == true))
1941 continue;
1942
1943 // see if we have a record for this file already
1944 std::vector<DiffInfo>::iterator cur = available_patches.begin();
1945 for (; cur != available_patches.end(); ++cur)
1946 {
1947 if (cur->file != filename)
1948 continue;
1949 if (cur->patch_hashes.empty())
1950 cur->patch_hashes.FileSize(size);
1951 cur->patch_hashes.push_back(HashString(*type, hash));
1952 break;
1953 }
1954 if (cur != available_patches.end())
1955 continue;
1956 if (Debug == true)
1957 std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename
1958 << " wasn't in the list for the first parsed hash! (patches)" << std::endl;
1959 break;
1960 }
1961 }
1962
1963 for (auto type = types.crbegin(); type != types.crend(); ++type)
1964 {
1965 std::string tagname = *type;
1966 tagname.append("-Download");
1967 std::string const tmp = Tags.FindS(tagname.c_str());
1968 if (tmp.empty() == true)
1969 continue;
1970
1971 string hash, filename;
1972 unsigned long long size;
1973 std::stringstream ss(tmp);
1974
1975 // FIXME: all of pdiff supports only .gz compressed patches
1976 while (ss >> hash >> size >> filename)
1977 {
1978 if (unlikely(hash.empty() == true || filename.empty() == true))
1979 continue;
1980 if (unlikely(APT::String::Endswith(filename, ".gz") == false))
1981 continue;
1982 filename.erase(filename.length() - 3);
1983
1984 // see if we have a record for this file already
1985 std::vector<DiffInfo>::iterator cur = available_patches.begin();
1986 for (; cur != available_patches.end(); ++cur)
1987 {
1988 if (cur->file != filename)
1989 continue;
1990 if (cur->download_hashes.empty())
1991 cur->download_hashes.FileSize(size);
1992 cur->download_hashes.push_back(HashString(*type, hash));
1993 break;
1994 }
1995 if (cur != available_patches.end())
1996 continue;
1997 if (Debug == true)
1998 std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename
1999 << " wasn't in the list for the first parsed hash! (download)" << std::endl;
2000 break;
2001 }
2002 }
2003
2004
2005 bool foundStart = false;
2006 for (std::vector<DiffInfo>::iterator cur = available_patches.begin();
2007 cur != available_patches.end(); ++cur)
2008 {
2009 if (LocalHashes != cur->result_hashes)
2010 continue;
2011
2012 available_patches.erase(available_patches.begin(), cur);
2013 foundStart = true;
2014 break;
2015 }
2016
2017 if (foundStart == false || unlikely(available_patches.empty() == true))
2018 {
2019 if (Debug)
2020 std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": "
2021 << "Couldn't find the start of the patch series." << std::endl;
2022 return false;
2023 }
2024
2025 for (auto const &patch: available_patches)
2026 if (patch.result_hashes.usable() == false ||
2027 patch.patch_hashes.usable() == false ||
2028 patch.download_hashes.usable() == false)
2029 {
2030 if (Debug)
2031 std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": provides no usable hashes for " << patch.file
2032 << " so fallback to complete download" << std::endl;
2033 return false;
2034 }
2035
2036 // patching with too many files is rather slow compared to a fast download
2037 unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
2038 if (fileLimit != 0 && fileLimit < available_patches.size())
2039 {
2040 if (Debug)
2041 std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
2042 << ") so fallback to complete download" << std::endl;
2043 return false;
2044 }
2045
2046 // calculate the size of all patches we have to get
2047 unsigned short const sizeLimitPercent = _config->FindI("Acquire::PDiffs::SizeLimit", 100);
2048 if (sizeLimitPercent > 0 && TransactionManager->MetaIndexParser != nullptr)
2049 {
2050 unsigned long long downloadSize = std::accumulate(available_patches.begin(),
2051 available_patches.end(), 0llu, [](unsigned long long const T, DiffInfo const &I) {
2052 return T + I.download_hashes.FileSize();
2053 });
2054 if (downloadSize != 0)
2055 {
2056 unsigned long long downloadSizeIdx = 0;
2057 auto const types = VectorizeString(Target.Option(IndexTarget::COMPRESSIONTYPES), ' ');
2058 for (auto const &t : types)
2059 {
2060 std::string MetaKey = Target.MetaKey;
2061 if (t != "uncompressed")
2062 MetaKey += '.' + t;
2063 HashStringList const hsl = GetExpectedHashesFor(MetaKey);
2064 if (unlikely(hsl.usable() == false))
2065 continue;
2066 downloadSizeIdx = hsl.FileSize();
2067 break;
2068 }
2069 unsigned long long const sizeLimit = downloadSizeIdx * sizeLimitPercent;
2070 if ((sizeLimit/100) < downloadSize)
2071 {
2072 if (Debug)
2073 std::clog << "Need " << downloadSize << " compressed bytes (Limit is " << (sizeLimit/100) << ", "
2074 << "original is " << downloadSizeIdx << ") so fallback to complete download" << std::endl;
2075 return false;
2076 }
2077 }
2078 }
2079
2080 // we have something, queue the diffs
2081 string::size_type const last_space = Description.rfind(" ");
2082 if(last_space != string::npos)
2083 Description.erase(last_space, Description.size()-last_space);
2084
2085 /* decide if we should download patches one by one or in one go:
2086 The first is good if the server merges patches, but many don't so client
2087 based merging can be attempt in which case the second is better.
2088 "bad things" will happen if patches are merged on the server,
2089 but client side merging is attempt as well */
2090 bool pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true);
2091 if (pdiff_merge == true)
2092 {
2093 // reprepro adds this flag if it has merged patches on the server
2094 std::string const precedence = Tags.FindS("X-Patch-Precedence");
2095 pdiff_merge = (precedence != "merged");
2096 }
2097
2098 // clean the plate
2099 {
2100 std::string const Final = GetExistingFilename(CurrentPackagesFile);
2101 if (unlikely(Final.empty())) // because we wouldn't be called in such a case
2102 return false;
2103 std::string const PartialFile = GetPartialFileNameFromURI(Target.URI);
2104 if (FileExists(PartialFile) && RemoveFile("Bootstrap-linking", PartialFile) == false)
2105 {
2106 if (Debug)
2107 std::clog << "Bootstrap-linking for patching " << CurrentPackagesFile
2108 << " by removing stale " << PartialFile << " failed!" << std::endl;
2109 return false;
2110 }
2111 for (auto const &ext : APT::Configuration::getCompressorExtensions())
2112 {
2113 std::string const Partial = PartialFile + ext;
2114 if (FileExists(Partial) && RemoveFile("Bootstrap-linking", Partial) == false)
2115 {
2116 if (Debug)
2117 std::clog << "Bootstrap-linking for patching " << CurrentPackagesFile
2118 << " by removing stale " << Partial << " failed!" << std::endl;
2119 return false;
2120 }
2121 }
2122 std::string const Ext = Final.substr(CurrentPackagesFile.length());
2123 std::string const Partial = PartialFile + Ext;
2124 if (symlink(Final.c_str(), Partial.c_str()) != 0)
2125 {
2126 if (Debug)
2127 std::clog << "Bootstrap-linking for patching " << CurrentPackagesFile
2128 << " by linking " << Final << " to " << Partial << " failed!" << std::endl;
2129 return false;
2130 }
2131 }
2132
2133 if (pdiff_merge == false)
2134 new pkgAcqIndexDiffs(Owner, TransactionManager, Target, available_patches);
2135 else
2136 {
2137 diffs = new std::vector<pkgAcqIndexMergeDiffs*>(available_patches.size());
2138 for(size_t i = 0; i < available_patches.size(); ++i)
2139 (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, TransactionManager,
2140 Target,
2141 available_patches[i],
2142 diffs);
2143 }
2144
2145 Complete = false;
2146 Status = StatDone;
2147 Dequeue();
2148 return true;
2149 }
2150 /*}}}*/
2151 void pkgAcqDiffIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
2152 {
2153 Item::Failed(Message,Cnf);
2154 Status = StatDone;
2155
2156 if(Debug)
2157 std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl
2158 << "Falling back to normal index file acquire" << std::endl;
2159
2160 new pkgAcqIndex(Owner, TransactionManager, Target);
2161 }
2162 /*}}}*/
2163 void pkgAcqDiffIndex::Done(string const &Message,HashStringList const &Hashes, /*{{{*/
2164 pkgAcquire::MethodConfig const * const Cnf)
2165 {
2166 if(Debug)
2167 std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
2168
2169 Item::Done(Message, Hashes, Cnf);
2170
2171 string const FinalFile = GetFinalFilename();
2172 if(StringToBool(LookupTag(Message,"IMS-Hit"),false))
2173 DestFile = FinalFile;
2174
2175 if(ParseDiffIndex(DestFile) == false)
2176 {
2177 Failed("Message: Couldn't parse pdiff index", Cnf);
2178 // queue for final move - this should happen even if we fail
2179 // while parsing (e.g. on sizelimit) and download the complete file.
2180 TransactionManager->TransactionStageCopy(this, DestFile, FinalFile);
2181 return;
2182 }
2183
2184 TransactionManager->TransactionStageCopy(this, DestFile, FinalFile);
2185
2186 Complete = true;
2187 Status = StatDone;
2188 Dequeue();
2189
2190 return;
2191 }
2192 /*}}}*/
2193 pkgAcqDiffIndex::~pkgAcqDiffIndex()
2194 {
2195 if (diffs != NULL)
2196 delete diffs;
2197 }
2198
2199 // AcqIndexDiffs::AcqIndexDiffs - Constructor /*{{{*/
2200 // ---------------------------------------------------------------------
2201 /* The package diff is added to the queue. one object is constructed
2202 * for each diff and the index
2203 */
2204 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire * const Owner,
2205 pkgAcqMetaClearSig * const TransactionManager,
2206 IndexTarget const &Target,
2207 vector<DiffInfo> const &diffs)
2208 : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL),
2209 available_patches(diffs)
2210 {
2211 DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target);
2212
2213 Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
2214
2215 Desc.Owner = this;
2216 Description = Target.Description;
2217 Desc.ShortDesc = Target.ShortDesc;
2218
2219 if(available_patches.empty() == true)
2220 {
2221 // we are done (yeah!), check hashes against the final file
2222 DestFile = GetKeepCompressedFileName(GetFinalFileNameFromURI(Target.URI), Target);
2223 Finish(true);
2224 }
2225 else
2226 {
2227 State = StateFetchDiff;
2228 QueueNextDiff();
2229 }
2230 }
2231 /*}}}*/
2232 void pkgAcqIndexDiffs::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
2233 {
2234 Item::Failed(Message,Cnf);
2235 Status = StatDone;
2236
2237 DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target);
2238 if(Debug)
2239 std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl
2240 << "Falling back to normal index file acquire " << std::endl;
2241 RenameOnError(PDiffError);
2242 std::string const patchname = GetDiffsPatchFileName(DestFile);
2243 if (RealFileExists(patchname))
2244 Rename(patchname, patchname + ".FAILED");
2245 std::string const UnpatchedFile = GetExistingFilename(GetPartialFileNameFromURI(Target.URI));
2246 if (UnpatchedFile.empty() == false && FileExists(UnpatchedFile))
2247 Rename(UnpatchedFile, UnpatchedFile + ".FAILED");
2248 new pkgAcqIndex(Owner, TransactionManager, Target);
2249 Finish();
2250 }
2251 /*}}}*/
2252 // Finish - helper that cleans the item out of the fetcher queue /*{{{*/
2253 void pkgAcqIndexDiffs::Finish(bool allDone)
2254 {
2255 if(Debug)
2256 std::clog << "pkgAcqIndexDiffs::Finish(): "
2257 << allDone << " "
2258 << Desc.URI << std::endl;
2259
2260 // we restore the original name, this is required, otherwise
2261 // the file will be cleaned
2262 if(allDone)
2263 {
2264 std::string const Final = GetKeepCompressedFileName(GetFinalFilename(), Target);
2265 TransactionManager->TransactionStageCopy(this, DestFile, Final);
2266
2267 // this is for the "real" finish
2268 Complete = true;
2269 Status = StatDone;
2270 Dequeue();
2271 if(Debug)
2272 std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
2273 return;
2274 }
2275 else
2276 DestFile.clear();
2277
2278 if(Debug)
2279 std::clog << "Finishing: " << Desc.URI << std::endl;
2280 Complete = false;
2281 Status = StatDone;
2282 Dequeue();
2283 return;
2284 }
2285 /*}}}*/
2286 bool pkgAcqIndexDiffs::QueueNextDiff() /*{{{*/
2287 {
2288 // calc sha1 of the just patched file
2289 std::string const PartialFile = GetExistingFilename(GetPartialFileNameFromURI(Target.URI));
2290 if(unlikely(PartialFile.empty()))
2291 {
2292 Failed("Message: The file " + GetPartialFileNameFromURI(Target.URI) + " isn't available", NULL);
2293 return false;
2294 }
2295
2296 FileFd fd(PartialFile, FileFd::ReadOnly, FileFd::Extension);
2297 Hashes LocalHashesCalc;
2298 LocalHashesCalc.AddFD(fd);
2299 HashStringList const LocalHashes = LocalHashesCalc.GetHashStringList();
2300
2301 if(Debug)
2302 std::clog << "QueueNextDiff: " << PartialFile << " (" << LocalHashes.find(NULL)->toStr() << ")" << std::endl;
2303
2304 HashStringList const TargetFileHashes = GetExpectedHashesFor(Target.MetaKey);
2305 if (unlikely(LocalHashes.usable() == false || TargetFileHashes.usable() == false))
2306 {
2307 Failed("Local/Expected hashes are not usable for " + PartialFile, NULL);
2308 return false;
2309 }
2310
2311 // final file reached before all patches are applied
2312 if(LocalHashes == TargetFileHashes)
2313 {
2314 Finish(true);
2315 return true;
2316 }
2317
2318 // remove all patches until the next matching patch is found
2319 // this requires the Index file to be ordered
2320 available_patches.erase(available_patches.begin(),
2321 std::find_if(available_patches.begin(), available_patches.end(), [&](DiffInfo const &I) {
2322 return I.result_hashes == LocalHashes;
2323 }));
2324
2325 // error checking and falling back if no patch was found
2326 if(available_patches.empty() == true)
2327 {
2328 Failed("No patches left to reach target for " + PartialFile, NULL);
2329 return false;
2330 }
2331
2332 // queue the right diff
2333 Desc.URI = Target.URI + ".diff/" + available_patches[0].file + ".gz";
2334 Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
2335 DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI + ".diff/" + available_patches[0].file), Target);
2336
2337 if(Debug)
2338 std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
2339
2340 QueueURI(Desc);
2341
2342 return true;
2343 }
2344 /*}}}*/
2345 void pkgAcqIndexDiffs::Done(string const &Message, HashStringList const &Hashes, /*{{{*/
2346 pkgAcquire::MethodConfig const * const Cnf)
2347 {
2348 if (Debug)
2349 std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
2350
2351 Item::Done(Message, Hashes, Cnf);
2352
2353 std::string const UncompressedUnpatchedFile = GetPartialFileNameFromURI(Target.URI);
2354 std::string const UnpatchedFile = GetExistingFilename(UncompressedUnpatchedFile);
2355 std::string const PatchFile = GetDiffsPatchFileName(UnpatchedFile);
2356 std::string const PatchedFile = GetKeepCompressedFileName(UncompressedUnpatchedFile, Target);
2357
2358 switch (State)
2359 {
2360 // success in downloading a diff, enter ApplyDiff state
2361 case StateFetchDiff:
2362 Rename(DestFile, PatchFile);
2363 DestFile = GetKeepCompressedFileName(UncompressedUnpatchedFile + "-patched", Target);
2364 if(Debug)
2365 std::clog << "Sending to rred method: " << UnpatchedFile << std::endl;
2366 State = StateApplyDiff;
2367 Local = true;
2368 Desc.URI = "rred:" + UnpatchedFile;
2369 QueueURI(Desc);
2370 SetActiveSubprocess("rred");
2371 return;
2372 // success in download/apply a diff, queue next (if needed)
2373 case StateApplyDiff:
2374 // remove the just applied patch and base file
2375 available_patches.erase(available_patches.begin());
2376 RemoveFile("pkgAcqIndexDiffs::Done", PatchFile);
2377 RemoveFile("pkgAcqIndexDiffs::Done", UnpatchedFile);
2378 if(Debug)
2379 std::clog << "Moving patched file in place: " << std::endl
2380 << DestFile << " -> " << PatchedFile << std::endl;
2381 Rename(DestFile, PatchedFile);
2382
2383 // see if there is more to download
2384 if(available_patches.empty() == false)
2385 {
2386 new pkgAcqIndexDiffs(Owner, TransactionManager, Target, available_patches);
2387 Finish();
2388 } else {
2389 DestFile = PatchedFile;
2390 Finish(true);
2391 }
2392 return;
2393 }
2394 }
2395 /*}}}*/
2396 std::string pkgAcqIndexDiffs::Custom600Headers() const /*{{{*/
2397 {
2398 if(State != StateApplyDiff)
2399 return pkgAcqBaseIndex::Custom600Headers();
2400 std::ostringstream patchhashes;
2401 HashStringList const ExpectedHashes = available_patches[0].patch_hashes;
2402 for (HashStringList::const_iterator hs = ExpectedHashes.begin(); hs != ExpectedHashes.end(); ++hs)
2403 patchhashes << "\nPatch-0-" << hs->HashType() << "-Hash: " << hs->HashValue();
2404 patchhashes << pkgAcqBaseIndex::Custom600Headers();
2405 return patchhashes.str();
2406 }
2407 /*}}}*/
2408 pkgAcqIndexDiffs::~pkgAcqIndexDiffs() {}
2409
2410 // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor /*{{{*/
2411 pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire * const Owner,
2412 pkgAcqMetaClearSig * const TransactionManager,
2413 IndexTarget const &Target,
2414 DiffInfo const &patch,
2415 std::vector<pkgAcqIndexMergeDiffs*> const * const allPatches)
2416 : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL),
2417 patch(patch), allPatches(allPatches), State(StateFetchDiff)
2418 {
2419 Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
2420
2421 Desc.Owner = this;
2422 Description = Target.Description;
2423 Desc.ShortDesc = Target.ShortDesc;
2424 Desc.URI = Target.URI + ".diff/" + patch.file + ".gz";
2425 Desc.Description = Description + " " + patch.file + ".pdiff";
2426 DestFile = GetPartialFileNameFromURI(Desc.URI);
2427
2428 if(Debug)
2429 std::clog << "pkgAcqIndexMergeDiffs: " << Desc.URI << std::endl;
2430
2431 QueueURI(Desc);
2432 }
2433 /*}}}*/
2434 void pkgAcqIndexMergeDiffs::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
2435 {
2436 if(Debug)
2437 std::clog << "pkgAcqIndexMergeDiffs failed: " << Desc.URI << " with " << Message << std::endl;
2438
2439 Item::Failed(Message,Cnf);
2440 Status = StatDone;
2441
2442 // check if we are the first to fail, otherwise we are done here
2443 State = StateDoneDiff;
2444 for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
2445 I != allPatches->end(); ++I)
2446 if ((*I)->State == StateErrorDiff)
2447 {
2448 State = StateErrorDiff;
2449 return;
2450 }
2451
2452 // first failure means we should fallback
2453 State = StateErrorDiff;
2454 if (Debug)
2455 std::clog << "Falling back to normal index file acquire" << std::endl;
2456 RenameOnError(PDiffError);
2457 if (RealFileExists(DestFile))
2458 Rename(DestFile, DestFile + ".FAILED");
2459 std::string const UnpatchedFile = GetExistingFilename(GetPartialFileNameFromURI(Target.URI));
2460 if (UnpatchedFile.empty() == false && FileExists(UnpatchedFile))
2461 Rename(UnpatchedFile, UnpatchedFile + ".FAILED");
2462 DestFile.clear();
2463 new pkgAcqIndex(Owner, TransactionManager, Target);
2464 }
2465 /*}}}*/
2466 void pkgAcqIndexMergeDiffs::Done(string const &Message, HashStringList const &Hashes, /*{{{*/
2467 pkgAcquire::MethodConfig const * const Cnf)
2468 {
2469 if(Debug)
2470 std::clog << "pkgAcqIndexMergeDiffs::Done(): " << Desc.URI << std::endl;
2471
2472 Item::Done(Message, Hashes, Cnf);
2473
2474 if (std::any_of(allPatches->begin(), allPatches->end(),
2475 [](pkgAcqIndexMergeDiffs const * const P) { return P->State == StateErrorDiff; }))
2476 {
2477 if(Debug)
2478 std::clog << "Another patch failed already, no point in processing this one." << std::endl;
2479 State = StateErrorDiff;
2480 return;
2481 }
2482
2483 std::string const UncompressedUnpatchedFile = GetPartialFileNameFromURI(Target.URI);
2484 std::string const UnpatchedFile = GetExistingFilename(UncompressedUnpatchedFile);
2485 if (UnpatchedFile.empty())
2486 {
2487 _error->Fatal("Unpatched file %s doesn't exist (anymore)!", UncompressedUnpatchedFile.c_str());
2488 State = StateErrorDiff;
2489 return;
2490 }
2491 std::string const PatchFile = GetMergeDiffsPatchFileName(UnpatchedFile, patch.file);
2492 std::string const PatchedFile = GetKeepCompressedFileName(UncompressedUnpatchedFile, Target);
2493
2494 switch (State)
2495 {
2496 case StateFetchDiff:
2497 Rename(DestFile, PatchFile);
2498
2499 // check if this is the last completed diff
2500 State = StateDoneDiff;
2501 for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
2502 I != allPatches->end(); ++I)
2503 if ((*I)->State != StateDoneDiff)
2504 {
2505 if(Debug)
2506 std::clog << "Not the last done diff in the batch: " << Desc.URI << std::endl;
2507 return;
2508 }
2509 // this is the last completed diff, so we are ready to apply now
2510 DestFile = GetKeepCompressedFileName(UncompressedUnpatchedFile + "-patched", Target);
2511 if(Debug)
2512 std::clog << "Sending to rred method: " << UnpatchedFile << std::endl;
2513 State = StateApplyDiff;
2514 Local = true;
2515 Desc.URI = "rred:" + UnpatchedFile;
2516 QueueURI(Desc);
2517 SetActiveSubprocess("rred");
2518 return;
2519 case StateApplyDiff:
2520 // success in download & apply all diffs, finialize and clean up
2521 if(Debug)
2522 std::clog << "Queue patched file in place: " << std::endl
2523 << DestFile << " -> " << PatchedFile << std::endl;
2524
2525 // queue for copy by the transaction manager
2526 TransactionManager->TransactionStageCopy(this, DestFile, GetKeepCompressedFileName(GetFinalFilename(), Target));
2527
2528 // ensure the ed's are gone regardless of list-cleanup
2529 for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
2530 I != allPatches->end(); ++I)
2531 RemoveFile("pkgAcqIndexMergeDiffs::Done", GetMergeDiffsPatchFileName(UnpatchedFile, (*I)->patch.file));
2532 RemoveFile("pkgAcqIndexMergeDiffs::Done", UnpatchedFile);
2533
2534 // all set and done
2535 Complete = true;
2536 if(Debug)
2537 std::clog << "allDone: " << DestFile << "\n" << std::endl;
2538 return;
2539 case StateDoneDiff: _error->Fatal("Done called for %s which is in an invalid Done state", PatchFile.c_str()); break;
2540 case StateErrorDiff: _error->Fatal("Done called for %s which is in an invalid Error state", PatchFile.c_str()); break;
2541 }
2542 }
2543 /*}}}*/
2544 std::string pkgAcqIndexMergeDiffs::Custom600Headers() const /*{{{*/
2545 {
2546 if(State != StateApplyDiff)
2547 return pkgAcqBaseIndex::Custom600Headers();
2548 std::ostringstream patchhashes;
2549 unsigned int seen_patches = 0;
2550 for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
2551 I != allPatches->end(); ++I)
2552 {
2553 HashStringList const ExpectedHashes = (*I)->patch.patch_hashes;
2554 for (HashStringList::const_iterator hs = ExpectedHashes.begin(); hs != ExpectedHashes.end(); ++hs)
2555 patchhashes << "\nPatch-" << seen_patches << "-" << hs->HashType() << "-Hash: " << hs->HashValue();
2556 ++seen_patches;
2557 }
2558 patchhashes << pkgAcqBaseIndex::Custom600Headers();
2559 return patchhashes.str();
2560 }
2561 /*}}}*/
2562 pkgAcqIndexMergeDiffs::~pkgAcqIndexMergeDiffs() {}
2563
2564 // AcqIndex::AcqIndex - Constructor /*{{{*/
2565 pkgAcqIndex::pkgAcqIndex(pkgAcquire * const Owner,
2566 pkgAcqMetaClearSig * const TransactionManager,
2567 IndexTarget const &Target)
2568 : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL), Stage(STAGE_DOWNLOAD),
2569 CompressionExtensions(Target.Option(IndexTarget::COMPRESSIONTYPES))
2570 {
2571 Init(Target.URI, Target.Description, Target.ShortDesc);
2572
2573 if(_config->FindB("Debug::Acquire::Transaction", false) == true)
2574 std::clog << "New pkgIndex with TransactionManager "
2575 << TransactionManager << std::endl;
2576 }
2577 /*}}}*/
2578 // AcqIndex::Init - defered Constructor /*{{{*/
2579 static void NextCompressionExtension(std::string &CurrentCompressionExtension, std::string &CompressionExtensions, bool const preview)
2580 {
2581 size_t const nextExt = CompressionExtensions.find(' ');
2582 if (nextExt == std::string::npos)
2583 {
2584 CurrentCompressionExtension = CompressionExtensions;
2585 if (preview == false)
2586 CompressionExtensions.clear();
2587 }
2588 else
2589 {
2590 CurrentCompressionExtension = CompressionExtensions.substr(0, nextExt);
2591 if (preview == false)
2592 CompressionExtensions = CompressionExtensions.substr(nextExt+1);
2593 }
2594 }
2595 void pkgAcqIndex::Init(string const &URI, string const &URIDesc,
2596 string const &ShortDesc)
2597 {
2598 Stage = STAGE_DOWNLOAD;
2599
2600 DestFile = GetPartialFileNameFromURI(URI);
2601 NextCompressionExtension(CurrentCompressionExtension, CompressionExtensions, false);
2602
2603 // store file size of the download to ensure the fetcher gives
2604 // accurate progress reporting
2605 FileSize = GetExpectedHashes().FileSize();
2606
2607 if (CurrentCompressionExtension == "uncompressed")
2608 {
2609 Desc.URI = URI;
2610 }
2611 else if (CurrentCompressionExtension == "by-hash")
2612 {
2613 NextCompressionExtension(CurrentCompressionExtension, CompressionExtensions, true);
2614 if(unlikely(TransactionManager->MetaIndexParser == NULL || CurrentCompressionExtension.empty()))
2615 return;
2616 if (CurrentCompressionExtension != "uncompressed")
2617 {
2618 Desc.URI = URI + '.' + CurrentCompressionExtension;
2619 DestFile = DestFile + '.' + CurrentCompressionExtension;
2620 }
2621
2622 HashStringList const Hashes = GetExpectedHashes();
2623 HashString const * const TargetHash = Hashes.find(NULL);
2624 if (unlikely(TargetHash == nullptr))
2625 return;
2626 std::string const ByHash = "/by-hash/" + TargetHash->HashType() + "/" + TargetHash->HashValue();
2627 size_t const trailing_slash = Desc.URI.find_last_of("/");
2628 if (unlikely(trailing_slash == std::string::npos))
2629 return;
2630 Desc.URI = Desc.URI.replace(
2631 trailing_slash,
2632 Desc.URI.substr(trailing_slash+1).size()+1,
2633 ByHash);
2634 }
2635 else if (unlikely(CurrentCompressionExtension.empty()))
2636 return;
2637 else
2638 {
2639 Desc.URI = URI + '.' + CurrentCompressionExtension;
2640 DestFile = DestFile + '.' + CurrentCompressionExtension;
2641 }
2642
2643
2644 Desc.Description = URIDesc;
2645 Desc.Owner = this;
2646 Desc.ShortDesc = ShortDesc;
2647
2648 QueueURI(Desc);
2649 }
2650 /*}}}*/
2651 // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/
2652 // ---------------------------------------------------------------------
2653 /* The only header we use is the last-modified header. */
2654 string pkgAcqIndex::Custom600Headers() const
2655 {
2656
2657 string msg = "\nIndex-File: true";
2658
2659 if (TransactionManager->LastMetaIndexParser == NULL)
2660 {
2661 std::string const Final = GetFinalFilename();
2662
2663 struct stat Buf;
2664 if (stat(Final.c_str(),&Buf) == 0)
2665 msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
2666 }
2667
2668 if(Target.IsOptional)
2669 msg += "\nFail-Ignore: true";
2670
2671 return msg;
2672 }
2673 /*}}}*/
2674 // AcqIndex::Failed - getting the indexfile failed /*{{{*/
2675 void pkgAcqIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)
2676 {
2677 Item::Failed(Message,Cnf);
2678
2679 // authorisation matches will not be fixed by other compression types
2680 if (Status != StatAuthError)
2681 {
2682 if (CompressionExtensions.empty() == false)
2683 {
2684 Init(Target.URI, Desc.Description, Desc.ShortDesc);
2685 Status = StatIdle;
2686 return;
2687 }
2688 }
2689
2690 if(Target.IsOptional && GetExpectedHashes().empty() && Stage == STAGE_DOWNLOAD)
2691 Status = StatDone;
2692 else
2693 TransactionManager->AbortTransaction();
2694 }
2695 /*}}}*/
2696 // AcqIndex::Done - Finished a fetch /*{{{*/
2697 // ---------------------------------------------------------------------
2698 /* This goes through a number of states.. On the initial fetch the
2699 method could possibly return an alternate filename which points
2700 to the uncompressed version of the file. If this is so the file
2701 is copied into the partial directory. In all other cases the file
2702 is decompressed with a compressed uri. */
2703 void pkgAcqIndex::Done(string const &Message,
2704 HashStringList const &Hashes,
2705 pkgAcquire::MethodConfig const * const Cfg)
2706 {
2707 Item::Done(Message,Hashes,Cfg);
2708
2709 switch(Stage)
2710 {
2711 case STAGE_DOWNLOAD:
2712 StageDownloadDone(Message);
2713 break;
2714 case STAGE_DECOMPRESS_AND_VERIFY:
2715 StageDecompressDone();
2716 break;
2717 }
2718 }
2719 /*}}}*/
2720 // AcqIndex::StageDownloadDone - Queue for decompress and verify /*{{{*/
2721 void pkgAcqIndex::StageDownloadDone(string const &Message)
2722 {
2723 Local = true;
2724 Complete = true;
2725
2726 std::string const AltFilename = LookupTag(Message,"Alt-Filename");
2727 std::string Filename = LookupTag(Message,"Filename");
2728
2729 // we need to verify the file against the current Release file again
2730 // on if-modfied-since hit to avoid a stale attack against us
2731 if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
2732 {
2733 // copy FinalFile into partial/ so that we check the hash again
2734 string const FinalFile = GetExistingFilename(GetFinalFileNameFromURI(Target.URI));
2735 if (symlink(FinalFile.c_str(), DestFile.c_str()) != 0)
2736 _error->WarningE("pkgAcqIndex::StageDownloadDone", "Symlinking final file %s back to %s failed", FinalFile.c_str(), DestFile.c_str());
2737 else
2738 {
2739 EraseFileName = DestFile;
2740 Filename = DestFile;
2741 }
2742 Stage = STAGE_DECOMPRESS_AND_VERIFY;
2743 Desc.URI = "store:" + Filename;
2744 QueueURI(Desc);
2745 SetActiveSubprocess(::URI(Desc.URI).Access);
2746 return;
2747 }
2748 // methods like file:// give us an alternative (uncompressed) file
2749 else if (Target.KeepCompressed == false && AltFilename.empty() == false)
2750 {
2751 Filename = AltFilename;
2752 EraseFileName.clear();
2753 }
2754 // Methods like e.g. "file:" will give us a (compressed) FileName that is
2755 // not the "DestFile" we set, in this case we uncompress from the local file
2756 else if (Filename != DestFile && RealFileExists(DestFile) == false)
2757 {
2758 // symlinking ensures that the filename can be used for compression detection
2759 // that is e.g. needed for by-hash which has no extension over file
2760 if (symlink(Filename.c_str(),DestFile.c_str()) != 0)
2761 _error->WarningE("pkgAcqIndex::StageDownloadDone", "Symlinking file %s to %s failed", Filename.c_str(), DestFile.c_str());
2762 else
2763 {
2764 EraseFileName = DestFile;
2765 Filename = DestFile;
2766 }
2767 }
2768
2769 Stage = STAGE_DECOMPRESS_AND_VERIFY;
2770 DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target);
2771 if (Filename != DestFile && flExtension(Filename) == flExtension(DestFile))
2772 Desc.URI = "copy:" + Filename;
2773 else
2774 Desc.URI = "store:" + Filename;
2775 if (DestFile == Filename)
2776 {
2777 if (CurrentCompressionExtension == "uncompressed")
2778 return StageDecompressDone();
2779 DestFile = "/dev/null";
2780 }
2781
2782 if (EraseFileName.empty() && Filename != AltFilename)
2783 EraseFileName = Filename;
2784
2785 // queue uri for the next stage
2786 QueueURI(Desc);
2787 SetActiveSubprocess(::URI(Desc.URI).Access);
2788 }
2789 /*}}}*/
2790 // AcqIndex::StageDecompressDone - Final verification /*{{{*/
2791 void pkgAcqIndex::StageDecompressDone()
2792 {
2793 if (DestFile == "/dev/null")
2794 DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target);
2795
2796 // Done, queue for rename on transaction finished
2797 TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename());
2798 }
2799 /*}}}*/
2800 pkgAcqIndex::~pkgAcqIndex() {}
2801
2802
2803 // AcqArchive::AcqArchive - Constructor /*{{{*/
2804 // ---------------------------------------------------------------------
2805 /* This just sets up the initial fetch environment and queues the first
2806 possibilitiy */
2807 pkgAcqArchive::pkgAcqArchive(pkgAcquire * const Owner,pkgSourceList * const Sources,
2808 pkgRecords * const Recs,pkgCache::VerIterator const &Version,
2809 string &StoreFilename) :
2810 Item(Owner), d(NULL), LocalSource(false), Version(Version), Sources(Sources), Recs(Recs),
2811 StoreFilename(StoreFilename), Vf(Version.FileList()),
2812 Trusted(false)
2813 {
2814 Retries = _config->FindI("Acquire::Retries",0);
2815
2816 if (Version.Arch() == 0)
2817 {
2818 _error->Error(_("I wasn't able to locate a file for the %s package. "
2819 "This might mean you need to manually fix this package. "
2820 "(due to missing arch)"),
2821 Version.ParentPkg().FullName().c_str());
2822 return;
2823 }
2824
2825 /* We need to find a filename to determine the extension. We make the
2826 assumption here that all the available sources for this version share
2827 the same extension.. */
2828 // Skip not source sources, they do not have file fields.
2829 for (; Vf.end() == false; ++Vf)
2830 {
2831 if (Vf.File().Flagged(pkgCache::Flag::NotSource))
2832 continue;
2833 break;
2834 }
2835
2836 // Does not really matter here.. we are going to fail out below
2837 if (Vf.end() != true)
2838 {
2839 // If this fails to get a file name we will bomb out below.
2840 pkgRecords::Parser &Parse = Recs->Lookup(Vf);
2841 if (_error->PendingError() == true)
2842 return;
2843
2844 // Generate the final file name as: package_version_arch.foo
2845 StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
2846 QuoteString(Version.VerStr(),"_:") + '_' +
2847 QuoteString(Version.Arch(),"_:.") +
2848 "." + flExtension(Parse.FileName());
2849 }
2850
2851 // check if we have one trusted source for the package. if so, switch
2852 // to "TrustedOnly" mode - but only if not in AllowUnauthenticated mode
2853 bool const allowUnauth = _config->FindB("APT::Get::AllowUnauthenticated", false);
2854 bool const debugAuth = _config->FindB("Debug::pkgAcquire::Auth", false);
2855 bool seenUntrusted = false;
2856 for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; ++i)
2857 {
2858 pkgIndexFile *Index;
2859 if (Sources->FindIndex(i.File(),Index) == false)
2860 continue;
2861
2862 if (debugAuth == true)
2863 std::cerr << "Checking index: " << Index->Describe()
2864 << "(Trusted=" << Index->IsTrusted() << ")" << std::endl;
2865
2866 if (Index->IsTrusted() == true)
2867 {
2868 Trusted = true;
2869 if (allowUnauth == false)
2870 break;
2871 }
2872 else
2873 seenUntrusted = true;
2874 }
2875
2876 // "allow-unauthenticated" restores apts old fetching behaviour
2877 // that means that e.g. unauthenticated file:// uris are higher
2878 // priority than authenticated http:// uris
2879 if (allowUnauth == true && seenUntrusted == true)
2880 Trusted = false;
2881
2882 // Select a source
2883 if (QueueNext() == false && _error->PendingError() == false)
2884 _error->Error(_("Can't find a source to download version '%s' of '%s'"),
2885 Version.VerStr(), Version.ParentPkg().FullName(false).c_str());
2886 }
2887 /*}}}*/
2888 // AcqArchive::QueueNext - Queue the next file source /*{{{*/
2889 // ---------------------------------------------------------------------
2890 /* This queues the next available file version for download. It checks if
2891 the archive is already available in the cache and stashs the MD5 for
2892 checking later. */
2893 bool pkgAcqArchive::QueueNext()
2894 {
2895 for (; Vf.end() == false; ++Vf)
2896 {
2897 pkgCache::PkgFileIterator const PkgF = Vf.File();
2898 // Ignore not source sources
2899 if (PkgF.Flagged(pkgCache::Flag::NotSource))
2900 continue;
2901
2902 // Try to cross match against the source list
2903 pkgIndexFile *Index;
2904 if (Sources->FindIndex(PkgF, Index) == false)
2905 continue;
2906 LocalSource = PkgF.Flagged(pkgCache::Flag::LocalSource);
2907
2908 // only try to get a trusted package from another source if that source
2909 // is also trusted
2910 if(Trusted && !Index->IsTrusted())
2911 continue;
2912
2913 // Grab the text package record
2914 pkgRecords::Parser &Parse = Recs->Lookup(Vf);
2915 if (_error->PendingError() == true)
2916 return false;
2917
2918 string PkgFile = Parse.FileName();
2919 ExpectedHashes = Parse.Hashes();
2920
2921 if (PkgFile.empty() == true)
2922 return _error->Error(_("The package index files are corrupted. No Filename: "
2923 "field for package %s."),
2924 Version.ParentPkg().Name());
2925
2926 Desc.URI = Index->ArchiveURI(PkgFile);
2927 Desc.Description = Index->ArchiveInfo(Version);
2928 Desc.Owner = this;
2929 Desc.ShortDesc = Version.ParentPkg().FullName(true);
2930
2931 // See if we already have the file. (Legacy filenames)
2932 FileSize = Version->Size;
2933 string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
2934 struct stat Buf;
2935 if (stat(FinalFile.c_str(),&Buf) == 0)
2936 {
2937 // Make sure the size matches
2938 if ((unsigned long long)Buf.st_size == Version->Size)
2939 {
2940 Complete = true;
2941 Local = true;
2942 Status = StatDone;
2943 StoreFilename = DestFile = FinalFile;
2944 return true;
2945 }
2946
2947 /* Hmm, we have a file and its size does not match, this means it is
2948 an old style mismatched arch */
2949 RemoveFile("pkgAcqArchive::QueueNext", FinalFile);
2950 }
2951
2952 // Check it again using the new style output filenames
2953 FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
2954 if (stat(FinalFile.c_str(),&Buf) == 0)
2955 {
2956 // Make sure the size matches
2957 if ((unsigned long long)Buf.st_size == Version->Size)
2958 {
2959 Complete = true;
2960 Local = true;
2961 Status = StatDone;
2962 StoreFilename = DestFile = FinalFile;
2963 return true;
2964 }
2965
2966 /* Hmm, we have a file and its size does not match, this shouldn't
2967 happen.. */
2968 RemoveFile("pkgAcqArchive::QueueNext", FinalFile);
2969 }
2970
2971 DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
2972
2973 // Check the destination file
2974 if (stat(DestFile.c_str(),&Buf) == 0)
2975 {
2976 // Hmm, the partial file is too big, erase it
2977 if ((unsigned long long)Buf.st_size > Version->Size)
2978 RemoveFile("pkgAcqArchive::QueueNext", DestFile);
2979 else
2980 PartialSize = Buf.st_size;
2981 }
2982
2983 // Disables download of archives - useful if no real installation follows,
2984 // e.g. if we are just interested in proposed installation order
2985 if (_config->FindB("Debug::pkgAcqArchive::NoQueue", false) == true)
2986 {
2987 Complete = true;
2988 Local = true;
2989 Status = StatDone;
2990 StoreFilename = DestFile = FinalFile;
2991 return true;
2992 }
2993
2994 // Create the item
2995 Local = false;
2996 QueueURI(Desc);
2997
2998 ++Vf;
2999 return true;
3000 }
3001 return false;
3002 }
3003 /*}}}*/
3004 // AcqArchive::Done - Finished fetching /*{{{*/
3005 // ---------------------------------------------------------------------
3006 /* */
3007 void pkgAcqArchive::Done(string const &Message, HashStringList const &Hashes,
3008 pkgAcquire::MethodConfig const * const Cfg)
3009 {
3010 Item::Done(Message, Hashes, Cfg);
3011
3012 // Grab the output filename
3013 std::string const FileName = LookupTag(Message,"Filename");
3014 if (DestFile != FileName && RealFileExists(DestFile) == false)
3015 {
3016 StoreFilename = DestFile = FileName;
3017 Local = true;
3018 Complete = true;
3019 return;
3020 }
3021
3022 // Done, move it into position
3023 string const FinalFile = GetFinalFilename();
3024 Rename(DestFile,FinalFile);
3025 StoreFilename = DestFile = FinalFile;
3026 Complete = true;
3027 }
3028 /*}}}*/
3029 // AcqArchive::Failed - Failure handler /*{{{*/
3030 // ---------------------------------------------------------------------
3031 /* Here we try other sources */
3032 void pkgAcqArchive::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)
3033 {
3034 Item::Failed(Message,Cnf);
3035
3036 /* We don't really want to retry on failed media swaps, this prevents
3037 that. An interesting observation is that permanent failures are not
3038 recorded. */
3039 if (Cnf->Removable == true &&
3040 StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
3041 {
3042 // Vf = Version.FileList();
3043 while (Vf.end() == false) ++Vf;
3044 StoreFilename = string();
3045 return;
3046 }
3047
3048 Status = StatIdle;
3049 if (QueueNext() == false)
3050 {
3051 // This is the retry counter
3052 if (Retries != 0 &&
3053 Cnf->LocalOnly == false &&
3054 StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
3055 {
3056 Retries--;
3057 Vf = Version.FileList();
3058 if (QueueNext() == true)
3059 return;
3060 }
3061
3062 StoreFilename = string();
3063 Status = StatError;
3064 }
3065 }
3066 /*}}}*/
3067 APT_PURE bool pkgAcqArchive::IsTrusted() const /*{{{*/
3068 {
3069 return Trusted;
3070 }
3071 /*}}}*/
3072 void pkgAcqArchive::Finished() /*{{{*/
3073 {
3074 if (Status == pkgAcquire::Item::StatDone &&
3075 Complete == true)
3076 return;
3077 StoreFilename = string();
3078 }
3079 /*}}}*/
3080 std::string pkgAcqArchive::DescURI() const /*{{{*/
3081 {
3082 return Desc.URI;
3083 }
3084 /*}}}*/
3085 std::string pkgAcqArchive::ShortDesc() const /*{{{*/
3086 {
3087 return Desc.ShortDesc;
3088 }
3089 /*}}}*/
3090 pkgAcqArchive::~pkgAcqArchive() {}
3091
3092 // AcqChangelog::pkgAcqChangelog - Constructors /*{{{*/
3093 class pkgAcqChangelog::Private
3094 {
3095 public:
3096 std::string FinalFile;
3097 };
3098 pkgAcqChangelog::pkgAcqChangelog(pkgAcquire * const Owner, pkgCache::VerIterator const &Ver,
3099 std::string const &DestDir, std::string const &DestFilename) :
3100 pkgAcquire::Item(Owner), d(new pkgAcqChangelog::Private()), SrcName(Ver.SourcePkgName()), SrcVersion(Ver.SourceVerStr())
3101 {
3102 Desc.URI = URI(Ver);
3103 Init(DestDir, DestFilename);
3104 }
3105 // some parameters are char* here as they come likely from char* interfaces – which can also return NULL
3106 pkgAcqChangelog::pkgAcqChangelog(pkgAcquire * const Owner, pkgCache::RlsFileIterator const &RlsFile,
3107 char const * const Component, char const * const SrcName, char const * const SrcVersion,
3108 const string &DestDir, const string &DestFilename) :
3109 pkgAcquire::Item(Owner), d(new pkgAcqChangelog::Private()), SrcName(SrcName), SrcVersion(SrcVersion)
3110 {
3111 Desc.URI = URI(RlsFile, Component, SrcName, SrcVersion);
3112 Init(DestDir, DestFilename);
3113 }
3114 pkgAcqChangelog::pkgAcqChangelog(pkgAcquire * const Owner,
3115 std::string const &URI, char const * const SrcName, char const * const SrcVersion,
3116 const string &DestDir, const string &DestFilename) :
3117 pkgAcquire::Item(Owner), d(new pkgAcqChangelog::Private()), SrcName(SrcName), SrcVersion(SrcVersion)
3118 {
3119 Desc.URI = URI;
3120 Init(DestDir, DestFilename);
3121 }
3122 void pkgAcqChangelog::Init(std::string const &DestDir, std::string const &DestFilename)
3123 {
3124 if (Desc.URI.empty())
3125 {
3126 Status = StatError;
3127 // TRANSLATOR: %s=%s is sourcename=sourceversion, e.g. apt=1.1
3128 strprintf(ErrorText, _("Changelog unavailable for %s=%s"), SrcName.c_str(), SrcVersion.c_str());
3129 // Let the error message print something sensible rather than "Failed to fetch /"
3130 if (DestFilename.empty())
3131 DestFile = SrcName + ".changelog";
3132 else
3133 DestFile = DestFilename;
3134 Desc.URI = "changelog:/" + DestFile;
3135 return;
3136 }
3137
3138 std::string DestFileName;
3139 if (DestFilename.empty())
3140 DestFileName = flCombine(DestFile, SrcName + ".changelog");
3141 else
3142 DestFileName = flCombine(DestFile, DestFilename);
3143
3144 std::string const SandboxUser = _config->Find("APT::Sandbox::User");
3145 std::string const systemTemp = GetTempDir(SandboxUser);
3146 char tmpname[1000];
3147 snprintf(tmpname, sizeof(tmpname), "%s/apt-changelog-XXXXXX", systemTemp.c_str());
3148 if (NULL == mkdtemp(tmpname))
3149 {
3150 _error->Errno("mkdtemp", "mkdtemp failed in changelog acquire of %s %s", SrcName.c_str(), SrcVersion.c_str());
3151 Status = StatError;
3152 return;
3153 }
3154 TemporaryDirectory = tmpname;
3155
3156 ChangeOwnerAndPermissionOfFile("Item::QueueURI", TemporaryDirectory.c_str(),
3157 SandboxUser.c_str(), "root", 0700);
3158
3159 DestFile = flCombine(TemporaryDirectory, DestFileName);
3160 if (DestDir.empty() == false)
3161 {
3162 d->FinalFile = flCombine(DestDir, DestFileName);
3163 if (RealFileExists(d->FinalFile))
3164 {
3165 FileFd file1, file2;
3166 if (file1.Open(DestFile, FileFd::WriteOnly | FileFd::Create | FileFd::Exclusive) &&
3167 file2.Open(d->FinalFile, FileFd::ReadOnly) && CopyFile(file2, file1))
3168 {
3169 struct timeval times[2];
3170 times[0].tv_sec = times[1].tv_sec = file2.ModificationTime();
3171 times[0].tv_usec = times[1].tv_usec = 0;
3172 utimes(DestFile.c_str(), times);
3173 }
3174 }
3175 }
3176
3177 Desc.ShortDesc = "Changelog";
3178 strprintf(Desc.Description, "%s %s %s Changelog", URI::SiteOnly(Desc.URI).c_str(), SrcName.c_str(), SrcVersion.c_str());
3179 Desc.Owner = this;
3180 QueueURI(Desc);
3181 }
3182 /*}}}*/
3183 std::string pkgAcqChangelog::URI(pkgCache::VerIterator const &Ver) /*{{{*/
3184 {
3185 std::string const confOnline = "Acquire::Changelogs::AlwaysOnline";
3186 bool AlwaysOnline = _config->FindB(confOnline, false);
3187 if (AlwaysOnline == false)
3188 for (pkgCache::VerFileIterator VF = Ver.FileList(); VF.end() == false; ++VF)
3189 {
3190 pkgCache::PkgFileIterator const PF = VF.File();
3191 if (PF.Flagged(pkgCache::Flag::NotSource) || PF->Release == 0)
3192 continue;
3193 pkgCache::RlsFileIterator const RF = PF.ReleaseFile();
3194 if (RF->Origin != 0 && _config->FindB(confOnline + "::Origin::" + RF.Origin(), false))
3195 {
3196 AlwaysOnline = true;
3197 break;
3198 }
3199 }
3200 if (AlwaysOnline == false)
3201 {
3202 pkgCache::PkgIterator const Pkg = Ver.ParentPkg();
3203 if (Pkg->CurrentVer != 0 && Pkg.CurrentVer() == Ver)
3204 {
3205 std::string const basename = std::string("/usr/share/doc/") + Pkg.Name() + "/changelog";
3206 std::string const debianname = basename + ".Debian";
3207 if (FileExists(debianname))
3208 return "copy://" + debianname;
3209 else if (FileExists(debianname + ".gz"))
3210 return "gzip://" + debianname + ".gz";
3211 else if (FileExists(basename))
3212 return "copy://" + basename;
3213 else if (FileExists(basename + ".gz"))
3214 return "gzip://" + basename + ".gz";
3215 }
3216 }
3217
3218 char const * const SrcName = Ver.SourcePkgName();
3219 char const * const SrcVersion = Ver.SourceVerStr();
3220 // find the first source for this version which promises a changelog
3221 for (pkgCache::VerFileIterator VF = Ver.FileList(); VF.end() == false; ++VF)
3222 {
3223 pkgCache::PkgFileIterator const PF = VF.File();
3224 if (PF.Flagged(pkgCache::Flag::NotSource) || PF->Release == 0)
3225 continue;
3226 pkgCache::RlsFileIterator const RF = PF.ReleaseFile();
3227 std::string const uri = URI(RF, PF.Component(), SrcName, SrcVersion);
3228 if (uri.empty())
3229 continue;
3230 return uri;
3231 }
3232 return "";
3233 }
3234 std::string pkgAcqChangelog::URITemplate(pkgCache::RlsFileIterator const &Rls)
3235 {
3236 if (Rls.end() == true || (Rls->Label == 0 && Rls->Origin == 0))
3237 return "";
3238 std::string const serverConfig = "Acquire::Changelogs::URI";
3239 std::string server;
3240 #define APT_EMPTY_SERVER \
3241 if (server.empty() == false) \
3242 { \
3243 if (server != "no") \
3244 return server; \
3245 return ""; \
3246 }
3247 #define APT_CHECK_SERVER(X, Y) \
3248 if (Rls->X != 0) \
3249 { \
3250 std::string const specialServerConfig = serverConfig + "::" + Y + #X + "::" + Rls.X(); \
3251 server = _config->Find(specialServerConfig); \
3252 APT_EMPTY_SERVER \
3253 }
3254 // this way e.g. Debian-Security can fallback to Debian
3255 APT_CHECK_SERVER(Label, "Override::")
3256 APT_CHECK_SERVER(Origin, "Override::")
3257
3258 if (RealFileExists(Rls.FileName()))
3259 {
3260 _error->PushToStack();
3261 FileFd rf;
3262 /* This can be costly. A caller wanting to get millions of URIs might
3263 want to do this on its own once and use Override settings.
3264 We don't do this here as Origin/Label are not as unique as they
3265 should be so this could produce request order-dependent anomalies */
3266 if (OpenMaybeClearSignedFile(Rls.FileName(), rf) == true)
3267 {
3268 pkgTagFile TagFile(&rf, rf.Size());
3269 pkgTagSection Section;
3270 if (TagFile.Step(Section) == true)
3271 server = Section.FindS("Changelogs");
3272 }
3273 _error->RevertToStack();
3274 APT_EMPTY_SERVER
3275 }
3276
3277 APT_CHECK_SERVER(Label, "")
3278 APT_CHECK_SERVER(Origin, "")
3279 #undef APT_CHECK_SERVER
3280 #undef APT_EMPTY_SERVER
3281 return "";
3282 }
3283 std::string pkgAcqChangelog::URI(pkgCache::RlsFileIterator const &Rls,
3284 char const * const Component, char const * const SrcName,
3285 char const * const SrcVersion)
3286 {
3287 return URI(URITemplate(Rls), Component, SrcName, SrcVersion);
3288 }
3289 std::string pkgAcqChangelog::URI(std::string const &Template,
3290 char const * const Component, char const * const SrcName,
3291 char const * const SrcVersion)
3292 {
3293 if (Template.find("@CHANGEPATH@") == std::string::npos)
3294 return "";
3295
3296 // the path is: COMPONENT/SRC/SRCNAME/SRCNAME_SRCVER, e.g. main/a/apt/1.1 or contrib/liba/libapt/2.0
3297 std::string Src = SrcName;
3298 std::string path = APT::String::Startswith(SrcName, "lib") ? Src.substr(0, 4) : Src.substr(0,1);
3299 path.append("/").append(Src).append("/");
3300 path.append(Src).append("_").append(StripEpoch(SrcVersion));
3301 // we omit component for releases without one (= flat-style repositories)
3302 if (Component != NULL && strlen(Component) != 0)
3303 path = std::string(Component) + "/" + path;
3304
3305 return SubstVar(Template, "@CHANGEPATH@", path);
3306 }
3307 /*}}}*/
3308 // AcqChangelog::Failed - Failure handler /*{{{*/
3309 void pkgAcqChangelog::Failed(string const &Message, pkgAcquire::MethodConfig const * const Cnf)
3310 {
3311 Item::Failed(Message,Cnf);
3312
3313 std::string errText;
3314 // TRANSLATOR: %s=%s is sourcename=sourceversion, e.g. apt=1.1
3315 strprintf(errText, _("Changelog unavailable for %s=%s"), SrcName.c_str(), SrcVersion.c_str());
3316
3317 // Error is probably something techy like 404 Not Found
3318 if (ErrorText.empty())
3319 ErrorText = errText;
3320 else
3321 ErrorText = errText + " (" + ErrorText + ")";
3322 }
3323 /*}}}*/
3324 // AcqChangelog::Done - Item downloaded OK /*{{{*/
3325 void pkgAcqChangelog::Done(string const &Message,HashStringList const &CalcHashes,
3326 pkgAcquire::MethodConfig const * const Cnf)
3327 {
3328 Item::Done(Message,CalcHashes,Cnf);
3329 if (d->FinalFile.empty() == false)
3330 {
3331 if (RemoveFile("pkgAcqChangelog::Done", d->FinalFile) == false ||
3332 Rename(DestFile, d->FinalFile) == false)
3333 Status = StatError;
3334 }
3335
3336 Complete = true;
3337 }
3338 /*}}}*/
3339 pkgAcqChangelog::~pkgAcqChangelog() /*{{{*/
3340 {
3341 if (TemporaryDirectory.empty() == false)
3342 {
3343 RemoveFile("~pkgAcqChangelog", DestFile);
3344 rmdir(TemporaryDirectory.c_str());
3345 }
3346 delete d;
3347 }
3348 /*}}}*/
3349
3350 // AcqFile::pkgAcqFile - Constructor /*{{{*/
3351 pkgAcqFile::pkgAcqFile(pkgAcquire * const Owner,string const &URI, HashStringList const &Hashes,
3352 unsigned long long const Size,string const &Dsc,string const &ShortDesc,
3353 const string &DestDir, const string &DestFilename,
3354 bool const IsIndexFile) :
3355 Item(Owner), d(NULL), IsIndexFile(IsIndexFile), ExpectedHashes(Hashes)
3356 {
3357 Retries = _config->FindI("Acquire::Retries",0);
3358
3359 if(!DestFilename.empty())
3360 DestFile = DestFilename;
3361 else if(!DestDir.empty())
3362 DestFile = DestDir + "/" + flNotDir(URI);
3363 else
3364 DestFile = flNotDir(URI);
3365
3366 // Create the item
3367 Desc.URI = URI;
3368 Desc.Description = Dsc;
3369 Desc.Owner = this;
3370
3371 // Set the short description to the archive component
3372 Desc.ShortDesc = ShortDesc;
3373
3374 // Get the transfer sizes
3375 FileSize = Size;
3376 struct stat Buf;
3377 if (stat(DestFile.c_str(),&Buf) == 0)
3378 {
3379 // Hmm, the partial file is too big, erase it
3380 if ((Size > 0) && (unsigned long long)Buf.st_size > Size)
3381 RemoveFile("pkgAcqFile", DestFile);
3382 else
3383 PartialSize = Buf.st_size;
3384 }
3385
3386 QueueURI(Desc);
3387 }
3388 /*}}}*/
3389 // AcqFile::Done - Item downloaded OK /*{{{*/
3390 void pkgAcqFile::Done(string const &Message,HashStringList const &CalcHashes,
3391 pkgAcquire::MethodConfig const * const Cnf)
3392 {
3393 Item::Done(Message,CalcHashes,Cnf);
3394
3395 std::string const FileName = LookupTag(Message,"Filename");
3396 Complete = true;
3397
3398 // The files timestamp matches
3399 if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
3400 return;
3401
3402 // We have to copy it into place
3403 if (RealFileExists(DestFile.c_str()) == false)
3404 {
3405 Local = true;
3406 if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
3407 Cnf->Removable == true)
3408 {
3409 Desc.URI = "copy:" + FileName;
3410 QueueURI(Desc);
3411 return;
3412 }
3413
3414 // Erase the file if it is a symlink so we can overwrite it
3415 struct stat St;
3416 if (lstat(DestFile.c_str(),&St) == 0)
3417 {
3418 if (S_ISLNK(St.st_mode) != 0)
3419 RemoveFile("pkgAcqFile::Done", DestFile);
3420 }
3421
3422 // Symlink the file
3423 if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
3424 {
3425 _error->PushToStack();
3426 _error->Errno("pkgAcqFile::Done", "Symlinking file %s failed", DestFile.c_str());
3427 std::stringstream msg;
3428 _error->DumpErrors(msg, GlobalError::DEBUG, false);
3429 _error->RevertToStack();
3430 ErrorText = msg.str();
3431 Status = StatError;
3432 Complete = false;
3433 }
3434 }
3435 }
3436 /*}}}*/
3437 // AcqFile::Failed - Failure handler /*{{{*/
3438 // ---------------------------------------------------------------------
3439 /* Here we try other sources */
3440 void pkgAcqFile::Failed(string const &Message, pkgAcquire::MethodConfig const * const Cnf)
3441 {
3442 Item::Failed(Message,Cnf);
3443
3444 // This is the retry counter
3445 if (Retries != 0 &&
3446 Cnf->LocalOnly == false &&
3447 StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
3448 {
3449 --Retries;
3450 QueueURI(Desc);
3451 Status = StatIdle;
3452 return;
3453 }
3454
3455 }
3456 /*}}}*/
3457 string pkgAcqFile::Custom600Headers() const /*{{{*/
3458 {
3459 if (IsIndexFile)
3460 return "\nIndex-File: true";
3461 return "";
3462 }
3463 /*}}}*/
3464 pkgAcqFile::~pkgAcqFile() {}