1 // -*- mode: cpp; mode: fold -*-
3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
4 /* ######################################################################
6 Acquire Item - Item to acquire
8 Each item can download to exactly one file at a time. This means you
9 cannot create an item that fetches two uri's to two files at the same
10 time. The pkgAcqIndex class creates a second class upon instantiation
11 to fetch the other index files because of this.
13 ##################################################################### */
15 // Include Files /*{{{*/
18 #include <apt-pkg/acquire-item.h>
19 #include <apt-pkg/configuration.h>
20 #include <apt-pkg/aptconfiguration.h>
21 #include <apt-pkg/sourcelist.h>
22 #include <apt-pkg/error.h>
23 #include <apt-pkg/strutl.h>
24 #include <apt-pkg/fileutl.h>
25 #include <apt-pkg/tagfile.h>
26 #include <apt-pkg/metaindex.h>
27 #include <apt-pkg/acquire.h>
28 #include <apt-pkg/hashes.h>
29 #include <apt-pkg/indexfile.h>
30 #include <apt-pkg/pkgcache.h>
31 #include <apt-pkg/cacheiterators.h>
32 #include <apt-pkg/pkgrecords.h>
33 #include <apt-pkg/gpgv.h>
54 static void printHashSumComparision(std::string
const &URI
, HashStringList
const &Expected
, HashStringList
const &Actual
) /*{{{*/
56 if (_config
->FindB("Debug::Acquire::HashSumMismatch", false) == false)
58 std::cerr
<< std::endl
<< URI
<< ":" << std::endl
<< " Expected Hash: " << std::endl
;
59 for (HashStringList::const_iterator hs
= Expected
.begin(); hs
!= Expected
.end(); ++hs
)
60 std::cerr
<< "\t- " << hs
->toStr() << std::endl
;
61 std::cerr
<< " Actual Hash: " << std::endl
;
62 for (HashStringList::const_iterator hs
= Actual
.begin(); hs
!= Actual
.end(); ++hs
)
63 std::cerr
<< "\t- " << hs
->toStr() << std::endl
;
66 static std::string
GetPartialFileName(std::string
const &file
) /*{{{*/
68 std::string DestFile
= _config
->FindDir("Dir::State::lists") + "partial/";
73 static std::string
GetPartialFileNameFromURI(std::string
const &uri
) /*{{{*/
75 return GetPartialFileName(URItoFileName(uri
));
78 static std::string
GetFinalFileNameFromURI(std::string
const &uri
) /*{{{*/
80 return _config
->FindDir("Dir::State::lists") + URItoFileName(uri
);
83 static std::string
GetKeepCompressedFileName(std::string file
, IndexTarget
const &Target
)/*{{{*/
85 if (Target
.KeepCompressed
== false)
88 std::string
const CompressionTypes
= Target
.Option(IndexTarget::COMPRESSIONTYPES
);
89 if (CompressionTypes
.empty() == false)
91 std::string
const ext
= CompressionTypes
.substr(0, CompressionTypes
.find(' '));
92 if (ext
!= "uncompressed")
93 file
.append(".").append(ext
);
98 static std::string
GetCompressedFileName(IndexTarget
const &Target
, std::string
const &Name
, std::string
const &Ext
) /*{{{*/
100 if (Ext
.empty() || Ext
== "uncompressed")
103 // do not reverify cdrom sources as apt-cdrom may rewrite the Packages
104 // file when its doing the indexcopy
105 if (Target
.URI
.substr(0,6) == "cdrom:")
108 // adjust DestFile if its compressed on disk
109 if (Target
.KeepCompressed
== true)
110 return Name
+ '.' + Ext
;
114 static std::string
GetMergeDiffsPatchFileName(std::string
const &Final
, std::string
const &Patch
)/*{{{*/
116 // rred expects the patch as $FinalFile.ed.$patchname.gz
117 return Final
+ ".ed." + Patch
+ ".gz";
120 static std::string
GetDiffsPatchFileName(std::string
const &Final
) /*{{{*/
122 // rred expects the patch as $FinalFile.ed
123 return Final
+ ".ed";
126 static std::string
GetExistingFilename(std::string
const &File
) /*{{{*/
128 if (RealFileExists(File
))
130 for (auto const &type
: APT::Configuration::getCompressorExtensions())
132 std::string
const Final
= File
+ type
;
133 if (RealFileExists(Final
))
140 static bool MessageInsecureRepository(bool const isError
, std::string
const &msg
)/*{{{*/
144 _error
->Error("%s", msg
.c_str());
145 _error
->Notice("%s", _("Updating from such a repository can't be done securely, and is therefore disabled by default."));
149 _error
->Warning("%s", msg
.c_str());
150 _error
->Notice("%s", _("Data from such a repository can't be authenticated and is therefore potentially dangerous to use."));
152 _error
->Notice("%s", _("See apt-secure(8) manpage for repository creation and user configuration details."));
155 static bool MessageInsecureRepository(bool const isError
, char const * const msg
, std::string
const &repo
)
158 strprintf(m
, msg
, repo
.c_str());
159 return MessageInsecureRepository(isError
, m
);
162 static bool AllowInsecureRepositories(char const * const msg
, std::string
const &repo
,/*{{{*/
163 metaIndex
const * const MetaIndexParser
, pkgAcqMetaClearSig
* const TransactionManager
, pkgAcquire::Item
* const I
)
165 if(MetaIndexParser
->GetTrusted() == metaIndex::TRI_YES
)
168 if (_config
->FindB("Acquire::AllowInsecureRepositories") == true)
170 MessageInsecureRepository(false, msg
, repo
);
174 MessageInsecureRepository(true, msg
, repo
);
175 TransactionManager
->AbortTransaction();
176 I
->Status
= pkgAcquire::Item::StatError
;
180 static HashStringList
GetExpectedHashesFromFor(metaIndex
* const Parser
, std::string
const &MetaKey
)/*{{{*/
183 return HashStringList();
184 metaIndex::checkSum
* const R
= Parser
->Lookup(MetaKey
);
186 return HashStringList();
191 // all ::HashesRequired and ::GetExpectedHashes implementations /*{{{*/
192 /* ::GetExpectedHashes is abstract and has to be implemented by all subclasses.
193 It is best to implement it as broadly as possible, while ::HashesRequired defaults
194 to true and should be as restrictive as possible for false cases. Note that if
195 a hash is returned by ::GetExpectedHashes it must match. Only if it doesn't
196 ::HashesRequired is called to evaluate if its okay to have no hashes. */
197 APT_CONST
bool pkgAcqTransactionItem::HashesRequired() const
199 /* signed repositories obviously have a parser and good hashes.
200 unsigned repositories, too, as even if we can't trust them for security,
201 we can at least trust them for integrity of the download itself.
202 Only repositories without a Release file can (obviously) not have
203 hashes – and they are very uncommon and strongly discouraged */
204 return TransactionManager
->MetaIndexParser
!= NULL
&&
205 TransactionManager
->MetaIndexParser
->GetLoadedSuccessfully() == metaIndex::TRI_YES
;
207 HashStringList
pkgAcqTransactionItem::GetExpectedHashes() const
209 return GetExpectedHashesFor(GetMetaKey());
212 APT_CONST
bool pkgAcqMetaBase::HashesRequired() const
214 // Release and co have no hashes 'by design'.
217 HashStringList
pkgAcqMetaBase::GetExpectedHashes() const
219 return HashStringList();
222 APT_CONST
bool pkgAcqIndexDiffs::HashesRequired() const
224 /* We don't always have the diff of the downloaded pdiff file.
225 What we have for sure is hashes for the uncompressed file,
226 but rred uncompresses them on the fly while parsing, so not handled here.
227 Hashes are (also) checked while searching for (next) patch to apply. */
228 if (State
== StateFetchDiff
)
229 return available_patches
[0].download_hashes
.empty() == false;
232 HashStringList
pkgAcqIndexDiffs::GetExpectedHashes() const
234 if (State
== StateFetchDiff
)
235 return available_patches
[0].download_hashes
;
236 return HashStringList();
239 APT_CONST
bool pkgAcqIndexMergeDiffs::HashesRequired() const
241 /* @see #pkgAcqIndexDiffs::HashesRequired, with the difference that
242 we can check the rred result after all patches are applied as
243 we know the expected result rather than potentially apply more patches */
244 if (State
== StateFetchDiff
)
245 return patch
.download_hashes
.empty() == false;
246 return State
== StateApplyDiff
;
248 HashStringList
pkgAcqIndexMergeDiffs::GetExpectedHashes() const
250 if (State
== StateFetchDiff
)
251 return patch
.download_hashes
;
252 else if (State
== StateApplyDiff
)
253 return GetExpectedHashesFor(Target
.MetaKey
);
254 return HashStringList();
257 APT_CONST
bool pkgAcqArchive::HashesRequired() const
259 return LocalSource
== false;
261 HashStringList
pkgAcqArchive::GetExpectedHashes() const
263 // figured out while parsing the records
264 return ExpectedHashes
;
267 APT_CONST
bool pkgAcqFile::HashesRequired() const
269 // supplied as parameter at creation time, so the caller decides
270 return ExpectedHashes
.usable();
272 HashStringList
pkgAcqFile::GetExpectedHashes() const
274 return ExpectedHashes
;
277 // Acquire::Item::QueueURI and specialisations from child classes /*{{{*/
278 bool pkgAcquire::Item::QueueURI(pkgAcquire::ItemDesc
&Item
)
280 Owner
->Enqueue(Item
);
283 /* The idea here is that an item isn't queued if it exists on disk and the
284 transition manager was a hit as this means that the files it contains
285 the checksums for can't be updated either (or they are and we are asking
286 for a hashsum mismatch to happen which helps nobody) */
287 bool pkgAcqTransactionItem::QueueURI(pkgAcquire::ItemDesc
&Item
)
289 std::string
const FinalFile
= GetFinalFilename();
290 if (TransactionManager
!= NULL
&& TransactionManager
->IMSHit
== true &&
291 FileExists(FinalFile
) == true)
293 PartialFile
= DestFile
= FinalFile
;
297 return pkgAcquire::Item::QueueURI(Item
);
299 /* The transition manager InRelease itself (or its older sisters-in-law
300 Release & Release.gpg) is always queued as this allows us to rerun gpgv
301 on it to verify that we aren't stalled with old files */
302 bool pkgAcqMetaBase::QueueURI(pkgAcquire::ItemDesc
&Item
)
304 return pkgAcquire::Item::QueueURI(Item
);
306 /* the Diff/Index needs to queue also the up-to-date complete index file
307 to ensure that the list cleaner isn't eating it */
308 bool pkgAcqDiffIndex::QueueURI(pkgAcquire::ItemDesc
&Item
)
310 if (pkgAcqTransactionItem::QueueURI(Item
) == true)
316 // Acquire::Item::GetFinalFilename and specialisations for child classes /*{{{*/
317 std::string
pkgAcquire::Item::GetFinalFilename() const
319 return GetFinalFileNameFromURI(Desc
.URI
);
321 std::string
pkgAcqDiffIndex::GetFinalFilename() const
323 // the logic we inherent from pkgAcqBaseIndex isn't what we need here
324 return pkgAcquire::Item::GetFinalFilename();
326 std::string
pkgAcqIndex::GetFinalFilename() const
328 std::string
const FinalFile
= GetFinalFileNameFromURI(Target
.URI
);
329 return GetCompressedFileName(Target
, FinalFile
, CurrentCompressionExtension
);
331 std::string
pkgAcqMetaSig::GetFinalFilename() const
333 return GetFinalFileNameFromURI(Target
.URI
);
335 std::string
pkgAcqBaseIndex::GetFinalFilename() const
337 return GetFinalFileNameFromURI(Target
.URI
);
339 std::string
pkgAcqMetaBase::GetFinalFilename() const
341 return GetFinalFileNameFromURI(Target
.URI
);
343 std::string
pkgAcqArchive::GetFinalFilename() const
345 return _config
->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename
);
348 // pkgAcqTransactionItem::GetMetaKey and specialisations for child classes /*{{{*/
349 std::string
pkgAcqTransactionItem::GetMetaKey() const
351 return Target
.MetaKey
;
353 std::string
pkgAcqIndex::GetMetaKey() const
355 if (Stage
== STAGE_DECOMPRESS_AND_VERIFY
|| CurrentCompressionExtension
== "uncompressed")
356 return Target
.MetaKey
;
357 return Target
.MetaKey
+ "." + CurrentCompressionExtension
;
359 std::string
pkgAcqDiffIndex::GetMetaKey() const
361 return Target
.MetaKey
+ ".diff/Index";
364 //pkgAcqTransactionItem::TransactionState and specialisations for child classes /*{{{*/
365 bool pkgAcqTransactionItem::TransactionState(TransactionStates
const state
)
367 bool const Debug
= _config
->FindB("Debug::Acquire::Transaction", false);
370 case TransactionAbort
:
372 std::clog
<< " Cancel: " << DestFile
<< std::endl
;
373 if (Status
== pkgAcquire::Item::StatIdle
)
375 Status
= pkgAcquire::Item::StatDone
;
379 case TransactionCommit
:
380 if(PartialFile
.empty() == false)
382 if (PartialFile
!= DestFile
)
384 // ensure that even without lists-cleanup all compressions are nuked
385 std::string FinalFile
= GetFinalFileNameFromURI(Target
.URI
);
386 if (FileExists(FinalFile
))
389 std::clog
<< "rm " << FinalFile
<< " # " << DescURI() << std::endl
;
390 if (RemoveFile("TransactionStates-Cleanup", FinalFile
) == false)
393 for (auto const &ext
: APT::Configuration::getCompressorExtensions())
395 auto const Final
= FinalFile
+ ext
;
396 if (FileExists(Final
))
399 std::clog
<< "rm " << Final
<< " # " << DescURI() << std::endl
;
400 if (RemoveFile("TransactionStates-Cleanup", Final
) == false)
405 std::clog
<< "mv " << PartialFile
<< " -> "<< DestFile
<< " # " << DescURI() << std::endl
;
406 if (Rename(PartialFile
, DestFile
) == false)
409 else if(Debug
== true)
410 std::clog
<< "keep " << PartialFile
<< " # " << DescURI() << std::endl
;
414 std::clog
<< "rm " << DestFile
<< " # " << DescURI() << std::endl
;
415 if (RemoveFile("TransactionCommit", DestFile
) == false)
422 bool pkgAcqMetaBase::TransactionState(TransactionStates
const state
)
424 // Do not remove InRelease on IMSHit of Release.gpg [yes, this is very edgecasey]
425 if (TransactionManager
->IMSHit
== false)
426 return pkgAcqTransactionItem::TransactionState(state
);
429 bool pkgAcqIndex::TransactionState(TransactionStates
const state
)
431 if (pkgAcqTransactionItem::TransactionState(state
) == false)
436 case TransactionAbort
:
437 if (Stage
== STAGE_DECOMPRESS_AND_VERIFY
)
439 // keep the compressed file, but drop the decompressed
440 EraseFileName
.clear();
441 if (PartialFile
.empty() == false && flExtension(PartialFile
) != CurrentCompressionExtension
)
442 RemoveFile("TransactionAbort", PartialFile
);
445 case TransactionCommit
:
446 if (EraseFileName
.empty() == false)
447 RemoveFile("TransactionCommit", EraseFileName
);
452 bool pkgAcqDiffIndex::TransactionState(TransactionStates
const state
)
454 if (pkgAcqTransactionItem::TransactionState(state
) == false)
459 case TransactionCommit
:
461 case TransactionAbort
:
462 std::string
const Partial
= GetPartialFileNameFromURI(Target
.URI
);
463 RemoveFile("TransactionAbort", Partial
);
471 class APT_HIDDEN NoActionItem
: public pkgAcquire::Item
/*{{{*/
472 /* The sole purpose of this class is having an item which does nothing to
473 reach its done state to prevent cleanup deleting the mentioned file.
474 Handy in cases in which we know we have the file already, like IMS-Hits. */
476 IndexTarget
const Target
;
478 virtual std::string
DescURI() const APT_OVERRIDE
{return Target
.URI
;};
479 virtual HashStringList
GetExpectedHashes() const APT_OVERRIDE
{return HashStringList();};
481 NoActionItem(pkgAcquire
* const Owner
, IndexTarget
const &Target
) :
482 pkgAcquire::Item(Owner
), Target(Target
)
485 DestFile
= GetFinalFileNameFromURI(Target
.URI
);
487 NoActionItem(pkgAcquire
* const Owner
, IndexTarget
const &Target
, std::string
const &FinalFile
) :
488 pkgAcquire::Item(Owner
), Target(Target
)
491 DestFile
= FinalFile
;
496 // Acquire::Item::Item - Constructor /*{{{*/
497 APT_IGNORE_DEPRECATED_PUSH
498 pkgAcquire::Item::Item(pkgAcquire
* const owner
) :
499 FileSize(0), PartialSize(0), Mode(0), ID(0), Complete(false), Local(false),
500 QueueCounter(0), ExpectedAdditionalItems(0), Owner(owner
), d(NULL
)
505 APT_IGNORE_DEPRECATED_POP
507 // Acquire::Item::~Item - Destructor /*{{{*/
508 pkgAcquire::Item::~Item()
513 std::string
pkgAcquire::Item::Custom600Headers() const /*{{{*/
515 return std::string();
518 std::string
pkgAcquire::Item::ShortDesc() const /*{{{*/
523 APT_CONST
void pkgAcquire::Item::Finished() /*{{{*/
527 APT_PURE pkgAcquire
* pkgAcquire::Item::GetOwner() const /*{{{*/
532 APT_CONST
pkgAcquire::ItemDesc
&pkgAcquire::Item::GetItemDesc() /*{{{*/
537 APT_CONST
bool pkgAcquire::Item::IsTrusted() const /*{{{*/
542 // Acquire::Item::Failed - Item failed to download /*{{{*/
543 // ---------------------------------------------------------------------
544 /* We return to an idle state if there are still other queues that could
546 void pkgAcquire::Item::Failed(string
const &Message
,pkgAcquire::MethodConfig
const * const Cnf
)
548 if(ErrorText
.empty())
549 ErrorText
= LookupTag(Message
,"Message");
550 if (QueueCounter
<= 1)
552 /* This indicates that the file is not available right now but might
553 be sometime later. If we do a retry cycle then this should be
555 if (Cnf
!= NULL
&& Cnf
->LocalOnly
== true &&
556 StringToBool(LookupTag(Message
,"Transient-Failure"),false) == true)
572 case StatTransientNetworkError
:
579 string
const FailReason
= LookupTag(Message
, "FailReason");
580 if (FailReason
== "MaximumSizeExceeded")
581 RenameOnError(MaximumSizeExceeded
);
582 else if (Status
== StatAuthError
)
583 RenameOnError(HashSumMismatch
);
585 // report mirror failure back to LP if we actually use a mirror
586 if (FailReason
.empty() == false)
587 ReportMirrorFailure(FailReason
);
589 ReportMirrorFailure(ErrorText
);
591 if (QueueCounter
> 1)
595 // Acquire::Item::Start - Item has begun to download /*{{{*/
596 // ---------------------------------------------------------------------
597 /* Stash status and the file size. Note that setting Complete means
598 sub-phases of the acquire process such as decompresion are operating */
599 void pkgAcquire::Item::Start(string
const &/*Message*/, unsigned long long const Size
)
601 Status
= StatFetching
;
603 if (FileSize
== 0 && Complete
== false)
607 // Acquire::Item::VerifyDone - check if Item was downloaded OK /*{{{*/
608 /* Note that hash-verification is 'hardcoded' in acquire-worker and has
609 * already passed if this method is called. */
610 bool pkgAcquire::Item::VerifyDone(std::string
const &Message
,
611 pkgAcquire::MethodConfig
const * const /*Cnf*/)
613 std::string
const FileName
= LookupTag(Message
,"Filename");
614 if (FileName
.empty() == true)
617 ErrorText
= "Method gave a blank filename";
624 // Acquire::Item::Done - Item downloaded OK /*{{{*/
625 void pkgAcquire::Item::Done(string
const &/*Message*/, HashStringList
const &Hashes
,
626 pkgAcquire::MethodConfig
const * const /*Cnf*/)
628 // We just downloaded something..
631 unsigned long long const downloadedSize
= Hashes
.FileSize();
632 if (downloadedSize
!= 0)
634 FileSize
= downloadedSize
;
638 ErrorText
= string();
639 Owner
->Dequeue(this);
642 // Acquire::Item::Rename - Rename a file /*{{{*/
643 // ---------------------------------------------------------------------
644 /* This helper function is used by a lot of item methods as their final
646 bool pkgAcquire::Item::Rename(string
const &From
,string
const &To
)
648 if (From
== To
|| rename(From
.c_str(),To
.c_str()) == 0)
652 strprintf(S
, _("rename failed, %s (%s -> %s)."), strerror(errno
),
653 From
.c_str(),To
.c_str());
655 if (ErrorText
.empty())
658 ErrorText
= ErrorText
+ ": " + S
;
662 void pkgAcquire::Item::Dequeue() /*{{{*/
664 Owner
->Dequeue(this);
667 bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState
const error
)/*{{{*/
669 if (RealFileExists(DestFile
))
670 Rename(DestFile
, DestFile
+ ".FAILED");
675 case HashSumMismatch
:
676 errtext
= _("Hash Sum mismatch");
677 Status
= StatAuthError
;
678 ReportMirrorFailure("HashChecksumFailure");
681 errtext
= _("Size mismatch");
682 Status
= StatAuthError
;
683 ReportMirrorFailure("SizeFailure");
686 errtext
= _("Invalid file format");
688 // do not report as usually its not the mirrors fault, but Portal/Proxy
691 errtext
= _("Signature error");
695 strprintf(errtext
, _("Clearsigned file isn't valid, got '%s' (does the network require authentication?)"), "NOSPLIT");
696 Status
= StatAuthError
;
698 case MaximumSizeExceeded
:
699 // the method is expected to report a good error for this
703 // no handling here, done by callers
706 if (ErrorText
.empty())
711 void pkgAcquire::Item::SetActiveSubprocess(const std::string
&subprocess
)/*{{{*/
713 ActiveSubprocess
= subprocess
;
714 APT_IGNORE_DEPRECATED(Mode
= ActiveSubprocess
.c_str();)
717 // Acquire::Item::ReportMirrorFailure /*{{{*/
718 void pkgAcquire::Item::ReportMirrorFailure(string
const &FailCode
)
720 // we only act if a mirror was used at all
721 if(UsedMirror
.empty())
724 std::cerr
<< "\nReportMirrorFailure: "
726 << " Uri: " << DescURI()
728 << FailCode
<< std::endl
;
730 string report
= _config
->Find("Methods::Mirror::ProblemReporting",
731 "/usr/lib/apt/apt-report-mirror-failure");
732 if(!FileExists(report
))
735 std::vector
<char const*> Args
;
736 Args
.push_back(report
.c_str());
737 Args
.push_back(UsedMirror
.c_str());
738 Args
.push_back(DescURI().c_str());
739 Args
.push_back(FailCode
.c_str());
740 Args
.push_back(NULL
);
742 pid_t pid
= ExecFork();
745 _error
->Error("ReportMirrorFailure Fork failed");
750 execvp(Args
[0], (char**)Args
.data());
751 std::cerr
<< "Could not exec " << Args
[0] << std::endl
;
754 if(!ExecWait(pid
, "report-mirror-failure"))
756 _error
->Warning("Couldn't report problem to '%s'",
757 _config
->Find("Methods::Mirror::ProblemReporting").c_str());
761 std::string
pkgAcquire::Item::HashSum() const /*{{{*/
763 HashStringList
const hashes
= GetExpectedHashes();
764 HashString
const * const hs
= hashes
.find(NULL
);
765 return hs
!= NULL
? hs
->toStr() : "";
769 pkgAcqTransactionItem::pkgAcqTransactionItem(pkgAcquire
* const Owner
, /*{{{*/
770 pkgAcqMetaClearSig
* const transactionManager
, IndexTarget
const &target
) :
771 pkgAcquire::Item(Owner
), d(NULL
), Target(target
), TransactionManager(transactionManager
)
773 if (TransactionManager
!= this)
774 TransactionManager
->Add(this);
777 pkgAcqTransactionItem::~pkgAcqTransactionItem() /*{{{*/
781 HashStringList
pkgAcqTransactionItem::GetExpectedHashesFor(std::string
const &MetaKey
) const /*{{{*/
783 return GetExpectedHashesFromFor(TransactionManager
->MetaIndexParser
, MetaKey
);
787 // AcqMetaBase - Constructor /*{{{*/
788 pkgAcqMetaBase::pkgAcqMetaBase(pkgAcquire
* const Owner
,
789 pkgAcqMetaClearSig
* const TransactionManager
,
790 std::vector
<IndexTarget
> const &IndexTargets
,
791 IndexTarget
const &DataTarget
)
792 : pkgAcqTransactionItem(Owner
, TransactionManager
, DataTarget
), d(NULL
),
793 IndexTargets(IndexTargets
),
794 AuthPass(false), IMSHit(false)
798 // AcqMetaBase::Add - Add a item to the current Transaction /*{{{*/
799 void pkgAcqMetaBase::Add(pkgAcqTransactionItem
* const I
)
801 Transaction
.push_back(I
);
804 // AcqMetaBase::AbortTransaction - Abort the current Transaction /*{{{*/
805 void pkgAcqMetaBase::AbortTransaction()
807 if(_config
->FindB("Debug::Acquire::Transaction", false) == true)
808 std::clog
<< "AbortTransaction: " << TransactionManager
<< std::endl
;
810 // ensure the toplevel is in error state too
811 for (std::vector
<pkgAcqTransactionItem
*>::iterator I
= Transaction
.begin();
812 I
!= Transaction
.end(); ++I
)
814 (*I
)->TransactionState(TransactionAbort
);
819 // AcqMetaBase::TransactionHasError - Check for errors in Transaction /*{{{*/
820 APT_PURE
bool pkgAcqMetaBase::TransactionHasError() const
822 for (std::vector
<pkgAcqTransactionItem
*>::const_iterator I
= Transaction
.begin();
823 I
!= Transaction
.end(); ++I
)
825 switch((*I
)->Status
) {
826 case StatDone
: break;
827 case StatIdle
: break;
828 case StatAuthError
: return true;
829 case StatError
: return true;
830 case StatTransientNetworkError
: return true;
831 case StatFetching
: break;
837 // AcqMetaBase::CommitTransaction - Commit a transaction /*{{{*/
838 void pkgAcqMetaBase::CommitTransaction()
840 if(_config
->FindB("Debug::Acquire::Transaction", false) == true)
841 std::clog
<< "CommitTransaction: " << this << std::endl
;
843 // move new files into place *and* remove files that are not
844 // part of the transaction but are still on disk
845 for (std::vector
<pkgAcqTransactionItem
*>::iterator I
= Transaction
.begin();
846 I
!= Transaction
.end(); ++I
)
848 (*I
)->TransactionState(TransactionCommit
);
853 // AcqMetaBase::TransactionStageCopy - Stage a file for copying /*{{{*/
854 void pkgAcqMetaBase::TransactionStageCopy(pkgAcqTransactionItem
* const I
,
855 const std::string
&From
,
856 const std::string
&To
)
858 I
->PartialFile
= From
;
862 // AcqMetaBase::TransactionStageRemoval - Stage a file for removal /*{{{*/
863 void pkgAcqMetaBase::TransactionStageRemoval(pkgAcqTransactionItem
* const I
,
864 const std::string
&FinalFile
)
867 I
->DestFile
= FinalFile
;
870 // AcqMetaBase::GenerateAuthWarning - Check gpg authentication error /*{{{*/
871 bool pkgAcqMetaBase::CheckStopAuthentication(pkgAcquire::Item
* const I
, const std::string
&Message
)
873 // FIXME: this entire function can do now that we disallow going to
874 // a unauthenticated state and can cleanly rollback
876 string
const Final
= I
->GetFinalFilename();
877 if(FileExists(Final
))
879 I
->Status
= StatTransientNetworkError
;
880 _error
->Warning(_("An error occurred during the signature "
881 "verification. The repository is not updated "
882 "and the previous index files will be used. "
883 "GPG error: %s: %s"),
884 Desc
.Description
.c_str(),
885 LookupTag(Message
,"Message").c_str());
886 RunScripts("APT::Update::Auth-Failure");
888 } else if (LookupTag(Message
,"Message").find("NODATA") != string::npos
) {
889 /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */
890 _error
->Error(_("GPG error: %s: %s"),
891 Desc
.Description
.c_str(),
892 LookupTag(Message
,"Message").c_str());
893 I
->Status
= StatAuthError
;
896 _error
->Warning(_("GPG error: %s: %s"),
897 Desc
.Description
.c_str(),
898 LookupTag(Message
,"Message").c_str());
900 // gpgv method failed
901 ReportMirrorFailure("GPGFailure");
905 // AcqMetaBase::Custom600Headers - Get header for AcqMetaBase /*{{{*/
906 // ---------------------------------------------------------------------
907 string
pkgAcqMetaBase::Custom600Headers() const
909 std::string Header
= "\nIndex-File: true";
910 std::string MaximumSize
;
911 strprintf(MaximumSize
, "\nMaximum-Size: %i",
912 _config
->FindI("Acquire::MaxReleaseFileSize", 10*1000*1000));
913 Header
+= MaximumSize
;
915 string
const FinalFile
= GetFinalFilename();
917 if (stat(FinalFile
.c_str(),&Buf
) == 0)
918 Header
+= "\nLast-Modified: " + TimeRFC1123(Buf
.st_mtime
);
923 // AcqMetaBase::QueueForSignatureVerify /*{{{*/
924 void pkgAcqMetaBase::QueueForSignatureVerify(pkgAcqTransactionItem
* const I
, std::string
const &File
, std::string
const &Signature
)
927 I
->Desc
.URI
= "gpgv:" + Signature
;
930 I
->SetActiveSubprocess("gpgv");
933 // AcqMetaBase::CheckDownloadDone /*{{{*/
934 bool pkgAcqMetaBase::CheckDownloadDone(pkgAcqTransactionItem
* const I
, const std::string
&Message
, HashStringList
const &Hashes
) const
936 // We have just finished downloading a Release file (it is not
939 std::string
const FileName
= LookupTag(Message
,"Filename");
940 if (FileName
!= I
->DestFile
&& RealFileExists(I
->DestFile
) == false)
943 I
->Desc
.URI
= "copy:" + FileName
;
944 I
->QueueURI(I
->Desc
);
948 // make sure to verify against the right file on I-M-S hit
949 bool IMSHit
= StringToBool(LookupTag(Message
,"IMS-Hit"), false);
950 if (IMSHit
== false && Hashes
.usable())
952 // detect IMS-Hits servers haven't detected by Hash comparison
953 std::string
const FinalFile
= I
->GetFinalFilename();
954 if (RealFileExists(FinalFile
) && Hashes
.VerifyFile(FinalFile
) == true)
957 RemoveFile("CheckDownloadDone", I
->DestFile
);
963 // for simplicity, the transaction manager is always InRelease
964 // even if it doesn't exist.
965 if (TransactionManager
!= NULL
)
966 TransactionManager
->IMSHit
= true;
967 I
->PartialFile
= I
->DestFile
= I
->GetFinalFilename();
970 // set Item to complete as the remaining work is all local (verify etc)
976 bool pkgAcqMetaBase::CheckAuthDone(string
const &Message
) /*{{{*/
978 // At this point, the gpgv method has succeeded, so there is a
979 // valid signature from a key in the trusted keyring. We
980 // perform additional verification of its contents, and use them
981 // to verify the indexes we are about to download
983 if (TransactionManager
->IMSHit
== false)
985 // open the last (In)Release if we have it
986 std::string
const FinalFile
= GetFinalFilename();
987 std::string FinalRelease
;
988 std::string FinalInRelease
;
989 if (APT::String::Endswith(FinalFile
, "InRelease"))
991 FinalInRelease
= FinalFile
;
992 FinalRelease
= FinalFile
.substr(0, FinalFile
.length() - strlen("InRelease")) + "Release";
996 FinalInRelease
= FinalFile
.substr(0, FinalFile
.length() - strlen("Release")) + "InRelease";
997 FinalRelease
= FinalFile
;
999 if (RealFileExists(FinalInRelease
) || RealFileExists(FinalRelease
))
1001 TransactionManager
->LastMetaIndexParser
= TransactionManager
->MetaIndexParser
->UnloadedClone();
1002 if (TransactionManager
->LastMetaIndexParser
!= NULL
)
1004 _error
->PushToStack();
1005 if (RealFileExists(FinalInRelease
))
1006 TransactionManager
->LastMetaIndexParser
->Load(FinalInRelease
, NULL
);
1008 TransactionManager
->LastMetaIndexParser
->Load(FinalRelease
, NULL
);
1009 // its unlikely to happen, but if what we have is bad ignore it
1010 if (_error
->PendingError())
1012 delete TransactionManager
->LastMetaIndexParser
;
1013 TransactionManager
->LastMetaIndexParser
= NULL
;
1015 _error
->RevertToStack();
1020 if (TransactionManager
->MetaIndexParser
->Load(DestFile
, &ErrorText
) == false)
1022 Status
= StatAuthError
;
1026 if (!VerifyVendor(Message
))
1028 Status
= StatAuthError
;
1032 if (_config
->FindB("Debug::pkgAcquire::Auth", false))
1033 std::cerr
<< "Signature verification succeeded: "
1034 << DestFile
<< std::endl
;
1036 // Download further indexes with verification
1042 void pkgAcqMetaBase::QueueIndexes(bool const verify
) /*{{{*/
1044 // at this point the real Items are loaded in the fetcher
1045 ExpectedAdditionalItems
= 0;
1047 bool metaBaseSupportsByHash
= false;
1048 if (TransactionManager
!= NULL
&& TransactionManager
->MetaIndexParser
!= NULL
)
1049 metaBaseSupportsByHash
= TransactionManager
->MetaIndexParser
->GetSupportsAcquireByHash();
1051 for (std::vector
<IndexTarget
>::iterator Target
= IndexTargets
.begin();
1052 Target
!= IndexTargets
.end();
1055 // all is an implementation detail. Users shouldn't use this as arch
1056 // We need this support trickery here as e.g. Debian has binary-all files already,
1057 // but arch:all packages are still in the arch:any files, so we would waste precious
1058 // download time, bandwidth and diskspace for nothing, BUT Debian doesn't feature all
1059 // in the set of supported architectures, so we can filter based on this property rather
1060 // than invent an entirely new flag we would need to carry for all of eternity.
1061 if (Target
->Option(IndexTarget::ARCHITECTURE
) == "all")
1063 if (TransactionManager
->MetaIndexParser
->IsArchitectureSupported("all") == false)
1065 if (TransactionManager
->MetaIndexParser
->IsArchitectureAllSupportedFor(*Target
) == false)
1069 bool trypdiff
= Target
->OptionBool(IndexTarget::PDIFFS
);
1072 if (TransactionManager
->MetaIndexParser
->Exists(Target
->MetaKey
) == false)
1074 // optional targets that we do not have in the Release file are skipped
1075 if (Target
->IsOptional
)
1078 std::string
const &arch
= Target
->Option(IndexTarget::ARCHITECTURE
);
1079 if (arch
.empty() == false)
1081 if (TransactionManager
->MetaIndexParser
->IsArchitectureSupported(arch
) == false)
1083 _error
->Notice(_("Skipping acquire of configured file '%s' as repository '%s' doesn't support architecture '%s'"),
1084 Target
->MetaKey
.c_str(), TransactionManager
->Target
.Description
.c_str(), arch
.c_str());
1087 // if the architecture is officially supported but currently no packages for it available,
1088 // ignore silently as this is pretty much the same as just shipping an empty file.
1089 // if we don't know which architectures are supported, we do NOT ignore it to notify user about this
1090 if (TransactionManager
->MetaIndexParser
->IsArchitectureSupported("*undefined*") == false)
1094 Status
= StatAuthError
;
1095 strprintf(ErrorText
, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), Target
->MetaKey
.c_str());
1100 auto const hashes
= GetExpectedHashesFor(Target
->MetaKey
);
1101 if (hashes
.usable() == false && hashes
.empty() == false)
1103 _error
->Warning(_("Skipping acquire of configured file '%s' as repository '%s' provides only weak security information for it"),
1104 Target
->MetaKey
.c_str(), TransactionManager
->Target
.Description
.c_str());
1109 // autoselect the compression method
1110 std::vector
<std::string
> types
= VectorizeString(Target
->Option(IndexTarget::COMPRESSIONTYPES
), ' ');
1111 types
.erase(std::remove_if(types
.begin(), types
.end(), [&](std::string
const &t
) {
1112 if (t
== "uncompressed")
1113 return TransactionManager
->MetaIndexParser
->Exists(Target
->MetaKey
) == false;
1114 std::string
const MetaKey
= Target
->MetaKey
+ "." + t
;
1115 return TransactionManager
->MetaIndexParser
->Exists(MetaKey
) == false;
1117 if (types
.empty() == false)
1119 std::ostringstream os
;
1120 // add the special compressiontype byhash first if supported
1121 std::string
const useByHashConf
= Target
->Option(IndexTarget::BY_HASH
);
1122 bool useByHash
= false;
1123 if(useByHashConf
== "force")
1126 useByHash
= StringToBool(useByHashConf
) == true && metaBaseSupportsByHash
;
1127 if (useByHash
== true)
1129 std::copy(types
.begin(), types
.end()-1, std::ostream_iterator
<std::string
>(os
, " "));
1130 os
<< *types
.rbegin();
1131 Target
->Options
["COMPRESSIONTYPES"] = os
.str();
1134 Target
->Options
["COMPRESSIONTYPES"].clear();
1136 std::string filename
= GetExistingFilename(GetFinalFileNameFromURI(Target
->URI
));
1137 if (filename
.empty() == false)
1139 // if the Release file is a hit and we have an index it must be the current one
1140 if (TransactionManager
->IMSHit
== true)
1142 else if (TransactionManager
->LastMetaIndexParser
!= NULL
)
1144 // see if the file changed since the last Release file
1145 // we use the uncompressed files as we might compress differently compared to the server,
1146 // so the hashes might not match, even if they contain the same data.
1147 HashStringList
const newFile
= GetExpectedHashesFromFor(TransactionManager
->MetaIndexParser
, Target
->MetaKey
);
1148 HashStringList
const oldFile
= GetExpectedHashesFromFor(TransactionManager
->LastMetaIndexParser
, Target
->MetaKey
);
1149 if (newFile
!= oldFile
)
1156 trypdiff
= false; // no file to patch
1158 if (filename
.empty() == false)
1160 new NoActionItem(Owner
, *Target
, filename
);
1161 std::string
const idxfilename
= GetFinalFileNameFromURI(Target
->URI
+ ".diff/Index");
1162 if (FileExists(idxfilename
))
1163 new NoActionItem(Owner
, *Target
, idxfilename
);
1167 // check if we have patches available
1168 trypdiff
&= TransactionManager
->MetaIndexParser
->Exists(Target
->MetaKey
+ ".diff/Index");
1172 // if we have no file to patch, no point in trying
1173 trypdiff
&= (GetExistingFilename(GetFinalFileNameFromURI(Target
->URI
)).empty() == false);
1176 // no point in patching from local sources
1179 std::string
const proto
= Target
->URI
.substr(0, strlen("file:/"));
1180 if (proto
== "file:/" || proto
== "copy:/" || proto
== "cdrom:")
1184 // Queue the Index file (Packages, Sources, Translation-$foo, …)
1186 new pkgAcqDiffIndex(Owner
, TransactionManager
, *Target
);
1188 new pkgAcqIndex(Owner
, TransactionManager
, *Target
);
1192 bool pkgAcqMetaBase::VerifyVendor(string
const &Message
) /*{{{*/
1194 string::size_type pos
;
1196 // check for missing sigs (that where not fatal because otherwise we had
1199 string msg
= _("There is no public key available for the "
1200 "following key IDs:\n");
1201 pos
= Message
.find("NO_PUBKEY ");
1202 if (pos
!= std::string::npos
)
1204 string::size_type start
= pos
+strlen("NO_PUBKEY ");
1205 string Fingerprint
= Message
.substr(start
, Message
.find("\n")-start
);
1206 missingkeys
+= (Fingerprint
);
1208 if(!missingkeys
.empty())
1209 _error
->Warning("%s", (msg
+ missingkeys
).c_str());
1211 string Transformed
= TransactionManager
->MetaIndexParser
->GetExpectedDist();
1213 if (Transformed
== "../project/experimental")
1215 Transformed
= "experimental";
1218 pos
= Transformed
.rfind('/');
1219 if (pos
!= string::npos
)
1221 Transformed
= Transformed
.substr(0, pos
);
1224 if (Transformed
== ".")
1229 if (TransactionManager
->MetaIndexParser
->GetValidUntil() > 0)
1231 time_t const invalid_since
= time(NULL
) - TransactionManager
->MetaIndexParser
->GetValidUntil();
1232 if (invalid_since
> 0)
1236 // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1237 // the time since then the file is invalid - formatted in the same way as in
1238 // the download progress display (e.g. 7d 3h 42min 1s)
1239 _("Release file for %s is expired (invalid since %s). "
1240 "Updates for this repository will not be applied."),
1241 Target
.URI
.c_str(), TimeToStr(invalid_since
).c_str());
1242 if (ErrorText
.empty())
1244 return _error
->Error("%s", errmsg
.c_str());
1248 /* Did we get a file older than what we have? This is a last minute IMS hit and doubles
1249 as a prevention of downgrading us to older (still valid) files */
1250 if (TransactionManager
->IMSHit
== false && TransactionManager
->LastMetaIndexParser
!= NULL
&&
1251 TransactionManager
->LastMetaIndexParser
->GetDate() > TransactionManager
->MetaIndexParser
->GetDate())
1253 TransactionManager
->IMSHit
= true;
1254 RemoveFile("VerifyVendor", DestFile
);
1255 PartialFile
= DestFile
= GetFinalFilename();
1256 // load the 'old' file in the 'new' one instead of flipping pointers as
1257 // the new one isn't owned by us, while the old one is so cleanup would be confused.
1258 TransactionManager
->MetaIndexParser
->swapLoad(TransactionManager
->LastMetaIndexParser
);
1259 delete TransactionManager
->LastMetaIndexParser
;
1260 TransactionManager
->LastMetaIndexParser
= NULL
;
1263 if (_config
->FindB("Debug::pkgAcquire::Auth", false))
1265 std::cerr
<< "Got Codename: " << TransactionManager
->MetaIndexParser
->GetCodename() << std::endl
;
1266 std::cerr
<< "Expecting Dist: " << TransactionManager
->MetaIndexParser
->GetExpectedDist() << std::endl
;
1267 std::cerr
<< "Transformed Dist: " << Transformed
<< std::endl
;
1270 if (TransactionManager
->MetaIndexParser
->CheckDist(Transformed
) == false)
1272 // This might become fatal one day
1273 // Status = StatAuthError;
1274 // ErrorText = "Conflicting distribution; expected "
1275 // + MetaIndexParser->GetExpectedDist() + " but got "
1276 // + MetaIndexParser->GetCodename();
1278 if (!Transformed
.empty())
1280 _error
->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1281 Desc
.Description
.c_str(),
1282 Transformed
.c_str(),
1283 TransactionManager
->MetaIndexParser
->GetCodename().c_str());
1290 pkgAcqMetaBase::~pkgAcqMetaBase()
1294 pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire
* const Owner
, /*{{{*/
1295 IndexTarget
const &ClearsignedTarget
,
1296 IndexTarget
const &DetachedDataTarget
, IndexTarget
const &DetachedSigTarget
,
1297 std::vector
<IndexTarget
> const &IndexTargets
,
1298 metaIndex
* const MetaIndexParser
) :
1299 pkgAcqMetaIndex(Owner
, this, ClearsignedTarget
, DetachedSigTarget
, IndexTargets
),
1300 d(NULL
), ClearsignedTarget(ClearsignedTarget
),
1301 DetachedDataTarget(DetachedDataTarget
),
1302 MetaIndexParser(MetaIndexParser
), LastMetaIndexParser(NULL
)
1304 // index targets + (worst case:) Release/Release.gpg
1305 ExpectedAdditionalItems
= IndexTargets
.size() + 2;
1306 TransactionManager
->Add(this);
1309 pkgAcqMetaClearSig::~pkgAcqMetaClearSig() /*{{{*/
1311 if (LastMetaIndexParser
!= NULL
)
1312 delete LastMetaIndexParser
;
1315 // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/
1316 string
pkgAcqMetaClearSig::Custom600Headers() const
1318 string Header
= pkgAcqMetaBase::Custom600Headers();
1319 Header
+= "\nFail-Ignore: true";
1320 std::string
const key
= TransactionManager
->MetaIndexParser
->GetSignedBy();
1321 if (key
.empty() == false)
1322 Header
+= "\nSigned-By: " + key
;
1327 bool pkgAcqMetaClearSig::VerifyDone(std::string
const &Message
, /*{{{*/
1328 pkgAcquire::MethodConfig
const * const Cnf
)
1330 Item::VerifyDone(Message
, Cnf
);
1332 if (FileExists(DestFile
) && !StartsWithGPGClearTextSignature(DestFile
))
1333 return RenameOnError(NotClearsigned
);
1338 // pkgAcqMetaClearSig::Done - We got a file /*{{{*/
1339 void pkgAcqMetaClearSig::Done(std::string
const &Message
,
1340 HashStringList
const &Hashes
,
1341 pkgAcquire::MethodConfig
const * const Cnf
)
1343 Item::Done(Message
, Hashes
, Cnf
);
1345 if(AuthPass
== false)
1347 if(CheckDownloadDone(this, Message
, Hashes
) == true)
1348 QueueForSignatureVerify(this, DestFile
, DestFile
);
1351 else if(CheckAuthDone(Message
) == true)
1353 if (TransactionManager
->IMSHit
== false)
1354 TransactionManager
->TransactionStageCopy(this, DestFile
, GetFinalFilename());
1355 else if (RealFileExists(GetFinalFilename()) == false)
1357 // We got an InRelease file IMSHit, but we haven't one, which means
1358 // we had a valid Release/Release.gpg combo stepping in, which we have
1359 // to 'acquire' now to ensure list cleanup isn't removing them
1360 new NoActionItem(Owner
, DetachedDataTarget
);
1361 new NoActionItem(Owner
, DetachedSigTarget
);
1366 void pkgAcqMetaClearSig::Failed(string
const &Message
,pkgAcquire::MethodConfig
const * const Cnf
) /*{{{*/
1368 Item::Failed(Message
, Cnf
);
1370 // we failed, we will not get additional items from this method
1371 ExpectedAdditionalItems
= 0;
1373 if (AuthPass
== false)
1375 if (Status
== StatAuthError
|| Status
== StatTransientNetworkError
)
1377 // if we expected a ClearTextSignature (InRelease) but got a network
1378 // error or got a file, but it wasn't valid, we end up here (see VerifyDone).
1379 // As these is usually called by web-portals we do not try Release/Release.gpg
1380 // as this is gonna fail anyway and instead abort our try (LP#346386)
1381 TransactionManager
->AbortTransaction();
1385 // Queue the 'old' InRelease file for removal if we try Release.gpg
1386 // as otherwise the file will stay around and gives a false-auth
1387 // impression (CVE-2012-0214)
1388 TransactionManager
->TransactionStageRemoval(this, GetFinalFilename());
1391 new pkgAcqMetaIndex(Owner
, TransactionManager
, DetachedDataTarget
, DetachedSigTarget
, IndexTargets
);
1395 if(CheckStopAuthentication(this, Message
))
1398 // No Release file was present, or verification failed, so fall
1399 // back to queueing Packages files without verification
1400 // only allow going further if the user explicitly wants it
1401 if(AllowInsecureRepositories(_("The repository '%s' is not signed."), ClearsignedTarget
.Description
, TransactionManager
->MetaIndexParser
, TransactionManager
, this) == true)
1405 /* InRelease files become Release files, otherwise
1406 * they would be considered as trusted later on */
1407 string
const FinalRelease
= GetFinalFileNameFromURI(DetachedDataTarget
.URI
);
1408 string
const PartialRelease
= GetPartialFileNameFromURI(DetachedDataTarget
.URI
);
1409 string
const FinalReleasegpg
= GetFinalFileNameFromURI(DetachedSigTarget
.URI
);
1410 string
const FinalInRelease
= GetFinalFilename();
1411 Rename(DestFile
, PartialRelease
);
1412 TransactionManager
->TransactionStageCopy(this, PartialRelease
, FinalRelease
);
1414 if (RealFileExists(FinalReleasegpg
) || RealFileExists(FinalInRelease
))
1416 // open the last Release if we have it
1417 if (TransactionManager
->IMSHit
== false)
1419 TransactionManager
->LastMetaIndexParser
= TransactionManager
->MetaIndexParser
->UnloadedClone();
1420 if (TransactionManager
->LastMetaIndexParser
!= NULL
)
1422 _error
->PushToStack();
1423 if (RealFileExists(FinalInRelease
))
1424 TransactionManager
->LastMetaIndexParser
->Load(FinalInRelease
, NULL
);
1426 TransactionManager
->LastMetaIndexParser
->Load(FinalRelease
, NULL
);
1427 // its unlikely to happen, but if what we have is bad ignore it
1428 if (_error
->PendingError())
1430 delete TransactionManager
->LastMetaIndexParser
;
1431 TransactionManager
->LastMetaIndexParser
= NULL
;
1433 _error
->RevertToStack();
1438 // we parse the indexes here because at this point the user wanted
1439 // a repository that may potentially harm him
1440 if (TransactionManager
->MetaIndexParser
->Load(PartialRelease
, &ErrorText
) == false || VerifyVendor(Message
) == false)
1441 /* expired Release files are still a problem you need extra force for */;
1449 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire
* const Owner
, /*{{{*/
1450 pkgAcqMetaClearSig
* const TransactionManager
,
1451 IndexTarget
const &DataTarget
,
1452 IndexTarget
const &DetachedSigTarget
,
1453 vector
<IndexTarget
> const &IndexTargets
) :
1454 pkgAcqMetaBase(Owner
, TransactionManager
, IndexTargets
, DataTarget
), d(NULL
),
1455 DetachedSigTarget(DetachedSigTarget
)
1457 if(_config
->FindB("Debug::Acquire::Transaction", false) == true)
1458 std::clog
<< "New pkgAcqMetaIndex with TransactionManager "
1459 << this->TransactionManager
<< std::endl
;
1461 DestFile
= GetPartialFileNameFromURI(DataTarget
.URI
);
1464 Desc
.Description
= DataTarget
.Description
;
1466 Desc
.ShortDesc
= DataTarget
.ShortDesc
;
1467 Desc
.URI
= DataTarget
.URI
;
1469 // we expect more item
1470 ExpectedAdditionalItems
= IndexTargets
.size();
1474 void pkgAcqMetaIndex::Done(string
const &Message
, /*{{{*/
1475 HashStringList
const &Hashes
,
1476 pkgAcquire::MethodConfig
const * const Cfg
)
1478 Item::Done(Message
,Hashes
,Cfg
);
1480 if(CheckDownloadDone(this, Message
, Hashes
))
1482 // we have a Release file, now download the Signature, all further
1483 // verify/queue for additional downloads will be done in the
1484 // pkgAcqMetaSig::Done() code
1485 new pkgAcqMetaSig(Owner
, TransactionManager
, DetachedSigTarget
, this);
1489 // pkgAcqMetaIndex::Failed - no Release file present /*{{{*/
1490 void pkgAcqMetaIndex::Failed(string
const &Message
,
1491 pkgAcquire::MethodConfig
const * const Cnf
)
1493 pkgAcquire::Item::Failed(Message
, Cnf
);
1496 // No Release file was present so fall
1497 // back to queueing Packages files without verification
1498 // only allow going further if the user explicitly wants it
1499 if(AllowInsecureRepositories(_("The repository '%s' does not have a Release file."), Target
.Description
, TransactionManager
->MetaIndexParser
, TransactionManager
, this) == true)
1501 // ensure old Release files are removed
1502 TransactionManager
->TransactionStageRemoval(this, GetFinalFilename());
1504 // queue without any kind of hashsum support
1505 QueueIndexes(false);
1509 void pkgAcqMetaIndex::Finished() /*{{{*/
1511 if(_config
->FindB("Debug::Acquire::Transaction", false) == true)
1512 std::clog
<< "Finished: " << DestFile
<<std::endl
;
1513 if(TransactionManager
!= NULL
&&
1514 TransactionManager
->TransactionHasError() == false)
1515 TransactionManager
->CommitTransaction();
1518 std::string
pkgAcqMetaIndex::DescURI() const /*{{{*/
1523 pkgAcqMetaIndex::~pkgAcqMetaIndex() {}
1525 // AcqMetaSig::AcqMetaSig - Constructor /*{{{*/
1526 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire
* const Owner
,
1527 pkgAcqMetaClearSig
* const TransactionManager
,
1528 IndexTarget
const &Target
,
1529 pkgAcqMetaIndex
* const MetaIndex
) :
1530 pkgAcqTransactionItem(Owner
, TransactionManager
, Target
), d(NULL
), MetaIndex(MetaIndex
)
1532 DestFile
= GetPartialFileNameFromURI(Target
.URI
);
1534 // remove any partial downloaded sig-file in partial/.
1535 // it may confuse proxies and is too small to warrant a
1536 // partial download anyway
1537 RemoveFile("pkgAcqMetaSig", DestFile
);
1539 // set the TransactionManager
1540 if(_config
->FindB("Debug::Acquire::Transaction", false) == true)
1541 std::clog
<< "New pkgAcqMetaSig with TransactionManager "
1542 << TransactionManager
<< std::endl
;
1545 Desc
.Description
= Target
.Description
;
1547 Desc
.ShortDesc
= Target
.ShortDesc
;
1548 Desc
.URI
= Target
.URI
;
1550 // If we got a hit for Release, we will get one for Release.gpg too (or obscure errors),
1551 // so we skip the download step and go instantly to verification
1552 if (TransactionManager
->IMSHit
== true && RealFileExists(GetFinalFilename()))
1556 PartialFile
= DestFile
= GetFinalFilename();
1557 MetaIndexFileSignature
= DestFile
;
1558 MetaIndex
->QueueForSignatureVerify(this, MetaIndex
->DestFile
, DestFile
);
1564 pkgAcqMetaSig::~pkgAcqMetaSig() /*{{{*/
1568 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers /*{{{*/
1569 std::string
pkgAcqMetaSig::Custom600Headers() const
1571 std::string Header
= pkgAcqTransactionItem::Custom600Headers();
1572 std::string
const key
= TransactionManager
->MetaIndexParser
->GetSignedBy();
1573 if (key
.empty() == false)
1574 Header
+= "\nSigned-By: " + key
;
1578 // AcqMetaSig::Done - The signature was downloaded/verified /*{{{*/
1579 void pkgAcqMetaSig::Done(string
const &Message
, HashStringList
const &Hashes
,
1580 pkgAcquire::MethodConfig
const * const Cfg
)
1582 if (MetaIndexFileSignature
.empty() == false)
1584 DestFile
= MetaIndexFileSignature
;
1585 MetaIndexFileSignature
.clear();
1587 Item::Done(Message
, Hashes
, Cfg
);
1589 if(MetaIndex
->AuthPass
== false)
1591 if(MetaIndex
->CheckDownloadDone(this, Message
, Hashes
) == true)
1593 // destfile will be modified to point to MetaIndexFile for the
1594 // gpgv method, so we need to save it here
1595 MetaIndexFileSignature
= DestFile
;
1596 MetaIndex
->QueueForSignatureVerify(this, MetaIndex
->DestFile
, DestFile
);
1600 else if(MetaIndex
->CheckAuthDone(Message
) == true)
1602 if (TransactionManager
->IMSHit
== false)
1604 TransactionManager
->TransactionStageCopy(this, DestFile
, GetFinalFilename());
1605 TransactionManager
->TransactionStageCopy(MetaIndex
, MetaIndex
->DestFile
, MetaIndex
->GetFinalFilename());
1610 void pkgAcqMetaSig::Failed(string
const &Message
,pkgAcquire::MethodConfig
const * const Cnf
)/*{{{*/
1612 Item::Failed(Message
,Cnf
);
1614 // check if we need to fail at this point
1615 if (MetaIndex
->AuthPass
== true && MetaIndex
->CheckStopAuthentication(this, Message
))
1618 string
const FinalRelease
= MetaIndex
->GetFinalFilename();
1619 string
const FinalReleasegpg
= GetFinalFilename();
1620 string
const FinalInRelease
= TransactionManager
->GetFinalFilename();
1622 if (RealFileExists(FinalReleasegpg
) || RealFileExists(FinalInRelease
))
1624 std::string downgrade_msg
;
1625 strprintf(downgrade_msg
, _("The repository '%s' is no longer signed."),
1626 MetaIndex
->Target
.Description
.c_str());
1627 if(_config
->FindB("Acquire::AllowDowngradeToInsecureRepositories"))
1629 // meh, the users wants to take risks (we still mark the packages
1630 // from this repository as unauthenticated)
1631 _error
->Warning("%s", downgrade_msg
.c_str());
1632 _error
->Warning(_("This is normally not allowed, but the option "
1633 "Acquire::AllowDowngradeToInsecureRepositories was "
1634 "given to override it."));
1637 MessageInsecureRepository(true, downgrade_msg
);
1638 if (TransactionManager
->IMSHit
== false)
1639 Rename(MetaIndex
->DestFile
, MetaIndex
->DestFile
+ ".FAILED");
1640 Item::Failed("Message: " + downgrade_msg
, Cnf
);
1641 TransactionManager
->AbortTransaction();
1646 // ensures that a Release.gpg file in the lists/ is removed by the transaction
1647 TransactionManager
->TransactionStageRemoval(this, DestFile
);
1649 // only allow going further if the user explicitly wants it
1650 if (AllowInsecureRepositories(_("The repository '%s' is not signed."), MetaIndex
->Target
.Description
, TransactionManager
->MetaIndexParser
, TransactionManager
, this) == true)
1652 if (RealFileExists(FinalReleasegpg
) || RealFileExists(FinalInRelease
))
1654 // open the last Release if we have it
1655 if (TransactionManager
->IMSHit
== false)
1657 TransactionManager
->LastMetaIndexParser
= TransactionManager
->MetaIndexParser
->UnloadedClone();
1658 if (TransactionManager
->LastMetaIndexParser
!= NULL
)
1660 _error
->PushToStack();
1661 if (RealFileExists(FinalInRelease
))
1662 TransactionManager
->LastMetaIndexParser
->Load(FinalInRelease
, NULL
);
1664 TransactionManager
->LastMetaIndexParser
->Load(FinalRelease
, NULL
);
1665 // its unlikely to happen, but if what we have is bad ignore it
1666 if (_error
->PendingError())
1668 delete TransactionManager
->LastMetaIndexParser
;
1669 TransactionManager
->LastMetaIndexParser
= NULL
;
1671 _error
->RevertToStack();
1676 // we parse the indexes here because at this point the user wanted
1677 // a repository that may potentially harm him
1678 bool const GoodLoad
= TransactionManager
->MetaIndexParser
->Load(MetaIndex
->DestFile
, &ErrorText
);
1679 if (MetaIndex
->VerifyVendor(Message
) == false)
1680 /* expired Release files are still a problem you need extra force for */;
1682 MetaIndex
->QueueIndexes(GoodLoad
);
1684 TransactionManager
->TransactionStageCopy(MetaIndex
, MetaIndex
->DestFile
, MetaIndex
->GetFinalFilename());
1687 // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor
1688 if (Cnf
->LocalOnly
== true ||
1689 StringToBool(LookupTag(Message
,"Transient-Failure"),false) == false)
1698 // AcqBaseIndex - Constructor /*{{{*/
1699 pkgAcqBaseIndex::pkgAcqBaseIndex(pkgAcquire
* const Owner
,
1700 pkgAcqMetaClearSig
* const TransactionManager
,
1701 IndexTarget
const &Target
)
1702 : pkgAcqTransactionItem(Owner
, TransactionManager
, Target
), d(NULL
)
1706 pkgAcqBaseIndex::~pkgAcqBaseIndex() {}
1708 // AcqDiffIndex::AcqDiffIndex - Constructor /*{{{*/
1709 // ---------------------------------------------------------------------
1710 /* Get the DiffIndex file first and see if there are patches available
1711 * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
1712 * patches. If anything goes wrong in that process, it will fall back to
1713 * the original packages file
1715 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire
* const Owner
,
1716 pkgAcqMetaClearSig
* const TransactionManager
,
1717 IndexTarget
const &Target
)
1718 : pkgAcqBaseIndex(Owner
, TransactionManager
, Target
), d(NULL
), diffs(NULL
)
1720 Debug
= _config
->FindB("Debug::pkgAcquire::Diffs",false);
1723 Desc
.Description
= Target
.Description
+ ".diff/Index";
1724 Desc
.ShortDesc
= Target
.ShortDesc
;
1725 Desc
.URI
= Target
.URI
+ ".diff/Index";
1727 DestFile
= GetPartialFileNameFromURI(Desc
.URI
);
1730 std::clog
<< "pkgAcqDiffIndex: " << Desc
.URI
<< std::endl
;
1735 // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/
1736 // ---------------------------------------------------------------------
1737 /* The only header we use is the last-modified header. */
1738 string
pkgAcqDiffIndex::Custom600Headers() const
1740 if (TransactionManager
->LastMetaIndexParser
!= NULL
)
1741 return "\nIndex-File: true";
1743 string
const Final
= GetFinalFilename();
1746 std::clog
<< "Custom600Header-IMS: " << Final
<< std::endl
;
1749 if (stat(Final
.c_str(),&Buf
) != 0)
1750 return "\nIndex-File: true";
1752 return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf
.st_mtime
);
1755 void pkgAcqDiffIndex::QueueOnIMSHit() const /*{{{*/
1757 // list cleanup needs to know that this file as well as the already
1758 // present index is ours, so we create an empty diff to save it for us
1759 new pkgAcqIndexDiffs(Owner
, TransactionManager
, Target
);
1762 bool pkgAcqDiffIndex::ParseDiffIndex(string
const &IndexDiffFile
) /*{{{*/
1764 // failing here is fine: our caller will take care of trying to
1765 // get the complete file if patching fails
1767 std::clog
<< "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile
1770 FileFd
Fd(IndexDiffFile
,FileFd::ReadOnly
);
1772 if (Fd
.IsOpen() == false || Fd
.Failed())
1776 if(unlikely(TF
.Step(Tags
) == false))
1779 HashStringList ServerHashes
;
1780 unsigned long long ServerSize
= 0;
1782 for (char const * const * type
= HashString::SupportedHashes(); *type
!= NULL
; ++type
)
1784 std::string tagname
= *type
;
1785 tagname
.append("-Current");
1786 std::string
const tmp
= Tags
.FindS(tagname
.c_str());
1787 if (tmp
.empty() == true)
1791 unsigned long long size
;
1792 std::stringstream
ss(tmp
);
1794 if (unlikely(hash
.empty() == true))
1796 if (unlikely(ServerSize
!= 0 && ServerSize
!= size
))
1798 ServerHashes
.push_back(HashString(*type
, hash
));
1802 if (ServerHashes
.usable() == false)
1805 std::clog
<< "pkgAcqDiffIndex: " << IndexDiffFile
<< ": Did not find a good hashsum in the index" << std::endl
;
1809 std::string
const CurrentPackagesFile
= GetFinalFileNameFromURI(Target
.URI
);
1810 HashStringList
const TargetFileHashes
= GetExpectedHashesFor(Target
.MetaKey
);
1811 if (TargetFileHashes
.usable() == false || ServerHashes
!= TargetFileHashes
)
1815 std::clog
<< "pkgAcqDiffIndex: " << IndexDiffFile
<< ": Index has different hashes than parser, probably older, so fail pdiffing" << std::endl
;
1816 printHashSumComparision(CurrentPackagesFile
, ServerHashes
, TargetFileHashes
);
1821 HashStringList LocalHashes
;
1822 // try avoiding calculating the hash here as this is costly
1823 if (TransactionManager
->LastMetaIndexParser
!= NULL
)
1824 LocalHashes
= GetExpectedHashesFromFor(TransactionManager
->LastMetaIndexParser
, Target
.MetaKey
);
1825 if (LocalHashes
.usable() == false)
1827 FileFd
fd(CurrentPackagesFile
, FileFd::ReadOnly
, FileFd::Auto
);
1828 Hashes
LocalHashesCalc(ServerHashes
);
1829 LocalHashesCalc
.AddFD(fd
);
1830 LocalHashes
= LocalHashesCalc
.GetHashStringList();
1833 if (ServerHashes
== LocalHashes
)
1835 // we have the same sha1 as the server so we are done here
1837 std::clog
<< "pkgAcqDiffIndex: Package file " << CurrentPackagesFile
<< " is up-to-date" << std::endl
;
1843 std::clog
<< "Server-Current: " << ServerHashes
.find(NULL
)->toStr() << " and we start at "
1844 << CurrentPackagesFile
<< " " << LocalHashes
.FileSize() << " " << LocalHashes
.find(NULL
)->toStr() << std::endl
;
1846 // historically, older hashes have more info than newer ones, so start
1847 // collecting with older ones first to avoid implementing complicated
1848 // information merging techniques… a failure is after all always
1849 // recoverable with a complete file and hashes aren't changed that often.
1850 std::vector
<char const *> types
;
1851 for (char const * const * type
= HashString::SupportedHashes(); *type
!= NULL
; ++type
)
1852 types
.push_back(*type
);
1854 // parse all of (provided) history
1855 vector
<DiffInfo
> available_patches
;
1856 bool firstAcceptedHashes
= true;
1857 for (auto type
= types
.crbegin(); type
!= types
.crend(); ++type
)
1859 if (LocalHashes
.find(*type
) == NULL
)
1862 std::string tagname
= *type
;
1863 tagname
.append("-History");
1864 std::string
const tmp
= Tags
.FindS(tagname
.c_str());
1865 if (tmp
.empty() == true)
1868 string hash
, filename
;
1869 unsigned long long size
;
1870 std::stringstream
ss(tmp
);
1872 while (ss
>> hash
>> size
>> filename
)
1874 if (unlikely(hash
.empty() == true || filename
.empty() == true))
1877 // see if we have a record for this file already
1878 std::vector
<DiffInfo
>::iterator cur
= available_patches
.begin();
1879 for (; cur
!= available_patches
.end(); ++cur
)
1881 if (cur
->file
!= filename
)
1883 cur
->result_hashes
.push_back(HashString(*type
, hash
));
1886 if (cur
!= available_patches
.end())
1888 if (firstAcceptedHashes
== true)
1891 next
.file
= filename
;
1892 next
.result_hashes
.push_back(HashString(*type
, hash
));
1893 next
.result_hashes
.FileSize(size
);
1894 available_patches
.push_back(next
);
1899 std::clog
<< "pkgAcqDiffIndex: " << IndexDiffFile
<< ": File " << filename
1900 << " wasn't in the list for the first parsed hash! (history)" << std::endl
;
1904 firstAcceptedHashes
= false;
1907 if (unlikely(available_patches
.empty() == true))
1910 std::clog
<< "pkgAcqDiffIndex: " << IndexDiffFile
<< ": "
1911 << "Couldn't find any patches for the patch series." << std::endl
;
1915 for (auto type
= types
.crbegin(); type
!= types
.crend(); ++type
)
1917 if (LocalHashes
.find(*type
) == NULL
)
1920 std::string tagname
= *type
;
1921 tagname
.append("-Patches");
1922 std::string
const tmp
= Tags
.FindS(tagname
.c_str());
1923 if (tmp
.empty() == true)
1926 string hash
, filename
;
1927 unsigned long long size
;
1928 std::stringstream
ss(tmp
);
1930 while (ss
>> hash
>> size
>> filename
)
1932 if (unlikely(hash
.empty() == true || filename
.empty() == true))
1935 // see if we have a record for this file already
1936 std::vector
<DiffInfo
>::iterator cur
= available_patches
.begin();
1937 for (; cur
!= available_patches
.end(); ++cur
)
1939 if (cur
->file
!= filename
)
1941 if (cur
->patch_hashes
.empty())
1942 cur
->patch_hashes
.FileSize(size
);
1943 cur
->patch_hashes
.push_back(HashString(*type
, hash
));
1946 if (cur
!= available_patches
.end())
1949 std::clog
<< "pkgAcqDiffIndex: " << IndexDiffFile
<< ": File " << filename
1950 << " wasn't in the list for the first parsed hash! (patches)" << std::endl
;
1955 for (auto type
= types
.crbegin(); type
!= types
.crend(); ++type
)
1957 std::string tagname
= *type
;
1958 tagname
.append("-Download");
1959 std::string
const tmp
= Tags
.FindS(tagname
.c_str());
1960 if (tmp
.empty() == true)
1963 string hash
, filename
;
1964 unsigned long long size
;
1965 std::stringstream
ss(tmp
);
1967 // FIXME: all of pdiff supports only .gz compressed patches
1968 while (ss
>> hash
>> size
>> filename
)
1970 if (unlikely(hash
.empty() == true || filename
.empty() == true))
1972 if (unlikely(APT::String::Endswith(filename
, ".gz") == false))
1974 filename
.erase(filename
.length() - 3);
1976 // see if we have a record for this file already
1977 std::vector
<DiffInfo
>::iterator cur
= available_patches
.begin();
1978 for (; cur
!= available_patches
.end(); ++cur
)
1980 if (cur
->file
!= filename
)
1982 if (cur
->download_hashes
.empty())
1983 cur
->download_hashes
.FileSize(size
);
1984 cur
->download_hashes
.push_back(HashString(*type
, hash
));
1987 if (cur
!= available_patches
.end())
1990 std::clog
<< "pkgAcqDiffIndex: " << IndexDiffFile
<< ": File " << filename
1991 << " wasn't in the list for the first parsed hash! (download)" << std::endl
;
1997 bool foundStart
= false;
1998 for (std::vector
<DiffInfo
>::iterator cur
= available_patches
.begin();
1999 cur
!= available_patches
.end(); ++cur
)
2001 if (LocalHashes
!= cur
->result_hashes
)
2004 available_patches
.erase(available_patches
.begin(), cur
);
2009 if (foundStart
== false || unlikely(available_patches
.empty() == true))
2012 std::clog
<< "pkgAcqDiffIndex: " << IndexDiffFile
<< ": "
2013 << "Couldn't find the start of the patch series." << std::endl
;
2017 // patching with too many files is rather slow compared to a fast download
2018 unsigned long const fileLimit
= _config
->FindI("Acquire::PDiffs::FileLimit", 0);
2019 if (fileLimit
!= 0 && fileLimit
< available_patches
.size())
2022 std::clog
<< "Need " << available_patches
.size() << " diffs (Limit is " << fileLimit
2023 << ") so fallback to complete download" << std::endl
;
2027 // calculate the size of all patches we have to get
2028 // note that all sizes are uncompressed, while we download compressed files
2029 unsigned long long patchesSize
= 0;
2030 for (std::vector
<DiffInfo
>::const_iterator cur
= available_patches
.begin();
2031 cur
!= available_patches
.end(); ++cur
)
2032 patchesSize
+= cur
->patch_hashes
.FileSize();
2033 unsigned long long const sizeLimit
= ServerSize
* _config
->FindI("Acquire::PDiffs::SizeLimit", 100);
2034 if (sizeLimit
> 0 && (sizeLimit
/100) < patchesSize
)
2037 std::clog
<< "Need " << patchesSize
<< " bytes (Limit is " << sizeLimit
/100
2038 << ") so fallback to complete download" << std::endl
;
2042 // we have something, queue the diffs
2043 string::size_type
const last_space
= Description
.rfind(" ");
2044 if(last_space
!= string::npos
)
2045 Description
.erase(last_space
, Description
.size()-last_space
);
2047 /* decide if we should download patches one by one or in one go:
2048 The first is good if the server merges patches, but many don't so client
2049 based merging can be attempt in which case the second is better.
2050 "bad things" will happen if patches are merged on the server,
2051 but client side merging is attempt as well */
2052 bool pdiff_merge
= _config
->FindB("Acquire::PDiffs::Merge", true);
2053 if (pdiff_merge
== true)
2055 // reprepro adds this flag if it has merged patches on the server
2056 std::string
const precedence
= Tags
.FindS("X-Patch-Precedence");
2057 pdiff_merge
= (precedence
!= "merged");
2062 std::string
const PartialFile
= GetPartialFileNameFromURI(Target
.URI
);
2063 std::vector
<std::string
> exts
= APT::Configuration::getCompressorExtensions();
2064 for (auto const &ext
: exts
)
2066 std::string
const Partial
= PartialFile
+ ext
;
2067 if (FileExists(Partial
))
2068 RemoveFile("PDiffs-Bootstrap", Partial
);
2070 std::string
const Final
= GetExistingFilename(CurrentPackagesFile
);
2071 if (unlikely(Final
.empty())) // because we wouldn't be called in such a case
2073 std::string
const Ext
= Final
.substr(CurrentPackagesFile
.length());
2074 std::string
const Partial
= PartialFile
+ Ext
;
2075 if (symlink(Final
.c_str(), Partial
.c_str()) != 0)
2077 std::clog
<< "Bootstrap-linking for patching " << CurrentPackagesFile
<< " by linking " << Final
<< " to " << Partial
<< " failed!" << std::endl
;
2082 if (pdiff_merge
== false)
2083 new pkgAcqIndexDiffs(Owner
, TransactionManager
, Target
, available_patches
);
2086 diffs
= new std::vector
<pkgAcqIndexMergeDiffs
*>(available_patches
.size());
2087 for(size_t i
= 0; i
< available_patches
.size(); ++i
)
2088 (*diffs
)[i
] = new pkgAcqIndexMergeDiffs(Owner
, TransactionManager
,
2090 available_patches
[i
],
2100 void pkgAcqDiffIndex::Failed(string
const &Message
,pkgAcquire::MethodConfig
const * const Cnf
)/*{{{*/
2102 Item::Failed(Message
,Cnf
);
2106 std::clog
<< "pkgAcqDiffIndex failed: " << Desc
.URI
<< " with " << Message
<< std::endl
2107 << "Falling back to normal index file acquire" << std::endl
;
2109 new pkgAcqIndex(Owner
, TransactionManager
, Target
);
2112 void pkgAcqDiffIndex::Done(string
const &Message
,HashStringList
const &Hashes
, /*{{{*/
2113 pkgAcquire::MethodConfig
const * const Cnf
)
2116 std::clog
<< "pkgAcqDiffIndex::Done(): " << Desc
.URI
<< std::endl
;
2118 Item::Done(Message
, Hashes
, Cnf
);
2120 string
const FinalFile
= GetFinalFilename();
2121 if(StringToBool(LookupTag(Message
,"IMS-Hit"),false))
2122 DestFile
= FinalFile
;
2124 if(ParseDiffIndex(DestFile
) == false)
2126 Failed("Message: Couldn't parse pdiff index", Cnf
);
2127 // queue for final move - this should happen even if we fail
2128 // while parsing (e.g. on sizelimit) and download the complete file.
2129 TransactionManager
->TransactionStageCopy(this, DestFile
, FinalFile
);
2133 TransactionManager
->TransactionStageCopy(this, DestFile
, FinalFile
);
2142 pkgAcqDiffIndex::~pkgAcqDiffIndex()
2148 // AcqIndexDiffs::AcqIndexDiffs - Constructor /*{{{*/
2149 // ---------------------------------------------------------------------
2150 /* The package diff is added to the queue. one object is constructed
2151 * for each diff and the index
2153 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire
* const Owner
,
2154 pkgAcqMetaClearSig
* const TransactionManager
,
2155 IndexTarget
const &Target
,
2156 vector
<DiffInfo
> const &diffs
)
2157 : pkgAcqBaseIndex(Owner
, TransactionManager
, Target
), d(NULL
),
2158 available_patches(diffs
)
2160 DestFile
= GetKeepCompressedFileName(GetPartialFileNameFromURI(Target
.URI
), Target
);
2162 Debug
= _config
->FindB("Debug::pkgAcquire::Diffs",false);
2165 Description
= Target
.Description
;
2166 Desc
.ShortDesc
= Target
.ShortDesc
;
2168 if(available_patches
.empty() == true)
2170 // we are done (yeah!), check hashes against the final file
2171 DestFile
= GetKeepCompressedFileName(GetFinalFileNameFromURI(Target
.URI
), Target
);
2176 State
= StateFetchDiff
;
2181 void pkgAcqIndexDiffs::Failed(string
const &Message
,pkgAcquire::MethodConfig
const * const Cnf
)/*{{{*/
2183 Item::Failed(Message
,Cnf
);
2186 DestFile
= GetKeepCompressedFileName(GetPartialFileNameFromURI(Target
.URI
), Target
);
2188 std::clog
<< "pkgAcqIndexDiffs failed: " << Desc
.URI
<< " with " << Message
<< std::endl
2189 << "Falling back to normal index file acquire " << std::endl
;
2190 RenameOnError(PDiffError
);
2191 std::string
const patchname
= GetDiffsPatchFileName(DestFile
);
2192 if (RealFileExists(patchname
))
2193 Rename(patchname
, patchname
+ ".FAILED");
2194 std::string
const UnpatchedFile
= GetExistingFilename(GetPartialFileNameFromURI(Target
.URI
));
2195 if (UnpatchedFile
.empty() == false && FileExists(UnpatchedFile
))
2196 Rename(UnpatchedFile
, UnpatchedFile
+ ".FAILED");
2197 new pkgAcqIndex(Owner
, TransactionManager
, Target
);
2201 // Finish - helper that cleans the item out of the fetcher queue /*{{{*/
2202 void pkgAcqIndexDiffs::Finish(bool allDone
)
2205 std::clog
<< "pkgAcqIndexDiffs::Finish(): "
2207 << Desc
.URI
<< std::endl
;
2209 // we restore the original name, this is required, otherwise
2210 // the file will be cleaned
2213 std::string
const Final
= GetKeepCompressedFileName(GetFinalFilename(), Target
);
2214 TransactionManager
->TransactionStageCopy(this, DestFile
, Final
);
2216 // this is for the "real" finish
2221 std::clog
<< "\n\nallDone: " << DestFile
<< "\n" << std::endl
;
2228 std::clog
<< "Finishing: " << Desc
.URI
<< std::endl
;
2235 bool pkgAcqIndexDiffs::QueueNextDiff() /*{{{*/
2237 // calc sha1 of the just patched file
2238 std::string
const PartialFile
= GetExistingFilename(GetPartialFileNameFromURI(Target
.URI
));
2239 if(unlikely(PartialFile
.empty()))
2241 Failed("Message: The file " + GetPartialFileNameFromURI(Target
.URI
) + " isn't available", NULL
);
2245 FileFd
fd(PartialFile
, FileFd::ReadOnly
, FileFd::Extension
);
2246 Hashes LocalHashesCalc
;
2247 LocalHashesCalc
.AddFD(fd
);
2248 HashStringList
const LocalHashes
= LocalHashesCalc
.GetHashStringList();
2251 std::clog
<< "QueueNextDiff: " << PartialFile
<< " (" << LocalHashes
.find(NULL
)->toStr() << ")" << std::endl
;
2253 HashStringList
const TargetFileHashes
= GetExpectedHashesFor(Target
.MetaKey
);
2254 if (unlikely(LocalHashes
.usable() == false || TargetFileHashes
.usable() == false))
2256 Failed("Local/Expected hashes are not usable for " + PartialFile
, NULL
);
2260 // final file reached before all patches are applied
2261 if(LocalHashes
== TargetFileHashes
)
2267 // remove all patches until the next matching patch is found
2268 // this requires the Index file to be ordered
2269 available_patches
.erase(available_patches
.begin(),
2270 std::find_if(available_patches
.begin(), available_patches
.end(), [&](DiffInfo
const &I
) {
2271 return I
.result_hashes
== LocalHashes
;
2274 // error checking and falling back if no patch was found
2275 if(available_patches
.empty() == true)
2277 Failed("No patches left to reach target for " + PartialFile
, NULL
);
2281 // queue the right diff
2282 Desc
.URI
= Target
.URI
+ ".diff/" + available_patches
[0].file
+ ".gz";
2283 Desc
.Description
= Description
+ " " + available_patches
[0].file
+ string(".pdiff");
2284 DestFile
= GetKeepCompressedFileName(GetPartialFileNameFromURI(Target
.URI
+ ".diff/" + available_patches
[0].file
), Target
);
2287 std::clog
<< "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc
.URI
<< std::endl
;
2294 void pkgAcqIndexDiffs::Done(string
const &Message
, HashStringList
const &Hashes
, /*{{{*/
2295 pkgAcquire::MethodConfig
const * const Cnf
)
2298 std::clog
<< "pkgAcqIndexDiffs::Done(): " << Desc
.URI
<< std::endl
;
2300 Item::Done(Message
, Hashes
, Cnf
);
2302 std::string
const UncompressedUnpatchedFile
= GetPartialFileNameFromURI(Target
.URI
);
2303 std::string
const UnpatchedFile
= GetExistingFilename(UncompressedUnpatchedFile
);
2304 std::string
const PatchFile
= GetDiffsPatchFileName(UnpatchedFile
);
2305 std::string
const PatchedFile
= GetKeepCompressedFileName(UncompressedUnpatchedFile
, Target
);
2309 // success in downloading a diff, enter ApplyDiff state
2310 case StateFetchDiff
:
2311 Rename(DestFile
, PatchFile
);
2312 DestFile
= GetKeepCompressedFileName(UncompressedUnpatchedFile
+ "-patched", Target
);
2314 std::clog
<< "Sending to rred method: " << UnpatchedFile
<< std::endl
;
2315 State
= StateApplyDiff
;
2317 Desc
.URI
= "rred:" + UnpatchedFile
;
2319 SetActiveSubprocess("rred");
2321 // success in download/apply a diff, queue next (if needed)
2322 case StateApplyDiff
:
2323 // remove the just applied patch and base file
2324 available_patches
.erase(available_patches
.begin());
2325 RemoveFile("pkgAcqIndexDiffs::Done", PatchFile
);
2326 RemoveFile("pkgAcqIndexDiffs::Done", UnpatchedFile
);
2328 std::clog
<< "Moving patched file in place: " << std::endl
2329 << DestFile
<< " -> " << PatchedFile
<< std::endl
;
2330 Rename(DestFile
, PatchedFile
);
2332 // see if there is more to download
2333 if(available_patches
.empty() == false)
2335 new pkgAcqIndexDiffs(Owner
, TransactionManager
, Target
, available_patches
);
2338 DestFile
= PatchedFile
;
2345 std::string
pkgAcqIndexDiffs::Custom600Headers() const /*{{{*/
2347 if(State
!= StateApplyDiff
)
2348 return pkgAcqBaseIndex::Custom600Headers();
2349 std::ostringstream patchhashes
;
2350 HashStringList
const ExpectedHashes
= available_patches
[0].patch_hashes
;
2351 for (HashStringList::const_iterator hs
= ExpectedHashes
.begin(); hs
!= ExpectedHashes
.end(); ++hs
)
2352 patchhashes
<< "\nPatch-0-" << hs
->HashType() << "-Hash: " << hs
->HashValue();
2353 patchhashes
<< pkgAcqBaseIndex::Custom600Headers();
2354 return patchhashes
.str();
2357 pkgAcqIndexDiffs::~pkgAcqIndexDiffs() {}
2359 // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor /*{{{*/
2360 pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire
* const Owner
,
2361 pkgAcqMetaClearSig
* const TransactionManager
,
2362 IndexTarget
const &Target
,
2363 DiffInfo
const &patch
,
2364 std::vector
<pkgAcqIndexMergeDiffs
*> const * const allPatches
)
2365 : pkgAcqBaseIndex(Owner
, TransactionManager
, Target
), d(NULL
),
2366 patch(patch
), allPatches(allPatches
), State(StateFetchDiff
)
2368 Debug
= _config
->FindB("Debug::pkgAcquire::Diffs",false);
2371 Description
= Target
.Description
;
2372 Desc
.ShortDesc
= Target
.ShortDesc
;
2373 Desc
.URI
= Target
.URI
+ ".diff/" + patch
.file
+ ".gz";
2374 Desc
.Description
= Description
+ " " + patch
.file
+ ".pdiff";
2375 DestFile
= GetPartialFileNameFromURI(Desc
.URI
);
2378 std::clog
<< "pkgAcqIndexMergeDiffs: " << Desc
.URI
<< std::endl
;
2383 void pkgAcqIndexMergeDiffs::Failed(string
const &Message
,pkgAcquire::MethodConfig
const * const Cnf
)/*{{{*/
2386 std::clog
<< "pkgAcqIndexMergeDiffs failed: " << Desc
.URI
<< " with " << Message
<< std::endl
;
2388 Item::Failed(Message
,Cnf
);
2391 // check if we are the first to fail, otherwise we are done here
2392 State
= StateDoneDiff
;
2393 for (std::vector
<pkgAcqIndexMergeDiffs
*>::const_iterator I
= allPatches
->begin();
2394 I
!= allPatches
->end(); ++I
)
2395 if ((*I
)->State
== StateErrorDiff
)
2398 // first failure means we should fallback
2399 State
= StateErrorDiff
;
2401 std::clog
<< "Falling back to normal index file acquire" << std::endl
;
2402 RenameOnError(PDiffError
);
2403 std::string
const patchname
= GetPartialFileNameFromURI(Desc
.URI
);
2404 if (RealFileExists(patchname
))
2405 Rename(patchname
, patchname
+ ".FAILED");
2406 std::string
const UnpatchedFile
= GetExistingFilename(GetPartialFileNameFromURI(Target
.URI
));
2407 if (UnpatchedFile
.empty() == false && FileExists(UnpatchedFile
))
2408 Rename(UnpatchedFile
, UnpatchedFile
+ ".FAILED");
2410 new pkgAcqIndex(Owner
, TransactionManager
, Target
);
2413 void pkgAcqIndexMergeDiffs::Done(string
const &Message
, HashStringList
const &Hashes
, /*{{{*/
2414 pkgAcquire::MethodConfig
const * const Cnf
)
2417 std::clog
<< "pkgAcqIndexMergeDiffs::Done(): " << Desc
.URI
<< std::endl
;
2419 Item::Done(Message
, Hashes
, Cnf
);
2421 std::string
const UncompressedUnpatchedFile
= GetPartialFileNameFromURI(Target
.URI
);
2422 std::string
const UnpatchedFile
= GetExistingFilename(UncompressedUnpatchedFile
);
2423 std::string
const PatchFile
= GetMergeDiffsPatchFileName(UnpatchedFile
, patch
.file
);
2424 std::string
const PatchedFile
= GetKeepCompressedFileName(UncompressedUnpatchedFile
, Target
);
2428 case StateFetchDiff
:
2429 Rename(DestFile
, PatchFile
);
2431 // check if this is the last completed diff
2432 State
= StateDoneDiff
;
2433 for (std::vector
<pkgAcqIndexMergeDiffs
*>::const_iterator I
= allPatches
->begin();
2434 I
!= allPatches
->end(); ++I
)
2435 if ((*I
)->State
!= StateDoneDiff
)
2438 std::clog
<< "Not the last done diff in the batch: " << Desc
.URI
<< std::endl
;
2441 // this is the last completed diff, so we are ready to apply now
2442 DestFile
= GetKeepCompressedFileName(UncompressedUnpatchedFile
+ "-patched", Target
);
2444 std::clog
<< "Sending to rred method: " << UnpatchedFile
<< std::endl
;
2445 State
= StateApplyDiff
;
2447 Desc
.URI
= "rred:" + UnpatchedFile
;
2449 SetActiveSubprocess("rred");
2451 case StateApplyDiff
:
2452 // success in download & apply all diffs, finialize and clean up
2454 std::clog
<< "Queue patched file in place: " << std::endl
2455 << DestFile
<< " -> " << PatchedFile
<< std::endl
;
2457 // queue for copy by the transaction manager
2458 TransactionManager
->TransactionStageCopy(this, DestFile
, GetKeepCompressedFileName(GetFinalFilename(), Target
));
2460 // ensure the ed's are gone regardless of list-cleanup
2461 for (std::vector
<pkgAcqIndexMergeDiffs
*>::const_iterator I
= allPatches
->begin();
2462 I
!= allPatches
->end(); ++I
)
2463 RemoveFile("pkgAcqIndexMergeDiffs::Done", GetMergeDiffsPatchFileName(UnpatchedFile
, (*I
)->patch
.file
));
2464 RemoveFile("pkgAcqIndexMergeDiffs::Done", UnpatchedFile
);
2469 std::clog
<< "allDone: " << DestFile
<< "\n" << std::endl
;
2471 case StateDoneDiff
: _error
->Fatal("Done called for %s which is in an invalid Done state", PatchFile
.c_str()); break;
2472 case StateErrorDiff
: _error
->Fatal("Done called for %s which is in an invalid Error state", PatchFile
.c_str()); break;
2476 std::string
pkgAcqIndexMergeDiffs::Custom600Headers() const /*{{{*/
2478 if(State
!= StateApplyDiff
)
2479 return pkgAcqBaseIndex::Custom600Headers();
2480 std::ostringstream patchhashes
;
2481 unsigned int seen_patches
= 0;
2482 for (std::vector
<pkgAcqIndexMergeDiffs
*>::const_iterator I
= allPatches
->begin();
2483 I
!= allPatches
->end(); ++I
)
2485 HashStringList
const ExpectedHashes
= (*I
)->patch
.patch_hashes
;
2486 for (HashStringList::const_iterator hs
= ExpectedHashes
.begin(); hs
!= ExpectedHashes
.end(); ++hs
)
2487 patchhashes
<< "\nPatch-" << seen_patches
<< "-" << hs
->HashType() << "-Hash: " << hs
->HashValue();
2490 patchhashes
<< pkgAcqBaseIndex::Custom600Headers();
2491 return patchhashes
.str();
2494 pkgAcqIndexMergeDiffs::~pkgAcqIndexMergeDiffs() {}
2496 // AcqIndex::AcqIndex - Constructor /*{{{*/
2497 pkgAcqIndex::pkgAcqIndex(pkgAcquire
* const Owner
,
2498 pkgAcqMetaClearSig
* const TransactionManager
,
2499 IndexTarget
const &Target
)
2500 : pkgAcqBaseIndex(Owner
, TransactionManager
, Target
), d(NULL
), Stage(STAGE_DOWNLOAD
),
2501 CompressionExtensions(Target
.Option(IndexTarget::COMPRESSIONTYPES
))
2503 Init(Target
.URI
, Target
.Description
, Target
.ShortDesc
);
2505 if(_config
->FindB("Debug::Acquire::Transaction", false) == true)
2506 std::clog
<< "New pkgIndex with TransactionManager "
2507 << TransactionManager
<< std::endl
;
2510 // AcqIndex::Init - defered Constructor /*{{{*/
2511 static void NextCompressionExtension(std::string
&CurrentCompressionExtension
, std::string
&CompressionExtensions
, bool const preview
)
2513 size_t const nextExt
= CompressionExtensions
.find(' ');
2514 if (nextExt
== std::string::npos
)
2516 CurrentCompressionExtension
= CompressionExtensions
;
2517 if (preview
== false)
2518 CompressionExtensions
.clear();
2522 CurrentCompressionExtension
= CompressionExtensions
.substr(0, nextExt
);
2523 if (preview
== false)
2524 CompressionExtensions
= CompressionExtensions
.substr(nextExt
+1);
2527 void pkgAcqIndex::Init(string
const &URI
, string
const &URIDesc
,
2528 string
const &ShortDesc
)
2530 Stage
= STAGE_DOWNLOAD
;
2532 DestFile
= GetPartialFileNameFromURI(URI
);
2533 NextCompressionExtension(CurrentCompressionExtension
, CompressionExtensions
, false);
2535 if (CurrentCompressionExtension
== "uncompressed")
2539 else if (CurrentCompressionExtension
== "by-hash")
2541 NextCompressionExtension(CurrentCompressionExtension
, CompressionExtensions
, true);
2542 if(unlikely(TransactionManager
->MetaIndexParser
== NULL
|| CurrentCompressionExtension
.empty()))
2544 if (CurrentCompressionExtension
!= "uncompressed")
2546 Desc
.URI
= URI
+ '.' + CurrentCompressionExtension
;
2547 DestFile
= DestFile
+ '.' + CurrentCompressionExtension
;
2550 HashStringList
const Hashes
= GetExpectedHashes();
2551 HashString
const * const TargetHash
= Hashes
.find(NULL
);
2552 if (unlikely(TargetHash
== nullptr))
2554 std::string
const ByHash
= "/by-hash/" + TargetHash
->HashType() + "/" + TargetHash
->HashValue();
2555 size_t const trailing_slash
= Desc
.URI
.find_last_of("/");
2556 if (unlikely(trailing_slash
== std::string::npos
))
2558 Desc
.URI
= Desc
.URI
.replace(
2560 Desc
.URI
.substr(trailing_slash
+1).size()+1,
2563 else if (unlikely(CurrentCompressionExtension
.empty()))
2567 Desc
.URI
= URI
+ '.' + CurrentCompressionExtension
;
2568 DestFile
= DestFile
+ '.' + CurrentCompressionExtension
;
2572 Desc
.Description
= URIDesc
;
2574 Desc
.ShortDesc
= ShortDesc
;
2579 // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/
2580 // ---------------------------------------------------------------------
2581 /* The only header we use is the last-modified header. */
2582 string
pkgAcqIndex::Custom600Headers() const
2585 string msg
= "\nIndex-File: true";
2587 if (TransactionManager
->LastMetaIndexParser
== NULL
)
2589 std::string
const Final
= GetFinalFilename();
2592 if (stat(Final
.c_str(),&Buf
) == 0)
2593 msg
+= "\nLast-Modified: " + TimeRFC1123(Buf
.st_mtime
);
2596 if(Target
.IsOptional
)
2597 msg
+= "\nFail-Ignore: true";
2602 // AcqIndex::Failed - getting the indexfile failed /*{{{*/
2603 void pkgAcqIndex::Failed(string
const &Message
,pkgAcquire::MethodConfig
const * const Cnf
)
2605 Item::Failed(Message
,Cnf
);
2607 // authorisation matches will not be fixed by other compression types
2608 if (Status
!= StatAuthError
)
2610 if (CompressionExtensions
.empty() == false)
2612 Init(Target
.URI
, Desc
.Description
, Desc
.ShortDesc
);
2618 if(Target
.IsOptional
&& GetExpectedHashes().empty() && Stage
== STAGE_DOWNLOAD
)
2621 TransactionManager
->AbortTransaction();
2624 // AcqIndex::ReverifyAfterIMS - Reverify index after an ims-hit /*{{{*/
2625 void pkgAcqIndex::ReverifyAfterIMS()
2627 // update destfile to *not* include the compression extension when doing
2628 // a reverify (as its uncompressed on disk already)
2629 DestFile
= GetCompressedFileName(Target
, GetPartialFileNameFromURI(Target
.URI
), CurrentCompressionExtension
);
2631 // copy FinalFile into partial/ so that we check the hash again
2632 string FinalFile
= GetFinalFilename();
2633 Stage
= STAGE_DECOMPRESS_AND_VERIFY
;
2634 Desc
.URI
= "copy:" + FinalFile
;
2638 // AcqIndex::Done - Finished a fetch /*{{{*/
2639 // ---------------------------------------------------------------------
2640 /* This goes through a number of states.. On the initial fetch the
2641 method could possibly return an alternate filename which points
2642 to the uncompressed version of the file. If this is so the file
2643 is copied into the partial directory. In all other cases the file
2644 is decompressed with a compressed uri. */
2645 void pkgAcqIndex::Done(string
const &Message
,
2646 HashStringList
const &Hashes
,
2647 pkgAcquire::MethodConfig
const * const Cfg
)
2649 Item::Done(Message
,Hashes
,Cfg
);
2653 case STAGE_DOWNLOAD
:
2654 StageDownloadDone(Message
, Hashes
, Cfg
);
2656 case STAGE_DECOMPRESS_AND_VERIFY
:
2657 StageDecompressDone(Message
, Hashes
, Cfg
);
2662 // AcqIndex::StageDownloadDone - Queue for decompress and verify /*{{{*/
2663 void pkgAcqIndex::StageDownloadDone(string
const &Message
, HashStringList
const &,
2664 pkgAcquire::MethodConfig
const * const)
2668 // Handle the unzipd case
2669 std::string FileName
= LookupTag(Message
,"Alt-Filename");
2670 if (FileName
.empty() == false)
2672 Stage
= STAGE_DECOMPRESS_AND_VERIFY
;
2674 if (CurrentCompressionExtension
!= "uncompressed")
2675 DestFile
.erase(DestFile
.length() - (CurrentCompressionExtension
.length() + 1));
2676 Desc
.URI
= "copy:" + FileName
;
2678 SetActiveSubprocess("copy");
2681 FileName
= LookupTag(Message
,"Filename");
2683 // Methods like e.g. "file:" will give us a (compressed) FileName that is
2684 // not the "DestFile" we set, in this case we uncompress from the local file
2685 if (FileName
!= DestFile
&& RealFileExists(DestFile
) == false)
2688 if (Target
.KeepCompressed
== true)
2690 // but if we don't keep the uncompress we copy the compressed file first
2691 Stage
= STAGE_DOWNLOAD
;
2692 Desc
.URI
= "copy:" + FileName
;
2694 SetActiveSubprocess("copy");
2699 // symlinking ensures that the filename can be used for compression detection
2700 // that is e.g. needed for by-hash over file
2701 if (symlink(FileName
.c_str(),DestFile
.c_str()) != 0)
2702 _error
->WarningE("pkgAcqIndex::StageDownloadDone", "Symlinking file %s to %s failed", FileName
.c_str(), DestFile
.c_str());
2705 EraseFileName
= DestFile
;
2706 FileName
= DestFile
;
2711 EraseFileName
= FileName
;
2713 // we need to verify the file against the current Release file again
2714 // on if-modfied-since hit to avoid a stale attack against us
2715 if(StringToBool(LookupTag(Message
,"IMS-Hit"),false) == true)
2717 // The files timestamp matches, reverify by copy into partial/
2723 string decompProg
= "store";
2724 if (Target
.KeepCompressed
== true)
2726 DestFile
= "/dev/null";
2727 EraseFileName
.clear();
2731 if (CurrentCompressionExtension
== "uncompressed")
2732 decompProg
= "copy";
2734 DestFile
.erase(DestFile
.length() - (CurrentCompressionExtension
.length() + 1));
2737 // queue uri for the next stage
2738 Stage
= STAGE_DECOMPRESS_AND_VERIFY
;
2739 Desc
.URI
= decompProg
+ ":" + FileName
;
2741 SetActiveSubprocess(decompProg
);
2744 // AcqIndex::StageDecompressDone - Final verification /*{{{*/
2745 void pkgAcqIndex::StageDecompressDone(string
const &,
2746 HashStringList
const &,
2747 pkgAcquire::MethodConfig
const * const)
2749 if (Target
.KeepCompressed
== true && DestFile
== "/dev/null")
2750 DestFile
= GetPartialFileNameFromURI(Target
.URI
+ '.' + CurrentCompressionExtension
);
2752 // Done, queue for rename on transaction finished
2753 TransactionManager
->TransactionStageCopy(this, DestFile
, GetFinalFilename());
2757 pkgAcqIndex::~pkgAcqIndex() {}
2760 // AcqArchive::AcqArchive - Constructor /*{{{*/
2761 // ---------------------------------------------------------------------
2762 /* This just sets up the initial fetch environment and queues the first
2764 pkgAcqArchive::pkgAcqArchive(pkgAcquire
* const Owner
,pkgSourceList
* const Sources
,
2765 pkgRecords
* const Recs
,pkgCache::VerIterator
const &Version
,
2766 string
&StoreFilename
) :
2767 Item(Owner
), d(NULL
), LocalSource(false), Version(Version
), Sources(Sources
), Recs(Recs
),
2768 StoreFilename(StoreFilename
), Vf(Version
.FileList()),
2771 Retries
= _config
->FindI("Acquire::Retries",0);
2773 if (Version
.Arch() == 0)
2775 _error
->Error(_("I wasn't able to locate a file for the %s package. "
2776 "This might mean you need to manually fix this package. "
2777 "(due to missing arch)"),
2778 Version
.ParentPkg().FullName().c_str());
2782 /* We need to find a filename to determine the extension. We make the
2783 assumption here that all the available sources for this version share
2784 the same extension.. */
2785 // Skip not source sources, they do not have file fields.
2786 for (; Vf
.end() == false; ++Vf
)
2788 if (Vf
.File().Flagged(pkgCache::Flag::NotSource
))
2793 // Does not really matter here.. we are going to fail out below
2794 if (Vf
.end() != true)
2796 // If this fails to get a file name we will bomb out below.
2797 pkgRecords::Parser
&Parse
= Recs
->Lookup(Vf
);
2798 if (_error
->PendingError() == true)
2801 // Generate the final file name as: package_version_arch.foo
2802 StoreFilename
= QuoteString(Version
.ParentPkg().Name(),"_:") + '_' +
2803 QuoteString(Version
.VerStr(),"_:") + '_' +
2804 QuoteString(Version
.Arch(),"_:.") +
2805 "." + flExtension(Parse
.FileName());
2808 // check if we have one trusted source for the package. if so, switch
2809 // to "TrustedOnly" mode - but only if not in AllowUnauthenticated mode
2810 bool const allowUnauth
= _config
->FindB("APT::Get::AllowUnauthenticated", false);
2811 bool const debugAuth
= _config
->FindB("Debug::pkgAcquire::Auth", false);
2812 bool seenUntrusted
= false;
2813 for (pkgCache::VerFileIterator i
= Version
.FileList(); i
.end() == false; ++i
)
2815 pkgIndexFile
*Index
;
2816 if (Sources
->FindIndex(i
.File(),Index
) == false)
2819 if (debugAuth
== true)
2820 std::cerr
<< "Checking index: " << Index
->Describe()
2821 << "(Trusted=" << Index
->IsTrusted() << ")" << std::endl
;
2823 if (Index
->IsTrusted() == true)
2826 if (allowUnauth
== false)
2830 seenUntrusted
= true;
2833 // "allow-unauthenticated" restores apts old fetching behaviour
2834 // that means that e.g. unauthenticated file:// uris are higher
2835 // priority than authenticated http:// uris
2836 if (allowUnauth
== true && seenUntrusted
== true)
2840 if (QueueNext() == false && _error
->PendingError() == false)
2841 _error
->Error(_("Can't find a source to download version '%s' of '%s'"),
2842 Version
.VerStr(), Version
.ParentPkg().FullName(false).c_str());
2845 // AcqArchive::QueueNext - Queue the next file source /*{{{*/
2846 // ---------------------------------------------------------------------
2847 /* This queues the next available file version for download. It checks if
2848 the archive is already available in the cache and stashs the MD5 for
2850 bool pkgAcqArchive::QueueNext()
2852 for (; Vf
.end() == false; ++Vf
)
2854 pkgCache::PkgFileIterator
const PkgF
= Vf
.File();
2855 // Ignore not source sources
2856 if (PkgF
.Flagged(pkgCache::Flag::NotSource
))
2859 // Try to cross match against the source list
2860 pkgIndexFile
*Index
;
2861 if (Sources
->FindIndex(PkgF
, Index
) == false)
2863 LocalSource
= PkgF
.Flagged(pkgCache::Flag::LocalSource
);
2865 // only try to get a trusted package from another source if that source
2867 if(Trusted
&& !Index
->IsTrusted())
2870 // Grab the text package record
2871 pkgRecords::Parser
&Parse
= Recs
->Lookup(Vf
);
2872 if (_error
->PendingError() == true)
2875 string PkgFile
= Parse
.FileName();
2876 ExpectedHashes
= Parse
.Hashes();
2878 if (PkgFile
.empty() == true)
2879 return _error
->Error(_("The package index files are corrupted. No Filename: "
2880 "field for package %s."),
2881 Version
.ParentPkg().Name());
2883 Desc
.URI
= Index
->ArchiveURI(PkgFile
);
2884 Desc
.Description
= Index
->ArchiveInfo(Version
);
2886 Desc
.ShortDesc
= Version
.ParentPkg().FullName(true);
2888 // See if we already have the file. (Legacy filenames)
2889 FileSize
= Version
->Size
;
2890 string FinalFile
= _config
->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile
);
2892 if (stat(FinalFile
.c_str(),&Buf
) == 0)
2894 // Make sure the size matches
2895 if ((unsigned long long)Buf
.st_size
== Version
->Size
)
2900 StoreFilename
= DestFile
= FinalFile
;
2904 /* Hmm, we have a file and its size does not match, this means it is
2905 an old style mismatched arch */
2906 RemoveFile("pkgAcqArchive::QueueNext", FinalFile
);
2909 // Check it again using the new style output filenames
2910 FinalFile
= _config
->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename
);
2911 if (stat(FinalFile
.c_str(),&Buf
) == 0)
2913 // Make sure the size matches
2914 if ((unsigned long long)Buf
.st_size
== Version
->Size
)
2919 StoreFilename
= DestFile
= FinalFile
;
2923 /* Hmm, we have a file and its size does not match, this shouldn't
2925 RemoveFile("pkgAcqArchive::QueueNext", FinalFile
);
2928 DestFile
= _config
->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename
);
2930 // Check the destination file
2931 if (stat(DestFile
.c_str(),&Buf
) == 0)
2933 // Hmm, the partial file is too big, erase it
2934 if ((unsigned long long)Buf
.st_size
> Version
->Size
)
2935 RemoveFile("pkgAcqArchive::QueueNext", DestFile
);
2937 PartialSize
= Buf
.st_size
;
2940 // Disables download of archives - useful if no real installation follows,
2941 // e.g. if we are just interested in proposed installation order
2942 if (_config
->FindB("Debug::pkgAcqArchive::NoQueue", false) == true)
2947 StoreFilename
= DestFile
= FinalFile
;
2961 // AcqArchive::Done - Finished fetching /*{{{*/
2962 // ---------------------------------------------------------------------
2964 void pkgAcqArchive::Done(string
const &Message
, HashStringList
const &Hashes
,
2965 pkgAcquire::MethodConfig
const * const Cfg
)
2967 Item::Done(Message
, Hashes
, Cfg
);
2969 // Grab the output filename
2970 std::string
const FileName
= LookupTag(Message
,"Filename");
2971 if (DestFile
!= FileName
&& RealFileExists(DestFile
) == false)
2973 StoreFilename
= DestFile
= FileName
;
2979 // Done, move it into position
2980 string
const FinalFile
= GetFinalFilename();
2981 Rename(DestFile
,FinalFile
);
2982 StoreFilename
= DestFile
= FinalFile
;
2986 // AcqArchive::Failed - Failure handler /*{{{*/
2987 // ---------------------------------------------------------------------
2988 /* Here we try other sources */
2989 void pkgAcqArchive::Failed(string
const &Message
,pkgAcquire::MethodConfig
const * const Cnf
)
2991 Item::Failed(Message
,Cnf
);
2993 /* We don't really want to retry on failed media swaps, this prevents
2994 that. An interesting observation is that permanent failures are not
2996 if (Cnf
->Removable
== true &&
2997 StringToBool(LookupTag(Message
,"Transient-Failure"),false) == true)
2999 // Vf = Version.FileList();
3000 while (Vf
.end() == false) ++Vf
;
3001 StoreFilename
= string();
3006 if (QueueNext() == false)
3008 // This is the retry counter
3010 Cnf
->LocalOnly
== false &&
3011 StringToBool(LookupTag(Message
,"Transient-Failure"),false) == true)
3014 Vf
= Version
.FileList();
3015 if (QueueNext() == true)
3019 StoreFilename
= string();
3024 APT_PURE
bool pkgAcqArchive::IsTrusted() const /*{{{*/
3029 void pkgAcqArchive::Finished() /*{{{*/
3031 if (Status
== pkgAcquire::Item::StatDone
&&
3034 StoreFilename
= string();
3037 std::string
pkgAcqArchive::DescURI() const /*{{{*/
3042 std::string
pkgAcqArchive::ShortDesc() const /*{{{*/
3044 return Desc
.ShortDesc
;
3047 pkgAcqArchive::~pkgAcqArchive() {}
3049 // AcqChangelog::pkgAcqChangelog - Constructors /*{{{*/
3050 pkgAcqChangelog::pkgAcqChangelog(pkgAcquire
* const Owner
, pkgCache::VerIterator
const &Ver
,
3051 std::string
const &DestDir
, std::string
const &DestFilename
) :
3052 pkgAcquire::Item(Owner
), d(NULL
), SrcName(Ver
.SourcePkgName()), SrcVersion(Ver
.SourceVerStr())
3054 Desc
.URI
= URI(Ver
);
3055 Init(DestDir
, DestFilename
);
3057 // some parameters are char* here as they come likely from char* interfaces – which can also return NULL
3058 pkgAcqChangelog::pkgAcqChangelog(pkgAcquire
* const Owner
, pkgCache::RlsFileIterator
const &RlsFile
,
3059 char const * const Component
, char const * const SrcName
, char const * const SrcVersion
,
3060 const string
&DestDir
, const string
&DestFilename
) :
3061 pkgAcquire::Item(Owner
), d(NULL
), SrcName(SrcName
), SrcVersion(SrcVersion
)
3063 Desc
.URI
= URI(RlsFile
, Component
, SrcName
, SrcVersion
);
3064 Init(DestDir
, DestFilename
);
3066 pkgAcqChangelog::pkgAcqChangelog(pkgAcquire
* const Owner
,
3067 std::string
const &URI
, char const * const SrcName
, char const * const SrcVersion
,
3068 const string
&DestDir
, const string
&DestFilename
) :
3069 pkgAcquire::Item(Owner
), d(NULL
), SrcName(SrcName
), SrcVersion(SrcVersion
)
3072 Init(DestDir
, DestFilename
);
3074 void pkgAcqChangelog::Init(std::string
const &DestDir
, std::string
const &DestFilename
)
3076 if (Desc
.URI
.empty())
3079 // TRANSLATOR: %s=%s is sourcename=sourceversion, e.g. apt=1.1
3080 strprintf(ErrorText
, _("Changelog unavailable for %s=%s"), SrcName
.c_str(), SrcVersion
.c_str());
3081 // Let the error message print something sensible rather than "Failed to fetch /"
3082 if (DestFilename
.empty())
3083 DestFile
= SrcName
+ ".changelog";
3085 DestFile
= DestFilename
;
3086 Desc
.URI
= "changelog:/" + DestFile
;
3090 if (DestDir
.empty())
3092 std::string
const SandboxUser
= _config
->Find("APT::Sandbox::User");
3093 std::string
const systemTemp
= GetTempDir(SandboxUser
);
3095 snprintf(tmpname
, sizeof(tmpname
), "%s/apt-changelog-XXXXXX", systemTemp
.c_str());
3096 if (NULL
== mkdtemp(tmpname
))
3098 _error
->Errno("mkdtemp", "mkdtemp failed in changelog acquire of %s %s", SrcName
.c_str(), SrcVersion
.c_str());
3102 DestFile
= TemporaryDirectory
= tmpname
;
3104 ChangeOwnerAndPermissionOfFile("Item::QueueURI", DestFile
.c_str(),
3105 SandboxUser
.c_str(), "root", 0700);
3110 if (DestFilename
.empty())
3111 DestFile
= flCombine(DestFile
, SrcName
+ ".changelog");
3113 DestFile
= flCombine(DestFile
, DestFilename
);
3115 Desc
.ShortDesc
= "Changelog";
3116 strprintf(Desc
.Description
, "%s %s %s Changelog", URI::SiteOnly(Desc
.URI
).c_str(), SrcName
.c_str(), SrcVersion
.c_str());
3121 std::string
pkgAcqChangelog::URI(pkgCache::VerIterator
const &Ver
) /*{{{*/
3123 char const * const SrcName
= Ver
.SourcePkgName();
3124 char const * const SrcVersion
= Ver
.SourceVerStr();
3125 pkgCache::PkgFileIterator PkgFile
;
3126 // find the first source for this version which promises a changelog
3127 for (pkgCache::VerFileIterator VF
= Ver
.FileList(); VF
.end() == false; ++VF
)
3129 pkgCache::PkgFileIterator
const PF
= VF
.File();
3130 if (PF
.Flagged(pkgCache::Flag::NotSource
) || PF
->Release
== 0)
3133 pkgCache::RlsFileIterator
const RF
= PF
.ReleaseFile();
3134 std::string
const uri
= URI(RF
, PF
.Component(), SrcName
, SrcVersion
);
3141 std::string
pkgAcqChangelog::URITemplate(pkgCache::RlsFileIterator
const &Rls
)
3143 if (Rls
.end() == true || (Rls
->Label
== 0 && Rls
->Origin
== 0))
3145 std::string
const serverConfig
= "Acquire::Changelogs::URI";
3147 #define APT_EMPTY_SERVER \
3148 if (server.empty() == false) \
3150 if (server != "no") \
3154 #define APT_CHECK_SERVER(X, Y) \
3157 std::string const specialServerConfig = serverConfig + "::" + Y + #X + "::" + Rls.X(); \
3158 server = _config->Find(specialServerConfig); \
3161 // this way e.g. Debian-Security can fallback to Debian
3162 APT_CHECK_SERVER(Label
, "Override::")
3163 APT_CHECK_SERVER(Origin
, "Override::")
3165 if (RealFileExists(Rls
.FileName()))
3167 _error
->PushToStack();
3169 /* This can be costly. A caller wanting to get millions of URIs might
3170 want to do this on its own once and use Override settings.
3171 We don't do this here as Origin/Label are not as unique as they
3172 should be so this could produce request order-dependent anomalies */
3173 if (OpenMaybeClearSignedFile(Rls
.FileName(), rf
) == true)
3175 pkgTagFile
TagFile(&rf
, rf
.Size());
3176 pkgTagSection Section
;
3177 if (TagFile
.Step(Section
) == true)
3178 server
= Section
.FindS("Changelogs");
3180 _error
->RevertToStack();
3184 APT_CHECK_SERVER(Label
, "")
3185 APT_CHECK_SERVER(Origin
, "")
3186 #undef APT_CHECK_SERVER
3187 #undef APT_EMPTY_SERVER
3190 std::string
pkgAcqChangelog::URI(pkgCache::RlsFileIterator
const &Rls
,
3191 char const * const Component
, char const * const SrcName
,
3192 char const * const SrcVersion
)
3194 return URI(URITemplate(Rls
), Component
, SrcName
, SrcVersion
);
3196 std::string
pkgAcqChangelog::URI(std::string
const &Template
,
3197 char const * const Component
, char const * const SrcName
,
3198 char const * const SrcVersion
)
3200 if (Template
.find("@CHANGEPATH@") == std::string::npos
)
3203 // the path is: COMPONENT/SRC/SRCNAME/SRCNAME_SRCVER, e.g. main/a/apt/1.1 or contrib/liba/libapt/2.0
3204 std::string Src
= SrcName
;
3205 std::string path
= APT::String::Startswith(SrcName
, "lib") ? Src
.substr(0, 4) : Src
.substr(0,1);
3206 path
.append("/").append(Src
).append("/");
3207 path
.append(Src
).append("_").append(StripEpoch(SrcVersion
));
3208 // we omit component for releases without one (= flat-style repositories)
3209 if (Component
!= NULL
&& strlen(Component
) != 0)
3210 path
= std::string(Component
) + "/" + path
;
3212 return SubstVar(Template
, "@CHANGEPATH@", path
);
3215 // AcqChangelog::Failed - Failure handler /*{{{*/
3216 void pkgAcqChangelog::Failed(string
const &Message
, pkgAcquire::MethodConfig
const * const Cnf
)
3218 Item::Failed(Message
,Cnf
);
3220 std::string errText
;
3221 // TRANSLATOR: %s=%s is sourcename=sourceversion, e.g. apt=1.1
3222 strprintf(errText
, _("Changelog unavailable for %s=%s"), SrcName
.c_str(), SrcVersion
.c_str());
3224 // Error is probably something techy like 404 Not Found
3225 if (ErrorText
.empty())
3226 ErrorText
= errText
;
3228 ErrorText
= errText
+ " (" + ErrorText
+ ")";
3232 // AcqChangelog::Done - Item downloaded OK /*{{{*/
3233 void pkgAcqChangelog::Done(string
const &Message
,HashStringList
const &CalcHashes
,
3234 pkgAcquire::MethodConfig
const * const Cnf
)
3236 Item::Done(Message
,CalcHashes
,Cnf
);
3241 pkgAcqChangelog::~pkgAcqChangelog() /*{{{*/
3243 if (TemporaryDirectory
.empty() == false)
3245 RemoveFile("~pkgAcqChangelog", DestFile
);
3246 rmdir(TemporaryDirectory
.c_str());
3251 // AcqFile::pkgAcqFile - Constructor /*{{{*/
3252 pkgAcqFile::pkgAcqFile(pkgAcquire
* const Owner
,string
const &URI
, HashStringList
const &Hashes
,
3253 unsigned long long const Size
,string
const &Dsc
,string
const &ShortDesc
,
3254 const string
&DestDir
, const string
&DestFilename
,
3255 bool const IsIndexFile
) :
3256 Item(Owner
), d(NULL
), IsIndexFile(IsIndexFile
), ExpectedHashes(Hashes
)
3258 Retries
= _config
->FindI("Acquire::Retries",0);
3260 if(!DestFilename
.empty())
3261 DestFile
= DestFilename
;
3262 else if(!DestDir
.empty())
3263 DestFile
= DestDir
+ "/" + flNotDir(URI
);
3265 DestFile
= flNotDir(URI
);
3269 Desc
.Description
= Dsc
;
3272 // Set the short description to the archive component
3273 Desc
.ShortDesc
= ShortDesc
;
3275 // Get the transfer sizes
3278 if (stat(DestFile
.c_str(),&Buf
) == 0)
3280 // Hmm, the partial file is too big, erase it
3281 if ((Size
> 0) && (unsigned long long)Buf
.st_size
> Size
)
3282 RemoveFile("pkgAcqFile", DestFile
);
3284 PartialSize
= Buf
.st_size
;
3290 // AcqFile::Done - Item downloaded OK /*{{{*/
3291 void pkgAcqFile::Done(string
const &Message
,HashStringList
const &CalcHashes
,
3292 pkgAcquire::MethodConfig
const * const Cnf
)
3294 Item::Done(Message
,CalcHashes
,Cnf
);
3296 std::string
const FileName
= LookupTag(Message
,"Filename");
3299 // The files timestamp matches
3300 if (StringToBool(LookupTag(Message
,"IMS-Hit"),false) == true)
3303 // We have to copy it into place
3304 if (RealFileExists(DestFile
.c_str()) == false)
3307 if (_config
->FindB("Acquire::Source-Symlinks",true) == false ||
3308 Cnf
->Removable
== true)
3310 Desc
.URI
= "copy:" + FileName
;
3315 // Erase the file if it is a symlink so we can overwrite it
3317 if (lstat(DestFile
.c_str(),&St
) == 0)
3319 if (S_ISLNK(St
.st_mode
) != 0)
3320 RemoveFile("pkgAcqFile::Done", DestFile
);
3324 if (symlink(FileName
.c_str(),DestFile
.c_str()) != 0)
3326 _error
->PushToStack();
3327 _error
->Errno("pkgAcqFile::Done", "Symlinking file %s failed", DestFile
.c_str());
3328 std::stringstream msg
;
3329 _error
->DumpErrors(msg
, GlobalError::DEBUG
, false);
3330 _error
->RevertToStack();
3331 ErrorText
= msg
.str();
3338 // AcqFile::Failed - Failure handler /*{{{*/
3339 // ---------------------------------------------------------------------
3340 /* Here we try other sources */
3341 void pkgAcqFile::Failed(string
const &Message
, pkgAcquire::MethodConfig
const * const Cnf
)
3343 Item::Failed(Message
,Cnf
);
3345 // This is the retry counter
3347 Cnf
->LocalOnly
== false &&
3348 StringToBool(LookupTag(Message
,"Transient-Failure"),false) == true)
3358 string
pkgAcqFile::Custom600Headers() const /*{{{*/
3361 return "\nIndex-File: true";
3365 pkgAcqFile::~pkgAcqFile() {}