1 // -*- mode: cpp; mode: fold -*-
3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
4 /* ######################################################################
6 Acquire Item - Item to acquire
8 Each item can download to exactly one file at a time. This means you
9 cannot create an item that fetches two uri's to two files at the same
10 time. The pkgAcqIndex class creates a second class upon instantiation
11 to fetch the other index files because of this.
13 ##################################################################### */
15 // Include Files /*{{{*/
18 #include <apt-pkg/acquire-item.h>
19 #include <apt-pkg/configuration.h>
20 #include <apt-pkg/aptconfiguration.h>
21 #include <apt-pkg/sourcelist.h>
22 #include <apt-pkg/error.h>
23 #include <apt-pkg/strutl.h>
24 #include <apt-pkg/fileutl.h>
25 #include <apt-pkg/sha1.h>
26 #include <apt-pkg/tagfile.h>
27 #include <apt-pkg/indexrecords.h>
28 #include <apt-pkg/acquire.h>
29 #include <apt-pkg/hashes.h>
30 #include <apt-pkg/indexfile.h>
31 #include <apt-pkg/pkgcache.h>
32 #include <apt-pkg/cacheiterators.h>
33 #include <apt-pkg/pkgrecords.h>
47 #include <sys/types.h>
56 static void printHashSumComparision(std::string
const &URI
, HashStringList
const &Expected
, HashStringList
const &Actual
) /*{{{*/
58 if (_config
->FindB("Debug::Acquire::HashSumMismatch", false) == false)
60 std::cerr
<< std::endl
<< URI
<< ":" << std::endl
<< " Expected Hash: " << std::endl
;
61 for (HashStringList::const_iterator hs
= Expected
.begin(); hs
!= Expected
.end(); ++hs
)
62 std::cerr
<< "\t- " << hs
->toStr() << std::endl
;
63 std::cerr
<< " Actual Hash: " << std::endl
;
64 for (HashStringList::const_iterator hs
= Actual
.begin(); hs
!= Actual
.end(); ++hs
)
65 std::cerr
<< "\t- " << hs
->toStr() << std::endl
;
68 static void ChangeOwnerAndPermissionOfFile(char const * const requester
, char const * const file
, char const * const user
, char const * const group
, mode_t
const mode
) /*{{{*/
70 // ensure the file is owned by root and has good permissions
71 struct passwd
const * const pw
= getpwnam(user
);
72 struct group
const * const gr
= getgrnam(group
);
73 if (getuid() == 0) // if we aren't root, we can't chown, so don't try it
75 if (pw
!= NULL
&& gr
!= NULL
&& chown(file
, pw
->pw_uid
, gr
->gr_gid
) != 0)
76 _error
->WarningE(requester
, "chown to %s:%s of file %s failed", user
, group
, file
);
78 if (chmod(file
, mode
) != 0)
79 _error
->WarningE(requester
, "chmod 0%o of file %s failed", mode
, file
);
82 static std::string
GetPartialFileName(std::string
const &file
) /*{{{*/
84 std::string DestFile
= _config
->FindDir("Dir::State::lists") + "partial/";
89 static std::string
GetPartialFileNameFromURI(std::string
const &uri
) /*{{{*/
91 return GetPartialFileName(URItoFileName(uri
));
94 static std::string
GetCompressedFileName(std::string
const &URI
, std::string
const &Name
, std::string
const &Ext
) /*{{{*/
96 if (Ext
.empty() || Ext
== "uncompressed")
99 // do not reverify cdrom sources as apt-cdrom may rewrite the Packages
100 // file when its doing the indexcopy
101 if (URI
.substr(0,6) == "cdrom:")
104 // adjust DestFile if its compressed on disk
105 if (_config
->FindB("Acquire::GzipIndexes",false) == true)
106 return Name
+ '.' + Ext
;
110 static bool AllowInsecureRepositories(indexRecords
const * const MetaIndexParser
, pkgAcqMetaBase
* const TransactionManager
, pkgAcquire::Item
* const I
) /*{{{*/
112 if(MetaIndexParser
->IsAlwaysTrusted() || _config
->FindB("Acquire::AllowInsecureRepositories") == true)
115 _error
->Error(_("Use --allow-insecure-repositories to force the update"));
116 TransactionManager
->AbortTransaction();
117 I
->Status
= pkgAcquire::Item::StatError
;
123 // Acquire::Item::Item - Constructor /*{{{*/
125 #pragma GCC diagnostic push
126 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
128 pkgAcquire::Item::Item(pkgAcquire
*Owner
,
129 HashStringList
const &ExpectedHashes
,
130 pkgAcqMetaBase
*TransactionManager
)
131 : Owner(Owner
), FileSize(0), PartialSize(0), Mode(0), ID(0), Complete(false),
132 Local(false), QueueCounter(0), TransactionManager(TransactionManager
),
133 ExpectedAdditionalItems(0), ExpectedHashes(ExpectedHashes
)
137 if(TransactionManager
!= NULL
)
138 TransactionManager
->Add(this);
141 #pragma GCC diagnostic pop
144 // Acquire::Item::~Item - Destructor /*{{{*/
145 // ---------------------------------------------------------------------
147 pkgAcquire::Item::~Item()
152 // Acquire::Item::Failed - Item failed to download /*{{{*/
153 // ---------------------------------------------------------------------
154 /* We return to an idle state if there are still other queues that could
156 void pkgAcquire::Item::Failed(string Message
,pkgAcquire::MethodConfig
*Cnf
)
159 ErrorText
= LookupTag(Message
,"Message");
160 UsedMirror
= LookupTag(Message
,"UsedMirror");
161 if (QueueCounter
<= 1)
163 /* This indicates that the file is not available right now but might
164 be sometime later. If we do a retry cycle then this should be
166 if (Cnf
!= NULL
&& Cnf
->LocalOnly
== true &&
167 StringToBool(LookupTag(Message
,"Transient-Failure"),false) == true)
182 string FailReason
= LookupTag(Message
, "FailReason");
183 if(FailReason
== "MaximumSizeExceeded")
184 Rename(DestFile
, DestFile
+".FAILED");
186 // report mirror failure back to LP if we actually use a mirror
187 if(FailReason
.size() != 0)
188 ReportMirrorFailure(FailReason
);
190 ReportMirrorFailure(ErrorText
);
193 // Acquire::Item::Start - Item has begun to download /*{{{*/
194 // ---------------------------------------------------------------------
195 /* Stash status and the file size. Note that setting Complete means
196 sub-phases of the acquire process such as decompresion are operating */
197 void pkgAcquire::Item::Start(string
/*Message*/,unsigned long long Size
)
199 Status
= StatFetching
;
200 if (FileSize
== 0 && Complete
== false)
204 // Acquire::Item::Done - Item downloaded OK /*{{{*/
205 // ---------------------------------------------------------------------
207 void pkgAcquire::Item::Done(string Message
,unsigned long long Size
,HashStringList
const &/*Hash*/,
208 pkgAcquire::MethodConfig
* /*Cnf*/)
210 // We just downloaded something..
211 string FileName
= LookupTag(Message
,"Filename");
212 UsedMirror
= LookupTag(Message
,"UsedMirror");
213 if (Complete
== false && !Local
&& FileName
== DestFile
)
216 Owner
->Log
->Fetched(Size
,atoi(LookupTag(Message
,"Resume-Point","0").c_str()));
222 ErrorText
= string();
223 Owner
->Dequeue(this);
226 // Acquire::Item::Rename - Rename a file /*{{{*/
227 // ---------------------------------------------------------------------
228 /* This helper function is used by a lot of item methods as their final
230 bool pkgAcquire::Item::Rename(string From
,string To
)
232 if (rename(From
.c_str(),To
.c_str()) == 0)
236 strprintf(S
, _("rename failed, %s (%s -> %s)."), strerror(errno
),
237 From
.c_str(),To
.c_str());
243 void pkgAcquire::Item::QueueURI(ItemDesc
&Item
) /*{{{*/
245 if (RealFileExists(DestFile
))
247 std::string SandboxUser
= _config
->Find("APT::Sandbox::User");
248 ChangeOwnerAndPermissionOfFile("Item::QueueURI", DestFile
.c_str(),
249 SandboxUser
.c_str(), "root", 0600);
251 Owner
->Enqueue(Item
);
254 void pkgAcquire::Item::Dequeue() /*{{{*/
256 Owner
->Dequeue(this);
259 bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState
const error
)/*{{{*/
261 if(FileExists(DestFile
))
262 Rename(DestFile
, DestFile
+ ".FAILED");
266 case HashSumMismatch
:
267 ErrorText
= _("Hash Sum mismatch");
268 Status
= StatAuthError
;
269 ReportMirrorFailure("HashChecksumFailure");
272 ErrorText
= _("Size mismatch");
273 Status
= StatAuthError
;
274 ReportMirrorFailure("SizeFailure");
277 ErrorText
= _("Invalid file format");
279 // do not report as usually its not the mirrors fault, but Portal/Proxy
282 ErrorText
= _("Signature error");
286 ErrorText
= _("Does not start with a cleartext signature");
293 void pkgAcquire::Item::SetActiveSubprocess(const std::string
&subprocess
)/*{{{*/
295 ActiveSubprocess
= subprocess
;
297 #pragma GCC diagnostic push
298 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
300 Mode
= ActiveSubprocess
.c_str();
302 #pragma GCC diagnostic pop
306 // Acquire::Item::ReportMirrorFailure /*{{{*/
307 // ---------------------------------------------------------------------
308 void pkgAcquire::Item::ReportMirrorFailure(string FailCode
)
310 // we only act if a mirror was used at all
311 if(UsedMirror
.empty())
314 std::cerr
<< "\nReportMirrorFailure: "
316 << " Uri: " << DescURI()
318 << FailCode
<< std::endl
;
320 string report
= _config
->Find("Methods::Mirror::ProblemReporting",
321 "/usr/lib/apt/apt-report-mirror-failure");
322 if(!FileExists(report
))
325 std::vector
<char const*> Args
;
326 Args
.push_back(report
.c_str());
327 Args
.push_back(UsedMirror
.c_str());
328 Args
.push_back(DescURI().c_str());
329 Args
.push_back(FailCode
.c_str());
330 Args
.push_back(NULL
);
332 pid_t pid
= ExecFork();
335 _error
->Error("ReportMirrorFailure Fork failed");
340 execvp(Args
[0], (char**)Args
.data());
341 std::cerr
<< "Could not exec " << Args
[0] << std::endl
;
344 if(!ExecWait(pid
, "report-mirror-failure"))
346 _error
->Warning("Couldn't report problem to '%s'",
347 _config
->Find("Methods::Mirror::ProblemReporting").c_str());
351 // AcqDiffIndex::AcqDiffIndex - Constructor /*{{{*/
352 // ---------------------------------------------------------------------
353 /* Get the DiffIndex file first and see if there are patches available
354 * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
355 * patches. If anything goes wrong in that process, it will fall back to
356 * the original packages file
358 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire
*Owner
,
359 pkgAcqMetaBase
*TransactionManager
,
360 IndexTarget
const * const Target
,
361 HashStringList
const &ExpectedHashes
,
362 indexRecords
*MetaIndexParser
)
363 : pkgAcqBaseIndex(Owner
, TransactionManager
, Target
, ExpectedHashes
,
364 MetaIndexParser
), PackagesFileReadyInPartial(false)
367 Debug
= _config
->FindB("Debug::pkgAcquire::Diffs",false);
369 RealURI
= Target
->URI
;
371 Desc
.Description
= Target
->Description
+ ".diff/Index";
372 Desc
.ShortDesc
= Target
->ShortDesc
;
373 Desc
.URI
= Target
->URI
+ ".diff/Index";
375 DestFile
= GetPartialFileNameFromURI(Desc
.URI
);
378 std::clog
<< "pkgAcqDiffIndex: " << Desc
.URI
<< std::endl
;
380 // look for the current package file
381 CurrentPackagesFile
= _config
->FindDir("Dir::State::lists");
382 CurrentPackagesFile
+= URItoFileName(RealURI
);
384 // FIXME: this file:/ check is a hack to prevent fetching
385 // from local sources. this is really silly, and
386 // should be fixed cleanly as soon as possible
387 if(!FileExists(CurrentPackagesFile
) ||
388 Desc
.URI
.substr(0,strlen("file:/")) == "file:/")
390 // we don't have a pkg file or we don't want to queue
391 Failed("No index file, local or canceld by user", NULL
);
396 std::clog
<< "pkgAcqDiffIndex::pkgAcqDiffIndex(): "
397 << CurrentPackagesFile
<< std::endl
;
403 // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/
404 // ---------------------------------------------------------------------
405 /* The only header we use is the last-modified header. */
406 string
pkgAcqDiffIndex::Custom600Headers() const
408 string Final
= _config
->FindDir("Dir::State::lists");
409 Final
+= URItoFileName(Desc
.URI
);
412 std::clog
<< "Custom600Header-IMS: " << Final
<< std::endl
;
415 if (stat(Final
.c_str(),&Buf
) != 0)
416 return "\nIndex-File: true";
418 return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf
.st_mtime
);
421 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile
) /*{{{*/
423 // failing here is fine: our caller will take care of trying to
424 // get the complete file if patching fails
426 std::clog
<< "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile
429 FileFd
Fd(IndexDiffFile
,FileFd::ReadOnly
);
431 if (_error
->PendingError() == true)
435 if(unlikely(TF
.Step(Tags
) == false))
438 HashStringList ServerHashes
;
439 unsigned long long ServerSize
= 0;
441 for (char const * const * type
= HashString::SupportedHashes(); *type
!= NULL
; ++type
)
443 std::string tagname
= *type
;
444 tagname
.append("-Current");
445 std::string
const tmp
= Tags
.FindS(tagname
.c_str());
446 if (tmp
.empty() == true)
450 unsigned long long size
;
451 std::stringstream
ss(tmp
);
453 if (unlikely(hash
.empty() == true))
455 if (unlikely(ServerSize
!= 0 && ServerSize
!= size
))
457 ServerHashes
.push_back(HashString(*type
, hash
));
461 if (ServerHashes
.usable() == false)
464 std::clog
<< "pkgAcqDiffIndex: " << IndexDiffFile
<< ": Did not find a good hashsum in the index" << std::endl
;
468 if (ServerHashes
!= HashSums())
472 std::clog
<< "pkgAcqDiffIndex: " << IndexDiffFile
<< ": Index has different hashes than parser, probably older, so fail pdiffing" << std::endl
;
473 printHashSumComparision(CurrentPackagesFile
, ServerHashes
, HashSums());
478 if (ServerHashes
.VerifyFile(CurrentPackagesFile
) == true)
480 // we have the same sha1 as the server so we are done here
482 std::clog
<< "pkgAcqDiffIndex: Package file " << CurrentPackagesFile
<< " is up-to-date" << std::endl
;
484 // list cleanup needs to know that this file as well as the already
485 // present index is ours, so we create an empty diff to save it for us
486 new pkgAcqIndexDiffs(Owner
, TransactionManager
, Target
,
487 ExpectedHashes
, MetaIndexParser
);
491 FileFd
fd(CurrentPackagesFile
, FileFd::ReadOnly
);
492 Hashes LocalHashesCalc
;
493 LocalHashesCalc
.AddFD(fd
);
494 HashStringList
const LocalHashes
= LocalHashesCalc
.GetHashStringList();
497 std::clog
<< "Server-Current: " << ServerHashes
.find(NULL
)->toStr() << " and we start at "
498 << fd
.Name() << " " << fd
.FileSize() << " " << LocalHashes
.find(NULL
)->toStr() << std::endl
;
500 // parse all of (provided) history
501 vector
<DiffInfo
> available_patches
;
502 bool firstAcceptedHashes
= true;
503 for (char const * const * type
= HashString::SupportedHashes(); *type
!= NULL
; ++type
)
505 if (LocalHashes
.find(*type
) == NULL
)
508 std::string tagname
= *type
;
509 tagname
.append("-History");
510 std::string
const tmp
= Tags
.FindS(tagname
.c_str());
511 if (tmp
.empty() == true)
514 string hash
, filename
;
515 unsigned long long size
;
516 std::stringstream
ss(tmp
);
518 while (ss
>> hash
>> size
>> filename
)
520 if (unlikely(hash
.empty() == true || filename
.empty() == true))
523 // see if we have a record for this file already
524 std::vector
<DiffInfo
>::iterator cur
= available_patches
.begin();
525 for (; cur
!= available_patches
.end(); ++cur
)
527 if (cur
->file
!= filename
|| unlikely(cur
->result_size
!= size
))
529 cur
->result_hashes
.push_back(HashString(*type
, hash
));
532 if (cur
!= available_patches
.end())
534 if (firstAcceptedHashes
== true)
537 next
.file
= filename
;
538 next
.result_hashes
.push_back(HashString(*type
, hash
));
539 next
.result_size
= size
;
541 available_patches
.push_back(next
);
546 std::clog
<< "pkgAcqDiffIndex: " << IndexDiffFile
<< ": File " << filename
547 << " wasn't in the list for the first parsed hash! (history)" << std::endl
;
551 firstAcceptedHashes
= false;
554 if (unlikely(available_patches
.empty() == true))
557 std::clog
<< "pkgAcqDiffIndex: " << IndexDiffFile
<< ": "
558 << "Couldn't find any patches for the patch series." << std::endl
;
562 for (char const * const * type
= HashString::SupportedHashes(); *type
!= NULL
; ++type
)
564 if (LocalHashes
.find(*type
) == NULL
)
567 std::string tagname
= *type
;
568 tagname
.append("-Patches");
569 std::string
const tmp
= Tags
.FindS(tagname
.c_str());
570 if (tmp
.empty() == true)
573 string hash
, filename
;
574 unsigned long long size
;
575 std::stringstream
ss(tmp
);
577 while (ss
>> hash
>> size
>> filename
)
579 if (unlikely(hash
.empty() == true || filename
.empty() == true))
582 // see if we have a record for this file already
583 std::vector
<DiffInfo
>::iterator cur
= available_patches
.begin();
584 for (; cur
!= available_patches
.end(); ++cur
)
586 if (cur
->file
!= filename
)
588 if (unlikely(cur
->patch_size
!= 0 && cur
->patch_size
!= size
))
590 cur
->patch_hashes
.push_back(HashString(*type
, hash
));
591 cur
->patch_size
= size
;
594 if (cur
!= available_patches
.end())
597 std::clog
<< "pkgAcqDiffIndex: " << IndexDiffFile
<< ": File " << filename
598 << " wasn't in the list for the first parsed hash! (patches)" << std::endl
;
603 bool foundStart
= false;
604 for (std::vector
<DiffInfo
>::iterator cur
= available_patches
.begin();
605 cur
!= available_patches
.end(); ++cur
)
607 if (LocalHashes
!= cur
->result_hashes
)
610 available_patches
.erase(available_patches
.begin(), cur
);
615 if (foundStart
== false || unlikely(available_patches
.empty() == true))
618 std::clog
<< "pkgAcqDiffIndex: " << IndexDiffFile
<< ": "
619 << "Couldn't find the start of the patch series." << std::endl
;
623 // patching with too many files is rather slow compared to a fast download
624 unsigned long const fileLimit
= _config
->FindI("Acquire::PDiffs::FileLimit", 0);
625 if (fileLimit
!= 0 && fileLimit
< available_patches
.size())
628 std::clog
<< "Need " << available_patches
.size() << " diffs (Limit is " << fileLimit
629 << ") so fallback to complete download" << std::endl
;
633 // calculate the size of all patches we have to get
634 // note that all sizes are uncompressed, while we download compressed files
635 unsigned long long patchesSize
= 0;
636 for (std::vector
<DiffInfo
>::const_iterator cur
= available_patches
.begin();
637 cur
!= available_patches
.end(); ++cur
)
638 patchesSize
+= cur
->patch_size
;
639 unsigned long long const sizeLimit
= ServerSize
* _config
->FindI("Acquire::PDiffs::SizeLimit", 100);
640 if (false && sizeLimit
> 0 && (sizeLimit
/100) < patchesSize
)
643 std::clog
<< "Need " << patchesSize
<< " bytes (Limit is " << sizeLimit
/100
644 << ") so fallback to complete download" << std::endl
;
648 // FIXME: make this use the method
649 PackagesFileReadyInPartial
= true;
650 std::string
const Partial
= GetPartialFileNameFromURI(RealURI
);
652 FileFd
From(CurrentPackagesFile
, FileFd::ReadOnly
);
653 FileFd
To(Partial
, FileFd::WriteEmpty
);
654 if(CopyFile(From
, To
) == false)
655 return _error
->Errno("CopyFile", "failed to copy");
658 std::cerr
<< "Done copying " << CurrentPackagesFile
662 // we have something, queue the diffs
663 string::size_type
const last_space
= Description
.rfind(" ");
664 if(last_space
!= string::npos
)
665 Description
.erase(last_space
, Description
.size()-last_space
);
667 /* decide if we should download patches one by one or in one go:
668 The first is good if the server merges patches, but many don't so client
669 based merging can be attempt in which case the second is better.
670 "bad things" will happen if patches are merged on the server,
671 but client side merging is attempt as well */
672 bool pdiff_merge
= _config
->FindB("Acquire::PDiffs::Merge", true);
673 if (pdiff_merge
== true)
675 // reprepro adds this flag if it has merged patches on the server
676 std::string
const precedence
= Tags
.FindS("X-Patch-Precedence");
677 pdiff_merge
= (precedence
!= "merged");
680 if (pdiff_merge
== false)
682 new pkgAcqIndexDiffs(Owner
, TransactionManager
, Target
, ExpectedHashes
,
683 MetaIndexParser
, available_patches
);
687 std::vector
<pkgAcqIndexMergeDiffs
*> *diffs
= new std::vector
<pkgAcqIndexMergeDiffs
*>(available_patches
.size());
688 for(size_t i
= 0; i
< available_patches
.size(); ++i
)
689 (*diffs
)[i
] = new pkgAcqIndexMergeDiffs(Owner
, TransactionManager
,
693 available_patches
[i
],
703 void pkgAcqDiffIndex::Failed(string Message
,pkgAcquire::MethodConfig
* Cnf
)/*{{{*/
706 std::clog
<< "pkgAcqDiffIndex failed: " << Desc
.URI
<< " with " << Message
<< std::endl
707 << "Falling back to normal index file acquire" << std::endl
;
709 new pkgAcqIndex(Owner
, TransactionManager
, Target
, ExpectedHashes
, MetaIndexParser
);
711 Item::Failed(Message
,Cnf
);
715 void pkgAcqDiffIndex::Done(string Message
,unsigned long long Size
,HashStringList
const &Hashes
, /*{{{*/
716 pkgAcquire::MethodConfig
*Cnf
)
719 std::clog
<< "pkgAcqDiffIndex::Done(): " << Desc
.URI
<< std::endl
;
721 Item::Done(Message
, Size
, Hashes
, Cnf
);
723 // verify the index target
724 if(Target
&& Target
->MetaKey
!= "" && MetaIndexParser
&& Hashes
.usable())
726 std::string IndexMetaKey
= Target
->MetaKey
+ ".diff/Index";
727 indexRecords::checkSum
*Record
= MetaIndexParser
->Lookup(IndexMetaKey
);
728 if(Record
&& Record
->Hashes
.usable() && Hashes
!= Record
->Hashes
)
730 RenameOnError(HashSumMismatch
);
731 printHashSumComparision(RealURI
, Record
->Hashes
, Hashes
);
732 Failed(Message
, Cnf
);
739 FinalFile
= _config
->FindDir("Dir::State::lists");
740 FinalFile
+= URItoFileName(Desc
.URI
);
742 if(StringToBool(LookupTag(Message
,"IMS-Hit"),false))
743 DestFile
= FinalFile
;
745 if(!ParseDiffIndex(DestFile
))
746 return Failed("Message: Couldn't parse pdiff index", Cnf
);
748 // queue for final move
749 TransactionManager
->TransactionStageCopy(this, DestFile
, FinalFile
);
757 // AcqIndexDiffs::AcqIndexDiffs - Constructor /*{{{*/
758 // ---------------------------------------------------------------------
759 /* The package diff is added to the queue. one object is constructed
760 * for each diff and the index
762 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire
*Owner
,
763 pkgAcqMetaBase
*TransactionManager
,
764 struct IndexTarget
const * const Target
,
765 HashStringList
const &ExpectedHashes
,
766 indexRecords
*MetaIndexParser
,
767 vector
<DiffInfo
> diffs
)
768 : pkgAcqBaseIndex(Owner
, TransactionManager
, Target
, ExpectedHashes
, MetaIndexParser
),
769 available_patches(diffs
)
771 DestFile
= GetPartialFileNameFromURI(Target
->URI
);
773 Debug
= _config
->FindB("Debug::pkgAcquire::Diffs",false);
775 RealURI
= Target
->URI
;
777 Description
= Target
->Description
;
778 Desc
.ShortDesc
= Target
->ShortDesc
;
780 if(available_patches
.empty() == true)
782 // we are done (yeah!), check hashes against the final file
783 DestFile
= _config
->FindDir("Dir::State::lists");
784 DestFile
+= URItoFileName(Target
->URI
);
790 State
= StateFetchDiff
;
795 void pkgAcqIndexDiffs::Failed(string Message
,pkgAcquire::MethodConfig
* /*Cnf*/)/*{{{*/
798 std::clog
<< "pkgAcqIndexDiffs failed: " << Desc
.URI
<< " with " << Message
<< std::endl
799 << "Falling back to normal index file acquire" << std::endl
;
800 new pkgAcqIndex(Owner
, TransactionManager
, Target
, ExpectedHashes
, MetaIndexParser
);
804 // Finish - helper that cleans the item out of the fetcher queue /*{{{*/
805 void pkgAcqIndexDiffs::Finish(bool allDone
)
808 std::clog
<< "pkgAcqIndexDiffs::Finish(): "
810 << Desc
.URI
<< std::endl
;
812 // we restore the original name, this is required, otherwise
813 // the file will be cleaned
816 if(HashSums().usable() && !HashSums().VerifyFile(DestFile
))
818 RenameOnError(HashSumMismatch
);
824 std::string FinalFile
= _config
->FindDir("Dir::State::lists");
825 FinalFile
+= URItoFileName(RealURI
);
826 TransactionManager
->TransactionStageCopy(this, DestFile
, FinalFile
);
828 // this is for the "real" finish
833 std::clog
<< "\n\nallDone: " << DestFile
<< "\n" << std::endl
;
838 std::clog
<< "Finishing: " << Desc
.URI
<< std::endl
;
845 bool pkgAcqIndexDiffs::QueueNextDiff() /*{{{*/
847 // calc sha1 of the just patched file
848 std::string
const FinalFile
= GetPartialFileNameFromURI(RealURI
);
850 if(!FileExists(FinalFile
))
852 Failed("Message: No FinalFile " + FinalFile
+ " available", NULL
);
856 FileFd
fd(FinalFile
, FileFd::ReadOnly
);
857 Hashes LocalHashesCalc
;
858 LocalHashesCalc
.AddFD(fd
);
859 HashStringList
const LocalHashes
= LocalHashesCalc
.GetHashStringList();
862 std::clog
<< "QueueNextDiff: " << FinalFile
<< " (" << LocalHashes
.find(NULL
)->toStr() << ")" << std::endl
;
864 if (unlikely(LocalHashes
.usable() == false || ExpectedHashes
.usable() == false))
866 Failed("Local/Expected hashes are not usable", NULL
);
871 // final file reached before all patches are applied
872 if(LocalHashes
== ExpectedHashes
)
878 // remove all patches until the next matching patch is found
879 // this requires the Index file to be ordered
880 for(vector
<DiffInfo
>::iterator I
= available_patches
.begin();
881 available_patches
.empty() == false &&
882 I
!= available_patches
.end() &&
883 I
->result_hashes
!= LocalHashes
;
886 available_patches
.erase(I
);
889 // error checking and falling back if no patch was found
890 if(available_patches
.empty() == true)
892 Failed("No patches left to reach target", NULL
);
896 // queue the right diff
897 Desc
.URI
= RealURI
+ ".diff/" + available_patches
[0].file
+ ".gz";
898 Desc
.Description
= Description
+ " " + available_patches
[0].file
+ string(".pdiff");
899 DestFile
= GetPartialFileNameFromURI(RealURI
+ ".diff/" + available_patches
[0].file
);
902 std::clog
<< "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc
.URI
<< std::endl
;
909 void pkgAcqIndexDiffs::Done(string Message
,unsigned long long Size
, HashStringList
const &Hashes
, /*{{{*/
910 pkgAcquire::MethodConfig
*Cnf
)
913 std::clog
<< "pkgAcqIndexDiffs::Done(): " << Desc
.URI
<< std::endl
;
915 Item::Done(Message
, Size
, Hashes
, Cnf
);
917 // FIXME: verify this download too before feeding it to rred
918 std::string
const FinalFile
= GetPartialFileNameFromURI(RealURI
);
920 // success in downloading a diff, enter ApplyDiff state
921 if(State
== StateFetchDiff
)
923 FileFd
fd(DestFile
, FileFd::ReadOnly
, FileFd::Gzip
);
924 class Hashes LocalHashesCalc
;
925 LocalHashesCalc
.AddFD(fd
);
926 HashStringList
const LocalHashes
= LocalHashesCalc
.GetHashStringList();
928 if (fd
.Size() != available_patches
[0].patch_size
||
929 available_patches
[0].patch_hashes
!= LocalHashes
)
931 Failed("Patch has Size/Hashsum mismatch", NULL
);
935 // rred excepts the patch as $FinalFile.ed
936 Rename(DestFile
,FinalFile
+".ed");
939 std::clog
<< "Sending to rred method: " << FinalFile
<< std::endl
;
941 State
= StateApplyDiff
;
943 Desc
.URI
= "rred:" + FinalFile
;
945 SetActiveSubprocess("rred");
950 // success in download/apply a diff, queue next (if needed)
951 if(State
== StateApplyDiff
)
953 // remove the just applied patch
954 available_patches
.erase(available_patches
.begin());
955 unlink((FinalFile
+ ".ed").c_str());
960 std::clog
<< "Moving patched file in place: " << std::endl
961 << DestFile
<< " -> " << FinalFile
<< std::endl
;
963 Rename(DestFile
,FinalFile
);
964 chmod(FinalFile
.c_str(),0644);
966 // see if there is more to download
967 if(available_patches
.empty() == false) {
968 new pkgAcqIndexDiffs(Owner
, TransactionManager
, Target
,
969 ExpectedHashes
, MetaIndexParser
,
974 DestFile
= FinalFile
;
979 // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor /*{{{*/
980 pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire
*Owner
,
981 pkgAcqMetaBase
*TransactionManager
,
982 struct IndexTarget
const * const Target
,
983 HashStringList
const &ExpectedHashes
,
984 indexRecords
*MetaIndexParser
,
985 DiffInfo
const &patch
,
986 std::vector
<pkgAcqIndexMergeDiffs
*> const * const allPatches
)
987 : pkgAcqBaseIndex(Owner
, TransactionManager
, Target
, ExpectedHashes
, MetaIndexParser
),
988 patch(patch
), allPatches(allPatches
), State(StateFetchDiff
)
990 Debug
= _config
->FindB("Debug::pkgAcquire::Diffs",false);
992 RealURI
= Target
->URI
;
994 Description
= Target
->Description
;
995 Desc
.ShortDesc
= Target
->ShortDesc
;
997 Desc
.URI
= RealURI
+ ".diff/" + patch
.file
+ ".gz";
998 Desc
.Description
= Description
+ " " + patch
.file
+ string(".pdiff");
1000 DestFile
= GetPartialFileNameFromURI(RealURI
+ ".diff/" + patch
.file
);
1003 std::clog
<< "pkgAcqIndexMergeDiffs: " << Desc
.URI
<< std::endl
;
1008 void pkgAcqIndexMergeDiffs::Failed(string Message
,pkgAcquire::MethodConfig
* Cnf
)/*{{{*/
1011 std::clog
<< "pkgAcqIndexMergeDiffs failed: " << Desc
.URI
<< " with " << Message
<< std::endl
;
1013 Item::Failed(Message
,Cnf
);
1016 // check if we are the first to fail, otherwise we are done here
1017 State
= StateDoneDiff
;
1018 for (std::vector
<pkgAcqIndexMergeDiffs
*>::const_iterator I
= allPatches
->begin();
1019 I
!= allPatches
->end(); ++I
)
1020 if ((*I
)->State
== StateErrorDiff
)
1023 // first failure means we should fallback
1024 State
= StateErrorDiff
;
1025 std::clog
<< "Falling back to normal index file acquire" << std::endl
;
1026 new pkgAcqIndex(Owner
, TransactionManager
, Target
, ExpectedHashes
, MetaIndexParser
);
1029 void pkgAcqIndexMergeDiffs::Done(string Message
,unsigned long long Size
,HashStringList
const &Hashes
, /*{{{*/
1030 pkgAcquire::MethodConfig
*Cnf
)
1033 std::clog
<< "pkgAcqIndexMergeDiffs::Done(): " << Desc
.URI
<< std::endl
;
1035 Item::Done(Message
,Size
,Hashes
,Cnf
);
1037 // FIXME: verify download before feeding it to rred
1038 string
const FinalFile
= GetPartialFileNameFromURI(RealURI
);
1040 if (State
== StateFetchDiff
)
1042 FileFd
fd(DestFile
, FileFd::ReadOnly
, FileFd::Gzip
);
1043 class Hashes LocalHashesCalc
;
1044 LocalHashesCalc
.AddFD(fd
);
1045 HashStringList
const LocalHashes
= LocalHashesCalc
.GetHashStringList();
1047 if (fd
.Size() != patch
.patch_size
|| patch
.patch_hashes
!= LocalHashes
)
1049 Failed("Patch has Size/Hashsum mismatch", NULL
);
1053 // rred expects the patch as $FinalFile.ed.$patchname.gz
1054 Rename(DestFile
, FinalFile
+ ".ed." + patch
.file
+ ".gz");
1056 // check if this is the last completed diff
1057 State
= StateDoneDiff
;
1058 for (std::vector
<pkgAcqIndexMergeDiffs
*>::const_iterator I
= allPatches
->begin();
1059 I
!= allPatches
->end(); ++I
)
1060 if ((*I
)->State
!= StateDoneDiff
)
1063 std::clog
<< "Not the last done diff in the batch: " << Desc
.URI
<< std::endl
;
1067 // this is the last completed diff, so we are ready to apply now
1068 State
= StateApplyDiff
;
1071 std::clog
<< "Sending to rred method: " << FinalFile
<< std::endl
;
1074 Desc
.URI
= "rred:" + FinalFile
;
1076 SetActiveSubprocess("rred");
1079 // success in download/apply all diffs, clean up
1080 else if (State
== StateApplyDiff
)
1082 // see if we really got the expected file
1083 if(ExpectedHashes
.usable() && !ExpectedHashes
.VerifyFile(DestFile
))
1085 RenameOnError(HashSumMismatch
);
1090 std::string FinalFile
= _config
->FindDir("Dir::State::lists");
1091 FinalFile
+= URItoFileName(RealURI
);
1093 // move the result into place
1095 std::clog
<< "Queue patched file in place: " << std::endl
1096 << DestFile
<< " -> " << FinalFile
<< std::endl
;
1098 // queue for copy by the transaction manager
1099 TransactionManager
->TransactionStageCopy(this, DestFile
, FinalFile
);
1101 // ensure the ed's are gone regardless of list-cleanup
1102 for (std::vector
<pkgAcqIndexMergeDiffs
*>::const_iterator I
= allPatches
->begin();
1103 I
!= allPatches
->end(); ++I
)
1105 std::string
const PartialFile
= GetPartialFileNameFromURI(RealURI
);
1106 std::string patch
= PartialFile
+ ".ed." + (*I
)->patch
.file
+ ".gz";
1107 unlink(patch
.c_str());
1113 std::clog
<< "allDone: " << DestFile
<< "\n" << std::endl
;
1117 // AcqBaseIndex::VerifyHashByMetaKey - verify hash for the given metakey /*{{{*/
1118 bool pkgAcqBaseIndex::VerifyHashByMetaKey(HashStringList
const &Hashes
)
1120 if(MetaKey
!= "" && Hashes
.usable())
1122 indexRecords::checkSum
*Record
= MetaIndexParser
->Lookup(MetaKey
);
1123 if(Record
&& Record
->Hashes
.usable() && Hashes
!= Record
->Hashes
)
1125 printHashSumComparision(RealURI
, Record
->Hashes
, Hashes
);
1132 // AcqIndex::AcqIndex - Constructor /*{{{*/
1133 // ---------------------------------------------------------------------
1134 /* The package file is added to the queue and a second class is
1135 instantiated to fetch the revision file */
1136 pkgAcqIndex::pkgAcqIndex(pkgAcquire
*Owner
,
1137 string URI
,string URIDesc
,string ShortDesc
,
1138 HashStringList
const &ExpectedHash
)
1139 : pkgAcqBaseIndex(Owner
, 0, NULL
, ExpectedHash
, NULL
)
1143 AutoSelectCompression();
1144 Init(URI
, URIDesc
, ShortDesc
);
1146 if(_config
->FindB("Debug::Acquire::Transaction", false) == true)
1147 std::clog
<< "New pkgIndex with TransactionManager "
1148 << TransactionManager
<< std::endl
;
1151 // AcqIndex::AcqIndex - Constructor /*{{{*/
1152 pkgAcqIndex::pkgAcqIndex(pkgAcquire
*Owner
,
1153 pkgAcqMetaBase
*TransactionManager
,
1154 IndexTarget
const *Target
,
1155 HashStringList
const &ExpectedHash
,
1156 indexRecords
*MetaIndexParser
)
1157 : pkgAcqBaseIndex(Owner
, TransactionManager
, Target
, ExpectedHash
,
1160 RealURI
= Target
->URI
;
1162 // autoselect the compression method
1163 AutoSelectCompression();
1164 Init(Target
->URI
, Target
->Description
, Target
->ShortDesc
);
1166 if(_config
->FindB("Debug::Acquire::Transaction", false) == true)
1167 std::clog
<< "New pkgIndex with TransactionManager "
1168 << TransactionManager
<< std::endl
;
1171 // AcqIndex::AutoSelectCompression - Select compression /*{{{*/
1172 void pkgAcqIndex::AutoSelectCompression()
1174 std::vector
<std::string
> types
= APT::Configuration::getCompressionTypes();
1175 CompressionExtensions
= "";
1176 if (ExpectedHashes
.usable())
1178 for (std::vector
<std::string
>::const_iterator t
= types
.begin();
1179 t
!= types
.end(); ++t
)
1181 std::string CompressedMetaKey
= string(Target
->MetaKey
).append(".").append(*t
);
1182 if (*t
== "uncompressed" ||
1183 MetaIndexParser
->Exists(CompressedMetaKey
) == true)
1184 CompressionExtensions
.append(*t
).append(" ");
1189 for (std::vector
<std::string
>::const_iterator t
= types
.begin(); t
!= types
.end(); ++t
)
1190 CompressionExtensions
.append(*t
).append(" ");
1192 if (CompressionExtensions
.empty() == false)
1193 CompressionExtensions
.erase(CompressionExtensions
.end()-1);
1196 // AcqIndex::Init - defered Constructor /*{{{*/
1197 void pkgAcqIndex::Init(string
const &URI
, string
const &URIDesc
,
1198 string
const &ShortDesc
)
1200 Stage
= STAGE_DOWNLOAD
;
1202 DestFile
= GetPartialFileNameFromURI(URI
);
1204 CurrentCompressionExtension
= CompressionExtensions
.substr(0, CompressionExtensions
.find(' '));
1205 if (CurrentCompressionExtension
== "uncompressed")
1209 MetaKey
= string(Target
->MetaKey
);
1213 Desc
.URI
= URI
+ '.' + CurrentCompressionExtension
;
1214 DestFile
= DestFile
+ '.' + CurrentCompressionExtension
;
1216 MetaKey
= string(Target
->MetaKey
) + '.' + CurrentCompressionExtension
;
1219 // load the filesize
1222 indexRecords::checkSum
*Record
= MetaIndexParser
->Lookup(MetaKey
);
1224 FileSize
= Record
->Size
;
1226 InitByHashIfNeeded(MetaKey
);
1229 Desc
.Description
= URIDesc
;
1231 Desc
.ShortDesc
= ShortDesc
;
1236 // AcqIndex::AdjustForByHash - modify URI for by-hash support /*{{{*/
1237 void pkgAcqIndex::InitByHashIfNeeded(const std::string MetaKey
)
1240 // - (maybe?) add support for by-hash into the sources.list as flag
1241 // - make apt-ftparchive generate the hashes (and expire?)
1242 std::string HostKnob
= "APT::Acquire::" + ::URI(Desc
.URI
).Host
+ "::By-Hash";
1243 if(_config
->FindB("APT::Acquire::By-Hash", false) == true ||
1244 _config
->FindB(HostKnob
, false) == true ||
1245 MetaIndexParser
->GetSupportsAcquireByHash())
1247 indexRecords::checkSum
*Record
= MetaIndexParser
->Lookup(MetaKey
);
1250 // FIXME: should we really use the best hash here? or a fixed one?
1251 const HashString
*TargetHash
= Record
->Hashes
.find("");
1252 std::string ByHash
= "/by-hash/" + TargetHash
->HashType() + "/" + TargetHash
->HashValue();
1253 size_t trailing_slash
= Desc
.URI
.find_last_of("/");
1254 Desc
.URI
= Desc
.URI
.replace(
1256 Desc
.URI
.substr(trailing_slash
+1).size()+1,
1260 "Fetching ByHash requested but can not find record for %s",
1266 // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/
1267 // ---------------------------------------------------------------------
1268 /* The only header we use is the last-modified header. */
1269 string
pkgAcqIndex::Custom600Headers() const
1271 string Final
= GetFinalFilename();
1273 string msg
= "\nIndex-File: true";
1275 if (stat(Final
.c_str(),&Buf
) == 0)
1276 msg
+= "\nLast-Modified: " + TimeRFC1123(Buf
.st_mtime
);
1278 if(Target
->IsOptional())
1279 msg
+= "\nFail-Ignore: true";
1284 // pkgAcqIndex::Failed - getting the indexfile failed /*{{{*/
1285 void pkgAcqIndex::Failed(string Message
,pkgAcquire::MethodConfig
*Cnf
)
1287 size_t const nextExt
= CompressionExtensions
.find(' ');
1288 if (nextExt
!= std::string::npos
)
1290 CompressionExtensions
= CompressionExtensions
.substr(nextExt
+1);
1291 Init(RealURI
, Desc
.Description
, Desc
.ShortDesc
);
1295 // on decompression failure, remove bad versions in partial/
1296 if (Stage
== STAGE_DECOMPRESS_AND_VERIFY
)
1298 unlink(EraseFileName
.c_str());
1301 Item::Failed(Message
,Cnf
);
1303 if(Target
->IsOptional() && ExpectedHashes
.empty() && Stage
== STAGE_DOWNLOAD
)
1306 TransactionManager
->AbortTransaction();
1309 // pkgAcqIndex::GetFinalFilename - Return the full final file path /*{{{*/
1310 std::string
pkgAcqIndex::GetFinalFilename() const
1312 std::string FinalFile
= _config
->FindDir("Dir::State::lists");
1313 FinalFile
+= URItoFileName(RealURI
);
1314 return GetCompressedFileName(RealURI
, FinalFile
, CurrentCompressionExtension
);
1317 // AcqIndex::ReverifyAfterIMS - Reverify index after an ims-hit /*{{{*/
1318 void pkgAcqIndex::ReverifyAfterIMS()
1320 // update destfile to *not* include the compression extension when doing
1321 // a reverify (as its uncompressed on disk already)
1322 DestFile
= GetCompressedFileName(RealURI
, GetPartialFileNameFromURI(RealURI
), CurrentCompressionExtension
);
1324 // copy FinalFile into partial/ so that we check the hash again
1325 string FinalFile
= GetFinalFilename();
1326 Stage
= STAGE_DECOMPRESS_AND_VERIFY
;
1327 Desc
.URI
= "copy:" + FinalFile
;
1331 // AcqIndex::ValidateFile - Validate the content of the downloaded file /*{{{*/
1332 bool pkgAcqIndex::ValidateFile(const std::string
&FileName
)
1334 // FIXME: this can go away once we only ever download stuff that
1335 // has a valid hash and we never do GET based probing
1336 // FIXME2: this also leaks debian-isms into the code and should go therefore
1338 /* Always validate the index file for correctness (all indexes must
1339 * have a Package field) (LP: #346386) (Closes: #627642)
1341 FileFd
fd(FileName
, FileFd::ReadOnly
, FileFd::Extension
);
1342 // Only test for correctness if the content of the file is not empty
1347 pkgTagFile
tag(&fd
);
1349 // all our current indexes have a field 'Package' in each section
1350 if (_error
->PendingError() == true ||
1351 tag
.Step(sec
) == false ||
1352 sec
.Exists("Package") == false)
1358 // AcqIndex::Done - Finished a fetch /*{{{*/
1359 // ---------------------------------------------------------------------
1360 /* This goes through a number of states.. On the initial fetch the
1361 method could possibly return an alternate filename which points
1362 to the uncompressed version of the file. If this is so the file
1363 is copied into the partial directory. In all other cases the file
1364 is decompressed with a compressed uri. */
1365 void pkgAcqIndex::Done(string Message
,
1366 unsigned long long Size
,
1367 HashStringList
const &Hashes
,
1368 pkgAcquire::MethodConfig
*Cfg
)
1370 Item::Done(Message
,Size
,Hashes
,Cfg
);
1374 case STAGE_DOWNLOAD
:
1375 StageDownloadDone(Message
, Hashes
, Cfg
);
1377 case STAGE_DECOMPRESS_AND_VERIFY
:
1378 StageDecompressDone(Message
, Hashes
, Cfg
);
1383 // AcqIndex::StageDownloadDone - Queue for decompress and verify /*{{{*/
1384 void pkgAcqIndex::StageDownloadDone(string Message
,
1385 HashStringList
const &Hashes
,
1386 pkgAcquire::MethodConfig
*Cfg
)
1388 // First check if the calculcated Hash of the (compressed) downloaded
1389 // file matches the hash we have in the MetaIndexRecords for this file
1390 if(VerifyHashByMetaKey(Hashes
) == false)
1392 RenameOnError(HashSumMismatch
);
1393 Failed(Message
, Cfg
);
1399 // Handle the unzipd case
1400 string FileName
= LookupTag(Message
,"Alt-Filename");
1401 if (FileName
.empty() == false)
1403 Stage
= STAGE_DECOMPRESS_AND_VERIFY
;
1405 DestFile
+= ".decomp";
1406 Desc
.URI
= "copy:" + FileName
;
1408 SetActiveSubprocess("copy");
1412 FileName
= LookupTag(Message
,"Filename");
1413 if (FileName
.empty() == true)
1416 ErrorText
= "Method gave a blank filename";
1419 // Methods like e.g. "file:" will give us a (compressed) FileName that is
1420 // not the "DestFile" we set, in this case we uncompress from the local file
1421 if (FileName
!= DestFile
)
1424 EraseFileName
= FileName
;
1426 // we need to verify the file against the current Release file again
1427 // on if-modfied-since hit to avoid a stale attack against us
1428 if(StringToBool(LookupTag(Message
,"IMS-Hit"),false) == true)
1430 // The files timestamp matches, reverify by copy into partial/
1436 // If we have compressed indexes enabled, queue for hash verification
1437 if (_config
->FindB("Acquire::GzipIndexes",false))
1439 DestFile
= GetPartialFileNameFromURI(RealURI
+ '.' + CurrentCompressionExtension
);
1441 Stage
= STAGE_DECOMPRESS_AND_VERIFY
;
1442 Desc
.URI
= "copy:" + FileName
;
1444 SetActiveSubprocess("copy");
1448 // get the binary name for your used compression type
1450 if(CurrentCompressionExtension
== "uncompressed")
1451 decompProg
= "copy";
1453 decompProg
= _config
->Find(string("Acquire::CompressionTypes::").append(CurrentCompressionExtension
),"");
1454 if(decompProg
.empty() == true)
1456 _error
->Error("Unsupported extension: %s", CurrentCompressionExtension
.c_str());
1460 // queue uri for the next stage
1461 Stage
= STAGE_DECOMPRESS_AND_VERIFY
;
1462 DestFile
+= ".decomp";
1463 Desc
.URI
= decompProg
+ ":" + FileName
;
1465 SetActiveSubprocess(decompProg
);
1468 // pkgAcqIndex::StageDecompressDone - Final verification /*{{{*/
1469 void pkgAcqIndex::StageDecompressDone(string Message
,
1470 HashStringList
const &Hashes
,
1471 pkgAcquire::MethodConfig
*Cfg
)
1473 if (ExpectedHashes
.usable() && ExpectedHashes
!= Hashes
)
1476 RenameOnError(HashSumMismatch
);
1477 printHashSumComparision(RealURI
, ExpectedHashes
, Hashes
);
1478 Failed(Message
, Cfg
);
1482 if(!ValidateFile(DestFile
))
1484 RenameOnError(InvalidFormat
);
1485 Failed(Message
, Cfg
);
1489 // remove the compressed version of the file
1490 unlink(EraseFileName
.c_str());
1492 // Done, queue for rename on transaction finished
1493 TransactionManager
->TransactionStageCopy(this, DestFile
, GetFinalFilename());
1498 // AcqMetaBase::Add - Add a item to the current Transaction /*{{{*/
1499 void pkgAcqMetaBase::Add(Item
*I
)
1501 Transaction
.push_back(I
);
1504 // AcqMetaBase::AbortTransaction - Abort the current Transaction /*{{{*/
1505 void pkgAcqMetaBase::AbortTransaction()
1507 if(_config
->FindB("Debug::Acquire::Transaction", false) == true)
1508 std::clog
<< "AbortTransaction: " << TransactionManager
<< std::endl
;
1510 // ensure the toplevel is in error state too
1511 for (std::vector
<Item
*>::iterator I
= Transaction
.begin();
1512 I
!= Transaction
.end(); ++I
)
1514 if(_config
->FindB("Debug::Acquire::Transaction", false) == true)
1515 std::clog
<< " Cancel: " << (*I
)->DestFile
<< std::endl
;
1516 // the transaction will abort, so stop anything that is idle
1517 if ((*I
)->Status
== pkgAcquire::Item::StatIdle
)
1518 (*I
)->Status
= pkgAcquire::Item::StatDone
;
1520 // kill failed files in partial
1521 if ((*I
)->Status
== pkgAcquire::Item::StatError
)
1523 std::string
const PartialFile
= GetPartialFileName(flNotDir((*I
)->DestFile
));
1524 if(FileExists(PartialFile
))
1525 Rename(PartialFile
, PartialFile
+ ".FAILED");
1527 // fix permissions for existing files which were part of a reverify
1528 // like InRelease files or files in partial we might work with next time
1529 else if (FileExists((*I
)->DestFile
))
1530 ChangeOwnerAndPermissionOfFile("AbortTransaction", (*I
)->DestFile
.c_str(), "root", "root", 0644);
1532 Transaction
.clear();
1535 // AcqMetaBase::TransactionHasError - Check for errors in Transaction /*{{{*/
1536 bool pkgAcqMetaBase::TransactionHasError()
1538 for (pkgAcquire::ItemIterator I
= Transaction
.begin();
1539 I
!= Transaction
.end(); ++I
)
1540 if((*I
)->Status
!= pkgAcquire::Item::StatDone
&&
1541 (*I
)->Status
!= pkgAcquire::Item::StatIdle
)
1547 // AcqMetaBase::CommitTransaction - Commit a transaction /*{{{*/
1548 void pkgAcqMetaBase::CommitTransaction()
1550 if(_config
->FindB("Debug::Acquire::Transaction", false) == true)
1551 std::clog
<< "CommitTransaction: " << this << std::endl
;
1553 // move new files into place *and* remove files that are not
1554 // part of the transaction but are still on disk
1555 for (std::vector
<Item
*>::iterator I
= Transaction
.begin();
1556 I
!= Transaction
.end(); ++I
)
1558 if((*I
)->PartialFile
!= "")
1560 if(_config
->FindB("Debug::Acquire::Transaction", false) == true)
1561 std::clog
<< "mv " << (*I
)->PartialFile
<< " -> "<< (*I
)->DestFile
<< " "
1562 << (*I
)->DescURI() << std::endl
;
1564 Rename((*I
)->PartialFile
, (*I
)->DestFile
);
1565 ChangeOwnerAndPermissionOfFile("CommitTransaction", (*I
)->DestFile
.c_str(), "root", "root", 0644);
1568 if(_config
->FindB("Debug::Acquire::Transaction", false) == true)
1574 unlink((*I
)->DestFile
.c_str());
1576 // mark that this transaction is finished
1577 (*I
)->TransactionManager
= 0;
1579 Transaction
.clear();
1582 // AcqMetaBase::TransactionStageCopy - Stage a file for copying /*{{{*/
1583 void pkgAcqMetaBase::TransactionStageCopy(Item
*I
,
1584 const std::string
&From
,
1585 const std::string
&To
)
1587 I
->PartialFile
= From
;
1591 // AcqMetaBase::TransactionStageRemoval - Sage a file for removal /*{{{*/
1592 void pkgAcqMetaBase::TransactionStageRemoval(Item
*I
,
1593 const std::string
&FinalFile
)
1595 I
->PartialFile
= "";
1596 I
->DestFile
= FinalFile
;
1599 // AcqMetaBase::GenerateAuthWarning - Check gpg authentication error /*{{{*/
1600 bool pkgAcqMetaBase::CheckStopAuthentication(const std::string
&RealURI
,
1601 const std::string
&Message
)
1603 // FIXME: this entire function can do now that we disallow going to
1604 // a unauthenticated state and can cleanly rollback
1606 string Final
= _config
->FindDir("Dir::State::lists") + URItoFileName(RealURI
);
1608 if(FileExists(Final
))
1610 Status
= StatTransientNetworkError
;
1611 _error
->Warning(_("An error occurred during the signature "
1612 "verification. The repository is not updated "
1613 "and the previous index files will be used. "
1614 "GPG error: %s: %s\n"),
1615 Desc
.Description
.c_str(),
1616 LookupTag(Message
,"Message").c_str());
1617 RunScripts("APT::Update::Auth-Failure");
1619 } else if (LookupTag(Message
,"Message").find("NODATA") != string::npos
) {
1620 /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */
1621 _error
->Error(_("GPG error: %s: %s"),
1622 Desc
.Description
.c_str(),
1623 LookupTag(Message
,"Message").c_str());
1627 _error
->Warning(_("GPG error: %s: %s"),
1628 Desc
.Description
.c_str(),
1629 LookupTag(Message
,"Message").c_str());
1631 // gpgv method failed
1632 ReportMirrorFailure("GPGFailure");
1636 // AcqMetaSig::AcqMetaSig - Constructor /*{{{*/
1637 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire
*Owner
,
1638 pkgAcqMetaBase
*TransactionManager
,
1639 string URI
,string URIDesc
,string ShortDesc
,
1640 string MetaIndexFile
,
1641 const vector
<IndexTarget
*>* IndexTargets
,
1642 indexRecords
* MetaIndexParser
) :
1643 pkgAcqMetaBase(Owner
, IndexTargets
, MetaIndexParser
,
1644 HashStringList(), TransactionManager
),
1645 RealURI(URI
), MetaIndexFile(MetaIndexFile
), URIDesc(URIDesc
),
1646 ShortDesc(ShortDesc
)
1648 DestFile
= _config
->FindDir("Dir::State::lists") + "partial/";
1649 DestFile
+= URItoFileName(RealURI
);
1651 // remove any partial downloaded sig-file in partial/.
1652 // it may confuse proxies and is too small to warrant a
1653 // partial download anyway
1654 unlink(DestFile
.c_str());
1656 // set the TransactionManager
1657 if(_config
->FindB("Debug::Acquire::Transaction", false) == true)
1658 std::clog
<< "New pkgAcqMetaSig with TransactionManager "
1659 << TransactionManager
<< std::endl
;
1662 Desc
.Description
= URIDesc
;
1664 Desc
.ShortDesc
= ShortDesc
;
1670 pkgAcqMetaSig::~pkgAcqMetaSig() /*{{{*/
1674 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers /*{{{*/
1675 // ---------------------------------------------------------------------
1676 string
pkgAcqMetaSig::Custom600Headers() const
1678 std::string Header
= GetCustom600Headers(RealURI
);
1682 // pkgAcqMetaSig::Done - The signature was downloaded/verified /*{{{*/
1683 // ---------------------------------------------------------------------
1684 /* The only header we use is the last-modified header. */
1685 void pkgAcqMetaSig::Done(string Message
,unsigned long long Size
,
1686 HashStringList
const &Hashes
,
1687 pkgAcquire::MethodConfig
*Cfg
)
1689 Item::Done(Message
, Size
, Hashes
, Cfg
);
1691 if(AuthPass
== false)
1693 if(CheckDownloadDone(Message
, RealURI
) == true)
1695 // destfile will be modified to point to MetaIndexFile for the
1696 // gpgv method, so we need to save it here
1697 MetaIndexFileSignature
= DestFile
;
1698 QueueForSignatureVerify(MetaIndexFile
, MetaIndexFileSignature
);
1704 if(CheckAuthDone(Message
, RealURI
) == true)
1706 std::string FinalFile
= _config
->FindDir("Dir::State::lists");
1707 FinalFile
+= URItoFileName(RealURI
);
1708 TransactionManager
->TransactionStageCopy(this, MetaIndexFileSignature
, FinalFile
);
1713 void pkgAcqMetaSig::Failed(string Message
,pkgAcquire::MethodConfig
*Cnf
)/*{{{*/
1715 string Final
= _config
->FindDir("Dir::State::lists") + URItoFileName(RealURI
);
1717 // check if we need to fail at this point
1718 if (AuthPass
== true && CheckStopAuthentication(RealURI
, Message
))
1721 // FIXME: meh, this is not really elegant
1722 string InReleaseURI
= RealURI
.replace(RealURI
.rfind("Release.gpg"), 12,
1724 string FinalInRelease
= _config
->FindDir("Dir::State::lists") + URItoFileName(InReleaseURI
);
1726 if (RealFileExists(Final
) || RealFileExists(FinalInRelease
))
1728 std::string downgrade_msg
;
1729 strprintf(downgrade_msg
, _("The repository '%s' is no longer signed."),
1731 if(_config
->FindB("Acquire::AllowDowngradeToInsecureRepositories"))
1733 // meh, the users wants to take risks (we still mark the packages
1734 // from this repository as unauthenticated)
1735 _error
->Warning("%s", downgrade_msg
.c_str());
1736 _error
->Warning(_("This is normally not allowed, but the option "
1737 "Acquire::AllowDowngradeToInsecureRepositories was "
1738 "given to override it."));
1741 _error
->Error("%s", downgrade_msg
.c_str());
1742 Rename(MetaIndexFile
, MetaIndexFile
+".FAILED");
1743 Item::Failed("Message: " + downgrade_msg
, Cnf
);
1744 TransactionManager
->AbortTransaction();
1749 _error
->Warning(_("The data from '%s' is not signed. Packages "
1750 "from that repository can not be authenticated."),
1753 // this ensures that any file in the lists/ dir is removed by the
1755 DestFile
= GetPartialFileNameFromURI(RealURI
);
1756 TransactionManager
->TransactionStageRemoval(this, DestFile
);
1758 // only allow going further if the users explicitely wants it
1759 if(AllowInsecureRepositories(MetaIndexParser
, TransactionManager
, this) == true)
1761 // we parse the indexes here because at this point the user wanted
1762 // a repository that may potentially harm him
1763 MetaIndexParser
->Load(MetaIndexFile
);
1767 Item::Failed(Message
,Cnf
);
1769 // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor
1770 if (Cnf
->LocalOnly
== true ||
1771 StringToBool(LookupTag(Message
,"Transient-Failure"),false) == false)
1778 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire
*Owner
, /*{{{*/
1779 pkgAcqMetaBase
*TransactionManager
,
1780 string URI
,string URIDesc
,string ShortDesc
,
1781 string MetaIndexSigURI
,string MetaIndexSigURIDesc
, string MetaIndexSigShortDesc
,
1782 const vector
<IndexTarget
*>* IndexTargets
,
1783 indexRecords
* MetaIndexParser
) :
1784 pkgAcqMetaBase(Owner
, IndexTargets
, MetaIndexParser
, HashStringList(),
1785 TransactionManager
),
1786 RealURI(URI
), URIDesc(URIDesc
), ShortDesc(ShortDesc
),
1787 MetaIndexSigURI(MetaIndexSigURI
), MetaIndexSigURIDesc(MetaIndexSigURIDesc
),
1788 MetaIndexSigShortDesc(MetaIndexSigShortDesc
)
1790 if(TransactionManager
== NULL
)
1792 this->TransactionManager
= this;
1793 this->TransactionManager
->Add(this);
1796 if(_config
->FindB("Debug::Acquire::Transaction", false) == true)
1797 std::clog
<< "New pkgAcqMetaIndex with TransactionManager "
1798 << this->TransactionManager
<< std::endl
;
1801 Init(URIDesc
, ShortDesc
);
1804 // pkgAcqMetaIndex::Init - Delayed constructor /*{{{*/
1805 void pkgAcqMetaIndex::Init(std::string URIDesc
, std::string ShortDesc
)
1807 DestFile
= GetPartialFileNameFromURI(RealURI
);
1810 Desc
.Description
= URIDesc
;
1812 Desc
.ShortDesc
= ShortDesc
;
1815 // we expect more item
1816 ExpectedAdditionalItems
= IndexTargets
->size();
1820 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers /*{{{*/
1821 // ---------------------------------------------------------------------
1822 string
pkgAcqMetaIndex::Custom600Headers() const
1824 return GetCustom600Headers(RealURI
);
1827 void pkgAcqMetaIndex::Done(string Message
,unsigned long long Size
, /*{{{*/
1828 HashStringList
const &Hashes
,
1829 pkgAcquire::MethodConfig
*Cfg
)
1831 Item::Done(Message
,Size
,Hashes
,Cfg
);
1833 if(CheckDownloadDone(Message
, RealURI
))
1835 // we have a Release file, now download the Signature, all further
1836 // verify/queue for additional downloads will be done in the
1837 // pkgAcqMetaSig::Done() code
1838 std::string MetaIndexFile
= DestFile
;
1839 new pkgAcqMetaSig(Owner
, TransactionManager
,
1840 MetaIndexSigURI
, MetaIndexSigURIDesc
,
1841 MetaIndexSigShortDesc
, MetaIndexFile
, IndexTargets
,
1844 string FinalFile
= _config
->FindDir("Dir::State::lists");
1845 FinalFile
+= URItoFileName(RealURI
);
1846 TransactionManager
->TransactionStageCopy(this, DestFile
, FinalFile
);
1850 bool pkgAcqMetaBase::CheckAuthDone(string Message
, const string
&RealURI
) /*{{{*/
1852 // At this point, the gpgv method has succeeded, so there is a
1853 // valid signature from a key in the trusted keyring. We
1854 // perform additional verification of its contents, and use them
1855 // to verify the indexes we are about to download
1857 if (!MetaIndexParser
->Load(DestFile
))
1859 Status
= StatAuthError
;
1860 ErrorText
= MetaIndexParser
->ErrorText
;
1864 if (!VerifyVendor(Message
, RealURI
))
1869 if (_config
->FindB("Debug::pkgAcquire::Auth", false))
1870 std::cerr
<< "Signature verification succeeded: "
1871 << DestFile
<< std::endl
;
1873 // Download further indexes with verification
1875 // it would be really nice if we could simply do
1876 // if (IMSHit == false) QueueIndexes(true)
1877 // and skip the download if the Release file has not changed
1878 // - but right now the list cleaner will needs to be tricked
1879 // to not delete all our packages/source indexes in this case
1885 // pkgAcqMetaBase::GetCustom600Headers - Get header for AcqMetaBase /*{{{*/
1886 // ---------------------------------------------------------------------
1887 string
pkgAcqMetaBase::GetCustom600Headers(const string
&RealURI
) const
1889 std::string Header
= "\nIndex-File: true";
1890 std::string MaximumSize
;
1891 strprintf(MaximumSize
, "\nMaximum-Size: %i",
1892 _config
->FindI("Acquire::MaxReleaseFileSize", 10*1000*1000));
1893 Header
+= MaximumSize
;
1895 string FinalFile
= _config
->FindDir("Dir::State::lists");
1896 FinalFile
+= URItoFileName(RealURI
);
1899 if (stat(FinalFile
.c_str(),&Buf
) == 0)
1900 Header
+= "\nLast-Modified: " + TimeRFC1123(Buf
.st_mtime
);
1905 // pkgAcqMetaBase::QueueForSignatureVerify /*{{{*/
1906 void pkgAcqMetaBase::QueueForSignatureVerify(const std::string
&MetaIndexFile
,
1907 const std::string
&MetaIndexFileSignature
)
1910 Desc
.URI
= "gpgv:" + MetaIndexFileSignature
;
1911 DestFile
= MetaIndexFile
;
1913 SetActiveSubprocess("gpgv");
1916 // pkgAcqMetaBase::CheckDownloadDone /*{{{*/
1917 bool pkgAcqMetaBase::CheckDownloadDone(const std::string
&Message
,
1918 const std::string
&RealURI
)
1920 // We have just finished downloading a Release file (it is not
1923 string FileName
= LookupTag(Message
,"Filename");
1924 if (FileName
.empty() == true)
1927 ErrorText
= "Method gave a blank filename";
1931 if (FileName
!= DestFile
)
1934 Desc
.URI
= "copy:" + FileName
;
1939 // make sure to verify against the right file on I-M-S hit
1940 IMSHit
= StringToBool(LookupTag(Message
,"IMS-Hit"),false);
1943 string FinalFile
= _config
->FindDir("Dir::State::lists");
1944 FinalFile
+= URItoFileName(RealURI
);
1945 DestFile
= FinalFile
;
1948 // set Item to complete as the remaining work is all local (verify etc)
1954 void pkgAcqMetaBase::QueueIndexes(bool verify
) /*{{{*/
1956 // at this point the real Items are loaded in the fetcher
1957 ExpectedAdditionalItems
= 0;
1959 vector
<struct IndexTarget
*>::const_iterator Target
;
1960 for (Target
= IndexTargets
->begin();
1961 Target
!= IndexTargets
->end();
1964 HashStringList ExpectedIndexHashes
;
1965 const indexRecords::checkSum
*Record
= MetaIndexParser
->Lookup((*Target
)->MetaKey
);
1967 // optional target that we do not have in the Release file are
1969 if (verify
== true && Record
== NULL
&& (*Target
)->IsOptional())
1972 // targets without a hash record are a error when verify is required
1973 if (verify
== true && Record
== NULL
)
1975 Status
= StatAuthError
;
1976 strprintf(ErrorText
, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), (*Target
)->MetaKey
.c_str());
1981 ExpectedIndexHashes
= Record
->Hashes
;
1983 if (_config
->FindB("Debug::pkgAcquire::Auth", false))
1985 std::cerr
<< "Queueing: " << (*Target
)->URI
<< std::endl
1986 << "Expected Hash:" << std::endl
;
1987 for (HashStringList::const_iterator hs
= ExpectedIndexHashes
.begin(); hs
!= ExpectedIndexHashes
.end(); ++hs
)
1988 std::cerr
<< "\t- " << hs
->toStr() << std::endl
;
1989 std::cerr
<< "For: " << Record
->MetaKeyFilename
<< std::endl
;
1992 if (verify
== true && ExpectedIndexHashes
.empty() == true)
1994 Status
= StatAuthError
;
1995 strprintf(ErrorText
, _("Unable to find hash sum for '%s' in Release file"), (*Target
)->MetaKey
.c_str());
1999 /* Queue the Index file (Packages, Sources, Translation-$foo
2000 (either diff or full packages files, depending
2001 on the users option) - we also check if the PDiff Index file is listed
2002 in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
2003 instead, but passing the required info to it is to much hassle */
2004 if(_config
->FindB("Acquire::PDiffs",true) == true && (verify
== false ||
2005 MetaIndexParser
->Exists((*Target
)->MetaKey
+ ".diff/Index") == true))
2006 new pkgAcqDiffIndex(Owner
, TransactionManager
, *Target
, ExpectedIndexHashes
, MetaIndexParser
);
2008 new pkgAcqIndex(Owner
, TransactionManager
, *Target
, ExpectedIndexHashes
, MetaIndexParser
);
2012 bool pkgAcqMetaBase::VerifyVendor(string Message
, const string
&RealURI
)/*{{{*/
2014 string::size_type pos
;
2016 // check for missing sigs (that where not fatal because otherwise we had
2019 string msg
= _("There is no public key available for the "
2020 "following key IDs:\n");
2021 pos
= Message
.find("NO_PUBKEY ");
2022 if (pos
!= std::string::npos
)
2024 string::size_type start
= pos
+strlen("NO_PUBKEY ");
2025 string Fingerprint
= Message
.substr(start
, Message
.find("\n")-start
);
2026 missingkeys
+= (Fingerprint
);
2028 if(!missingkeys
.empty())
2029 _error
->Warning("%s", (msg
+ missingkeys
).c_str());
2031 string Transformed
= MetaIndexParser
->GetExpectedDist();
2033 if (Transformed
== "../project/experimental")
2035 Transformed
= "experimental";
2038 pos
= Transformed
.rfind('/');
2039 if (pos
!= string::npos
)
2041 Transformed
= Transformed
.substr(0, pos
);
2044 if (Transformed
== ".")
2049 if (_config
->FindB("Acquire::Check-Valid-Until", true) == true &&
2050 MetaIndexParser
->GetValidUntil() > 0) {
2051 time_t const invalid_since
= time(NULL
) - MetaIndexParser
->GetValidUntil();
2052 if (invalid_since
> 0)
2053 // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
2054 // the time since then the file is invalid - formated in the same way as in
2055 // the download progress display (e.g. 7d 3h 42min 1s)
2056 return _error
->Error(
2057 _("Release file for %s is expired (invalid since %s). "
2058 "Updates for this repository will not be applied."),
2059 RealURI
.c_str(), TimeToStr(invalid_since
).c_str());
2062 if (_config
->FindB("Debug::pkgAcquire::Auth", false))
2064 std::cerr
<< "Got Codename: " << MetaIndexParser
->GetDist() << std::endl
;
2065 std::cerr
<< "Expecting Dist: " << MetaIndexParser
->GetExpectedDist() << std::endl
;
2066 std::cerr
<< "Transformed Dist: " << Transformed
<< std::endl
;
2069 if (MetaIndexParser
->CheckDist(Transformed
) == false)
2071 // This might become fatal one day
2072 // Status = StatAuthError;
2073 // ErrorText = "Conflicting distribution; expected "
2074 // + MetaIndexParser->GetExpectedDist() + " but got "
2075 // + MetaIndexParser->GetDist();
2077 if (!Transformed
.empty())
2079 _error
->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
2080 Desc
.Description
.c_str(),
2081 Transformed
.c_str(),
2082 MetaIndexParser
->GetDist().c_str());
2089 // pkgAcqMetaIndex::Failed - no Release file present /*{{{*/
2090 void pkgAcqMetaIndex::Failed(string Message
,
2091 pkgAcquire::MethodConfig
* Cnf
)
2093 pkgAcquire::Item::Failed(Message
, Cnf
);
2096 string FinalFile
= _config
->FindDir("Dir::State::lists") + URItoFileName(RealURI
);
2098 _error
->Warning(_("The repository '%s' does not have a Release file. "
2099 "This is deprecated, please contact the owner of the "
2100 "repository."), URIDesc
.c_str());
2102 // No Release file was present so fall
2103 // back to queueing Packages files without verification
2104 // only allow going further if the users explicitely wants it
2105 if(AllowInsecureRepositories(MetaIndexParser
, TransactionManager
, this) == true)
2107 // Done, queue for rename on transaction finished
2108 if (FileExists(DestFile
))
2109 TransactionManager
->TransactionStageCopy(this, DestFile
, FinalFile
);
2111 // queue without any kind of hashsum support
2112 QueueIndexes(false);
2116 void pkgAcqMetaIndex::Finished() /*{{{*/
2118 if(_config
->FindB("Debug::Acquire::Transaction", false) == true)
2119 std::clog
<< "Finished: " << DestFile
<<std::endl
;
2120 if(TransactionManager
!= NULL
&&
2121 TransactionManager
->TransactionHasError() == false)
2122 TransactionManager
->CommitTransaction();
2125 pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire
*Owner
, /*{{{*/
2126 string
const &URI
, string
const &URIDesc
, string
const &ShortDesc
,
2127 string
const &MetaIndexURI
, string
const &MetaIndexURIDesc
, string
const &MetaIndexShortDesc
,
2128 string
const &MetaSigURI
, string
const &MetaSigURIDesc
, string
const &MetaSigShortDesc
,
2129 const vector
<IndexTarget
*>* IndexTargets
,
2130 indexRecords
* MetaIndexParser
) :
2131 pkgAcqMetaIndex(Owner
, NULL
, URI
, URIDesc
, ShortDesc
, MetaSigURI
, MetaSigURIDesc
,MetaSigShortDesc
, IndexTargets
, MetaIndexParser
),
2132 MetaIndexURI(MetaIndexURI
), MetaIndexURIDesc(MetaIndexURIDesc
), MetaIndexShortDesc(MetaIndexShortDesc
),
2133 MetaSigURI(MetaSigURI
), MetaSigURIDesc(MetaSigURIDesc
), MetaSigShortDesc(MetaSigShortDesc
)
2135 // index targets + (worst case:) Release/Release.gpg
2136 ExpectedAdditionalItems
= IndexTargets
->size() + 2;
2140 pkgAcqMetaClearSig::~pkgAcqMetaClearSig() /*{{{*/
2144 // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/
2145 // ---------------------------------------------------------------------
2146 string
pkgAcqMetaClearSig::Custom600Headers() const
2148 string Header
= GetCustom600Headers(RealURI
);
2149 Header
+= "\nFail-Ignore: true";
2153 // pkgAcqMetaClearSig::Done - We got a file /*{{{*/
2154 // ---------------------------------------------------------------------
2155 void pkgAcqMetaClearSig::Done(std::string Message
,unsigned long long /*Size*/,
2156 HashStringList
const &/*Hashes*/,
2157 pkgAcquire::MethodConfig
*Cnf
)
2159 // if we expect a ClearTextSignature (InRelase), ensure that
2160 // this is what we get and if not fail to queue a
2161 // Release/Release.gpg, see #346386
2162 if (FileExists(DestFile
) && !StartsWithGPGClearTextSignature(DestFile
))
2164 pkgAcquire::Item::Failed(Message
, Cnf
);
2165 RenameOnError(NotClearsigned
);
2166 TransactionManager
->AbortTransaction();
2170 if(AuthPass
== false)
2172 if(CheckDownloadDone(Message
, RealURI
) == true)
2173 QueueForSignatureVerify(DestFile
, DestFile
);
2178 if(CheckAuthDone(Message
, RealURI
) == true)
2180 string FinalFile
= _config
->FindDir("Dir::State::lists");
2181 FinalFile
+= URItoFileName(RealURI
);
2183 // queue for copy in place
2184 TransactionManager
->TransactionStageCopy(this, DestFile
, FinalFile
);
2189 void pkgAcqMetaClearSig::Failed(string Message
,pkgAcquire::MethodConfig
*Cnf
) /*{{{*/
2191 Item::Failed(Message
, Cnf
);
2193 // we failed, we will not get additional items from this method
2194 ExpectedAdditionalItems
= 0;
2196 if (AuthPass
== false)
2198 // Queue the 'old' InRelease file for removal if we try Release.gpg
2199 // as otherwise the file will stay around and gives a false-auth
2200 // impression (CVE-2012-0214)
2201 string FinalFile
= _config
->FindDir("Dir::State::lists");
2202 FinalFile
.append(URItoFileName(RealURI
));
2203 TransactionManager
->TransactionStageRemoval(this, FinalFile
);
2206 new pkgAcqMetaIndex(Owner
, TransactionManager
,
2207 MetaIndexURI
, MetaIndexURIDesc
, MetaIndexShortDesc
,
2208 MetaSigURI
, MetaSigURIDesc
, MetaSigShortDesc
,
2209 IndexTargets
, MetaIndexParser
);
2213 if(CheckStopAuthentication(RealURI
, Message
))
2216 _error
->Warning(_("The data from '%s' is not signed. Packages "
2217 "from that repository can not be authenticated."),
2220 // No Release file was present, or verification failed, so fall
2221 // back to queueing Packages files without verification
2222 // only allow going further if the users explicitely wants it
2223 if(AllowInsecureRepositories(MetaIndexParser
, TransactionManager
, this) == true)
2227 /* Always move the meta index, even if gpgv failed. This ensures
2228 * that PackageFile objects are correctly filled in */
2229 if (FileExists(DestFile
))
2231 string FinalFile
= _config
->FindDir("Dir::State::lists");
2232 FinalFile
+= URItoFileName(RealURI
);
2233 /* InRelease files become Release files, otherwise
2234 * they would be considered as trusted later on */
2235 RealURI
= RealURI
.replace(RealURI
.rfind("InRelease"), 9,
2237 FinalFile
= FinalFile
.replace(FinalFile
.rfind("InRelease"), 9,
2240 // Done, queue for rename on transaction finished
2241 TransactionManager
->TransactionStageCopy(this, DestFile
, FinalFile
);
2243 QueueIndexes(false);
2248 // AcqArchive::AcqArchive - Constructor /*{{{*/
2249 // ---------------------------------------------------------------------
2250 /* This just sets up the initial fetch environment and queues the first
2252 pkgAcqArchive::pkgAcqArchive(pkgAcquire
*Owner
,pkgSourceList
*Sources
,
2253 pkgRecords
*Recs
,pkgCache::VerIterator
const &Version
,
2254 string
&StoreFilename
) :
2255 Item(Owner
, HashStringList()), Version(Version
), Sources(Sources
), Recs(Recs
),
2256 StoreFilename(StoreFilename
), Vf(Version
.FileList()),
2259 Retries
= _config
->FindI("Acquire::Retries",0);
2261 if (Version
.Arch() == 0)
2263 _error
->Error(_("I wasn't able to locate a file for the %s package. "
2264 "This might mean you need to manually fix this package. "
2265 "(due to missing arch)"),
2266 Version
.ParentPkg().FullName().c_str());
2270 /* We need to find a filename to determine the extension. We make the
2271 assumption here that all the available sources for this version share
2272 the same extension.. */
2273 // Skip not source sources, they do not have file fields.
2274 for (; Vf
.end() == false; ++Vf
)
2276 if ((Vf
.File()->Flags
& pkgCache::Flag::NotSource
) != 0)
2281 // Does not really matter here.. we are going to fail out below
2282 if (Vf
.end() != true)
2284 // If this fails to get a file name we will bomb out below.
2285 pkgRecords::Parser
&Parse
= Recs
->Lookup(Vf
);
2286 if (_error
->PendingError() == true)
2289 // Generate the final file name as: package_version_arch.foo
2290 StoreFilename
= QuoteString(Version
.ParentPkg().Name(),"_:") + '_' +
2291 QuoteString(Version
.VerStr(),"_:") + '_' +
2292 QuoteString(Version
.Arch(),"_:.") +
2293 "." + flExtension(Parse
.FileName());
2296 // check if we have one trusted source for the package. if so, switch
2297 // to "TrustedOnly" mode - but only if not in AllowUnauthenticated mode
2298 bool const allowUnauth
= _config
->FindB("APT::Get::AllowUnauthenticated", false);
2299 bool const debugAuth
= _config
->FindB("Debug::pkgAcquire::Auth", false);
2300 bool seenUntrusted
= false;
2301 for (pkgCache::VerFileIterator i
= Version
.FileList(); i
.end() == false; ++i
)
2303 pkgIndexFile
*Index
;
2304 if (Sources
->FindIndex(i
.File(),Index
) == false)
2307 if (debugAuth
== true)
2308 std::cerr
<< "Checking index: " << Index
->Describe()
2309 << "(Trusted=" << Index
->IsTrusted() << ")" << std::endl
;
2311 if (Index
->IsTrusted() == true)
2314 if (allowUnauth
== false)
2318 seenUntrusted
= true;
2321 // "allow-unauthenticated" restores apts old fetching behaviour
2322 // that means that e.g. unauthenticated file:// uris are higher
2323 // priority than authenticated http:// uris
2324 if (allowUnauth
== true && seenUntrusted
== true)
2328 if (QueueNext() == false && _error
->PendingError() == false)
2329 _error
->Error(_("Can't find a source to download version '%s' of '%s'"),
2330 Version
.VerStr(), Version
.ParentPkg().FullName(false).c_str());
2333 // AcqArchive::QueueNext - Queue the next file source /*{{{*/
2334 // ---------------------------------------------------------------------
2335 /* This queues the next available file version for download. It checks if
2336 the archive is already available in the cache and stashs the MD5 for
2338 bool pkgAcqArchive::QueueNext()
2340 for (; Vf
.end() == false; ++Vf
)
2342 // Ignore not source sources
2343 if ((Vf
.File()->Flags
& pkgCache::Flag::NotSource
) != 0)
2346 // Try to cross match against the source list
2347 pkgIndexFile
*Index
;
2348 if (Sources
->FindIndex(Vf
.File(),Index
) == false)
2351 // only try to get a trusted package from another source if that source
2353 if(Trusted
&& !Index
->IsTrusted())
2356 // Grab the text package record
2357 pkgRecords::Parser
&Parse
= Recs
->Lookup(Vf
);
2358 if (_error
->PendingError() == true)
2361 string PkgFile
= Parse
.FileName();
2362 ExpectedHashes
= Parse
.Hashes();
2364 if (PkgFile
.empty() == true)
2365 return _error
->Error(_("The package index files are corrupted. No Filename: "
2366 "field for package %s."),
2367 Version
.ParentPkg().Name());
2369 Desc
.URI
= Index
->ArchiveURI(PkgFile
);
2370 Desc
.Description
= Index
->ArchiveInfo(Version
);
2372 Desc
.ShortDesc
= Version
.ParentPkg().FullName(true);
2374 // See if we already have the file. (Legacy filenames)
2375 FileSize
= Version
->Size
;
2376 string FinalFile
= _config
->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile
);
2378 if (stat(FinalFile
.c_str(),&Buf
) == 0)
2380 // Make sure the size matches
2381 if ((unsigned long long)Buf
.st_size
== Version
->Size
)
2386 StoreFilename
= DestFile
= FinalFile
;
2390 /* Hmm, we have a file and its size does not match, this means it is
2391 an old style mismatched arch */
2392 unlink(FinalFile
.c_str());
2395 // Check it again using the new style output filenames
2396 FinalFile
= _config
->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename
);
2397 if (stat(FinalFile
.c_str(),&Buf
) == 0)
2399 // Make sure the size matches
2400 if ((unsigned long long)Buf
.st_size
== Version
->Size
)
2405 StoreFilename
= DestFile
= FinalFile
;
2409 /* Hmm, we have a file and its size does not match, this shouldn't
2411 unlink(FinalFile
.c_str());
2414 DestFile
= _config
->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename
);
2416 // Check the destination file
2417 if (stat(DestFile
.c_str(),&Buf
) == 0)
2419 // Hmm, the partial file is too big, erase it
2420 if ((unsigned long long)Buf
.st_size
> Version
->Size
)
2421 unlink(DestFile
.c_str());
2424 PartialSize
= Buf
.st_size
;
2425 std::string SandboxUser
= _config
->Find("APT::Sandbox::User");
2426 ChangeOwnerAndPermissionOfFile("pkgAcqArchive::QueueNext",DestFile
.c_str(), SandboxUser
.c_str(), "root", 0600);
2430 // Disables download of archives - useful if no real installation follows,
2431 // e.g. if we are just interested in proposed installation order
2432 if (_config
->FindB("Debug::pkgAcqArchive::NoQueue", false) == true)
2437 StoreFilename
= DestFile
= FinalFile
;
2451 // AcqArchive::Done - Finished fetching /*{{{*/
2452 // ---------------------------------------------------------------------
2454 void pkgAcqArchive::Done(string Message
,unsigned long long Size
, HashStringList
const &CalcHashes
,
2455 pkgAcquire::MethodConfig
*Cfg
)
2457 Item::Done(Message
, Size
, CalcHashes
, Cfg
);
2460 if (Size
!= Version
->Size
)
2462 RenameOnError(SizeMismatch
);
2466 // FIXME: could this empty() check impose *any* sort of security issue?
2467 if(ExpectedHashes
.usable() && ExpectedHashes
!= CalcHashes
)
2469 RenameOnError(HashSumMismatch
);
2470 printHashSumComparision(DestFile
, ExpectedHashes
, CalcHashes
);
2474 // Grab the output filename
2475 string FileName
= LookupTag(Message
,"Filename");
2476 if (FileName
.empty() == true)
2479 ErrorText
= "Method gave a blank filename";
2483 // Reference filename
2484 if (FileName
!= DestFile
)
2486 StoreFilename
= DestFile
= FileName
;
2492 // Done, move it into position
2493 string FinalFile
= _config
->FindDir("Dir::Cache::Archives");
2494 FinalFile
+= flNotDir(StoreFilename
);
2495 Rename(DestFile
,FinalFile
);
2496 ChangeOwnerAndPermissionOfFile("pkgAcqArchive::Done", FinalFile
.c_str(), "root", "root", 0644);
2497 StoreFilename
= DestFile
= FinalFile
;
2501 // AcqArchive::Failed - Failure handler /*{{{*/
2502 // ---------------------------------------------------------------------
2503 /* Here we try other sources */
2504 void pkgAcqArchive::Failed(string Message
,pkgAcquire::MethodConfig
*Cnf
)
2506 ErrorText
= LookupTag(Message
,"Message");
2508 /* We don't really want to retry on failed media swaps, this prevents
2509 that. An interesting observation is that permanent failures are not
2511 if (Cnf
->Removable
== true &&
2512 StringToBool(LookupTag(Message
,"Transient-Failure"),false) == true)
2514 // Vf = Version.FileList();
2515 while (Vf
.end() == false) ++Vf
;
2516 StoreFilename
= string();
2517 Item::Failed(Message
,Cnf
);
2521 if (QueueNext() == false)
2523 // This is the retry counter
2525 Cnf
->LocalOnly
== false &&
2526 StringToBool(LookupTag(Message
,"Transient-Failure"),false) == true)
2529 Vf
= Version
.FileList();
2530 if (QueueNext() == true)
2534 StoreFilename
= string();
2535 Item::Failed(Message
,Cnf
);
2539 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
2540 // ---------------------------------------------------------------------
2541 APT_PURE
bool pkgAcqArchive::IsTrusted() const
2546 // AcqArchive::Finished - Fetching has finished, tidy up /*{{{*/
2547 // ---------------------------------------------------------------------
2549 void pkgAcqArchive::Finished()
2551 if (Status
== pkgAcquire::Item::StatDone
&&
2554 StoreFilename
= string();
2557 // AcqFile::pkgAcqFile - Constructor /*{{{*/
2558 // ---------------------------------------------------------------------
2559 /* The file is added to the queue */
2560 pkgAcqFile::pkgAcqFile(pkgAcquire
*Owner
,string URI
, HashStringList
const &Hashes
,
2561 unsigned long long Size
,string Dsc
,string ShortDesc
,
2562 const string
&DestDir
, const string
&DestFilename
,
2564 Item(Owner
, Hashes
), IsIndexFile(IsIndexFile
)
2566 Retries
= _config
->FindI("Acquire::Retries",0);
2568 if(!DestFilename
.empty())
2569 DestFile
= DestFilename
;
2570 else if(!DestDir
.empty())
2571 DestFile
= DestDir
+ "/" + flNotDir(URI
);
2573 DestFile
= flNotDir(URI
);
2577 Desc
.Description
= Dsc
;
2580 // Set the short description to the archive component
2581 Desc
.ShortDesc
= ShortDesc
;
2583 // Get the transfer sizes
2586 if (stat(DestFile
.c_str(),&Buf
) == 0)
2588 // Hmm, the partial file is too big, erase it
2589 if ((Size
> 0) && (unsigned long long)Buf
.st_size
> Size
)
2590 unlink(DestFile
.c_str());
2593 PartialSize
= Buf
.st_size
;
2594 std::string SandboxUser
= _config
->Find("APT::Sandbox::User");
2595 ChangeOwnerAndPermissionOfFile("pkgAcqFile", DestFile
.c_str(), SandboxUser
.c_str(), "root", 0600);
2602 // AcqFile::Done - Item downloaded OK /*{{{*/
2603 // ---------------------------------------------------------------------
2605 void pkgAcqFile::Done(string Message
,unsigned long long Size
,HashStringList
const &CalcHashes
,
2606 pkgAcquire::MethodConfig
*Cnf
)
2608 Item::Done(Message
,Size
,CalcHashes
,Cnf
);
2611 if(ExpectedHashes
.usable() && ExpectedHashes
!= CalcHashes
)
2613 RenameOnError(HashSumMismatch
);
2614 printHashSumComparision(DestFile
, ExpectedHashes
, CalcHashes
);
2618 string FileName
= LookupTag(Message
,"Filename");
2619 if (FileName
.empty() == true)
2622 ErrorText
= "Method gave a blank filename";
2628 // The files timestamp matches
2629 if (StringToBool(LookupTag(Message
,"IMS-Hit"),false) == true)
2632 // We have to copy it into place
2633 if (FileName
!= DestFile
)
2636 if (_config
->FindB("Acquire::Source-Symlinks",true) == false ||
2637 Cnf
->Removable
== true)
2639 Desc
.URI
= "copy:" + FileName
;
2644 // Erase the file if it is a symlink so we can overwrite it
2646 if (lstat(DestFile
.c_str(),&St
) == 0)
2648 if (S_ISLNK(St
.st_mode
) != 0)
2649 unlink(DestFile
.c_str());
2653 if (symlink(FileName
.c_str(),DestFile
.c_str()) != 0)
2655 ErrorText
= "Link to " + DestFile
+ " failure ";
2662 // AcqFile::Failed - Failure handler /*{{{*/
2663 // ---------------------------------------------------------------------
2664 /* Here we try other sources */
2665 void pkgAcqFile::Failed(string Message
,pkgAcquire::MethodConfig
*Cnf
)
2667 ErrorText
= LookupTag(Message
,"Message");
2669 // This is the retry counter
2671 Cnf
->LocalOnly
== false &&
2672 StringToBool(LookupTag(Message
,"Transient-Failure"),false) == true)
2679 Item::Failed(Message
,Cnf
);
2682 // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/
2683 // ---------------------------------------------------------------------
2684 /* The only header we use is the last-modified header. */
2685 string
pkgAcqFile::Custom600Headers() const
2688 return "\nIndex-File: true";