1 // -*- mode: cpp; mode: fold -*-
3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
4 /* ######################################################################
6 Acquire Item - Item to acquire
8 Each item can download to exactly one file at a time. This means you
9 cannot create an item that fetches two uri's to two files at the same
10 time. The pkgAcqIndex class creates a second class upon instantiation
11 to fetch the other index files because of this.
13 ##################################################################### */
15 // Include Files /*{{{*/
16 #include <apt-pkg/acquire-item.h>
17 #include <apt-pkg/configuration.h>
18 #include <apt-pkg/sourcelist.h>
19 #include <apt-pkg/vendorlist.h>
20 #include <apt-pkg/error.h>
21 #include <apt-pkg/strutl.h>
22 #include <apt-pkg/fileutl.h>
23 #include <apt-pkg/md5.h>
24 #include <apt-pkg/sha1.h>
25 #include <apt-pkg/tagfile.h>
39 // Acquire::Item::Item - Constructor /*{{{*/
40 // ---------------------------------------------------------------------
42 pkgAcquire::Item::Item(pkgAcquire
*Owner
) : Owner(Owner
), FileSize(0),
43 PartialSize(0), Mode(0), ID(0), Complete(false),
44 Local(false), QueueCounter(0)
50 // Acquire::Item::~Item - Destructor /*{{{*/
51 // ---------------------------------------------------------------------
53 pkgAcquire::Item::~Item()
58 // Acquire::Item::Failed - Item failed to download /*{{{*/
59 // ---------------------------------------------------------------------
60 /* We return to an idle state if there are still other queues that could
62 void pkgAcquire::Item::Failed(string Message
,pkgAcquire::MethodConfig
*Cnf
)
65 ErrorText
= LookupTag(Message
,"Message");
66 if (QueueCounter
<= 1)
68 /* This indicates that the file is not available right now but might
69 be sometime later. If we do a retry cycle then this should be
71 if (Cnf
->LocalOnly
== true &&
72 StringToBool(LookupTag(Message
,"Transient-Failure"),false) == true)
84 // Acquire::Item::Start - Item has begun to download /*{{{*/
85 // ---------------------------------------------------------------------
86 /* Stash status and the file size. Note that setting Complete means
87 sub-phases of the acquire process such as decompresion are operating */
88 void pkgAcquire::Item::Start(string
/*Message*/,unsigned long Size
)
90 Status
= StatFetching
;
91 if (FileSize
== 0 && Complete
== false)
95 // Acquire::Item::Done - Item downloaded OK /*{{{*/
96 // ---------------------------------------------------------------------
98 void pkgAcquire::Item::Done(string Message
,unsigned long Size
,string Hash
,
99 pkgAcquire::MethodConfig
*Cnf
)
101 // We just downloaded something..
102 string FileName
= LookupTag(Message
,"Filename");
103 // we only inform the Log class if it was actually not a local thing
104 if (Complete
== false && !Local
&& FileName
== DestFile
)
107 Owner
->Log
->Fetched(Size
,atoi(LookupTag(Message
,"Resume-Point","0").c_str()));
114 ErrorText
= string();
115 Owner
->Dequeue(this);
118 // Acquire::Item::Rename - Rename a file /*{{{*/
119 // ---------------------------------------------------------------------
120 /* This helper function is used by alot of item methods as thier final
122 void pkgAcquire::Item::Rename(string From
,string To
)
124 if (rename(From
.c_str(),To
.c_str()) != 0)
127 snprintf(S
,sizeof(S
),_("rename failed, %s (%s -> %s)."),strerror(errno
),
128 From
.c_str(),To
.c_str());
136 // AcqDiffIndex::AcqDiffIndex - Constructor
137 // ---------------------------------------------------------------------
138 /* Get the DiffIndex file first and see if there are patches availabe
139 * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
140 * patches. If anything goes wrong in that process, it will fall back to
141 * the original packages file
143 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire
*Owner
,
144 string URI
,string URIDesc
,string ShortDesc
,
145 HashString ExpectedHash
)
146 : Item(Owner
), RealURI(URI
), ExpectedHash(ExpectedHash
),
150 Debug
= _config
->FindB("Debug::pkgAcquire::Diffs",false);
152 Desc
.Description
= URIDesc
+ "/DiffIndex";
154 Desc
.ShortDesc
= ShortDesc
;
155 Desc
.URI
= URI
+ ".diff/Index";
157 DestFile
= _config
->FindDir("Dir::State::lists") + "partial/";
158 DestFile
+= URItoFileName(URI
) + string(".DiffIndex");
161 std::clog
<< "pkgAcqDiffIndex: " << Desc
.URI
<< std::endl
;
163 // look for the current package file
164 CurrentPackagesFile
= _config
->FindDir("Dir::State::lists");
165 CurrentPackagesFile
+= URItoFileName(RealURI
);
167 // FIXME: this file:/ check is a hack to prevent fetching
168 // from local sources. this is really silly, and
169 // should be fixed cleanly as soon as possible
170 if(!FileExists(CurrentPackagesFile
) ||
171 Desc
.URI
.substr(0,strlen("file:/")) == "file:/")
173 // we don't have a pkg file or we don't want to queue
175 std::clog
<< "No index file, local or canceld by user" << std::endl
;
181 std::clog
<< "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): "
182 << CurrentPackagesFile
<< std::endl
;
188 // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/
189 // ---------------------------------------------------------------------
190 /* The only header we use is the last-modified header. */
191 string
pkgAcqDiffIndex::Custom600Headers()
193 string Final
= _config
->FindDir("Dir::State::lists");
194 Final
+= URItoFileName(RealURI
) + string(".IndexDiff");
197 std::clog
<< "Custom600Header-IMS: " << Final
<< std::endl
;
200 if (stat(Final
.c_str(),&Buf
) != 0)
201 return "\nIndex-File: true";
203 return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf
.st_mtime
);
207 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile
)
210 std::clog
<< "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile
215 vector
<DiffInfo
> available_patches
;
217 FileFd
Fd(IndexDiffFile
,FileFd::ReadOnly
);
219 if (_error
->PendingError() == true)
222 if(TF
.Step(Tags
) == true)
229 string tmp
= Tags
.FindS("SHA1-Current");
230 std::stringstream
ss(tmp
);
233 FileFd
fd(CurrentPackagesFile
, FileFd::ReadOnly
);
235 SHA1
.AddFD(fd
.Fd(), fd
.Size());
236 local_sha1
= string(SHA1
.Result());
238 if(local_sha1
== ServerSha1
)
240 // we have the same sha1 as the server
242 std::clog
<< "Package file is up-to-date" << std::endl
;
243 // set found to true, this will queue a pkgAcqIndexDiffs with
244 // a empty availabe_patches
250 std::clog
<< "SHA1-Current: " << ServerSha1
<< std::endl
;
252 // check the historie and see what patches we need
253 string history
= Tags
.FindS("SHA1-History");
254 std::stringstream
hist(history
);
255 while(hist
>> d
.sha1
>> size
>> d
.file
)
257 d
.size
= atoi(size
.c_str());
258 // read until the first match is found
259 if(d
.sha1
== local_sha1
)
261 // from that point on, we probably need all diffs
265 std::clog
<< "Need to get diff: " << d
.file
<< std::endl
;
266 available_patches
.push_back(d
);
271 // we have something, queue the next diff
275 string::size_type last_space
= Description
.rfind(" ");
276 if(last_space
!= string::npos
)
277 Description
.erase(last_space
, Description
.size()-last_space
);
278 new pkgAcqIndexDiffs(Owner
, RealURI
, Description
, Desc
.ShortDesc
,
279 ExpectedHash
, available_patches
);
287 // Nothing found, report and return false
288 // Failing here is ok, if we return false later, the full
289 // IndexFile is queued
291 std::clog
<< "Can't find a patch in the index file" << std::endl
;
295 void pkgAcqDiffIndex::Failed(string Message
,pkgAcquire::MethodConfig
*Cnf
)
298 std::clog
<< "pkgAcqDiffIndex failed: " << Desc
.URI
<< std::endl
299 << "Falling back to normal index file aquire" << std::endl
;
301 new pkgAcqIndex(Owner
, RealURI
, Description
, Desc
.ShortDesc
,
309 void pkgAcqDiffIndex::Done(string Message
,unsigned long Size
,string Md5Hash
,
310 pkgAcquire::MethodConfig
*Cnf
)
313 std::clog
<< "pkgAcqDiffIndex::Done(): " << Desc
.URI
<< std::endl
;
315 Item::Done(Message
,Size
,Md5Hash
,Cnf
);
318 FinalFile
= _config
->FindDir("Dir::State::lists")+URItoFileName(RealURI
);
320 // sucess in downloading the index
322 FinalFile
+= string(".IndexDiff");
324 std::clog
<< "Renaming: " << DestFile
<< " -> " << FinalFile
326 Rename(DestFile
,FinalFile
);
327 chmod(FinalFile
.c_str(),0644);
328 DestFile
= FinalFile
;
330 if(!ParseDiffIndex(DestFile
))
331 return Failed("", NULL
);
341 // AcqIndexDiffs::AcqIndexDiffs - Constructor
342 // ---------------------------------------------------------------------
343 /* The package diff is added to the queue. one object is constructed
344 * for each diff and the index
346 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire
*Owner
,
347 string URI
,string URIDesc
,string ShortDesc
,
348 HashString ExpectedMD5
,
349 vector
<DiffInfo
> diffs
)
350 : Item(Owner
), RealURI(URI
), ExpectedHash(ExpectedHash
),
351 available_patches(diffs
)
354 DestFile
= _config
->FindDir("Dir::State::lists") + "partial/";
355 DestFile
+= URItoFileName(URI
);
357 Debug
= _config
->FindB("Debug::pkgAcquire::Diffs",false);
359 Description
= URIDesc
;
361 Desc
.ShortDesc
= ShortDesc
;
363 if(available_patches
.size() == 0)
365 // we are done (yeah!)
371 State
= StateFetchDiff
;
377 void pkgAcqIndexDiffs::Failed(string Message
,pkgAcquire::MethodConfig
*Cnf
)
380 std::clog
<< "pkgAcqIndexDiffs failed: " << Desc
.URI
<< std::endl
381 << "Falling back to normal index file aquire" << std::endl
;
382 new pkgAcqIndex(Owner
, RealURI
, Description
,Desc
.ShortDesc
,
388 // helper that cleans the item out of the fetcher queue
389 void pkgAcqIndexDiffs::Finish(bool allDone
)
391 // we restore the original name, this is required, otherwise
392 // the file will be cleaned
395 DestFile
= _config
->FindDir("Dir::State::lists");
396 DestFile
+= URItoFileName(RealURI
);
398 if(!ExpectedHash
.empty() && !ExpectedHash
.VerifyFile(DestFile
))
400 Status
= StatAuthError
;
401 ErrorText
= _("MD5Sum mismatch");
402 Rename(DestFile
,DestFile
+ ".FAILED");
407 // this is for the "real" finish
412 std::clog
<< "\n\nallDone: " << DestFile
<< "\n" << std::endl
;
417 std::clog
<< "Finishing: " << Desc
.URI
<< std::endl
;
426 bool pkgAcqIndexDiffs::QueueNextDiff()
429 // calc sha1 of the just patched file
430 string FinalFile
= _config
->FindDir("Dir::State::lists");
431 FinalFile
+= URItoFileName(RealURI
);
433 FileFd
fd(FinalFile
, FileFd::ReadOnly
);
435 SHA1
.AddFD(fd
.Fd(), fd
.Size());
436 string local_sha1
= string(SHA1
.Result());
438 std::clog
<< "QueueNextDiff: "
439 << FinalFile
<< " (" << local_sha1
<< ")"<<std::endl
;
441 // remove all patches until the next matching patch is found
442 // this requires the Index file to be ordered
443 for(vector
<DiffInfo
>::iterator I
=available_patches
.begin();
444 available_patches
.size() > 0 &&
445 I
!= available_patches
.end() &&
446 (*I
).sha1
!= local_sha1
;
449 available_patches
.erase(I
);
452 // error checking and falling back if no patch was found
453 if(available_patches
.size() == 0)
459 // queue the right diff
460 Desc
.URI
= string(RealURI
) + ".diff/" + available_patches
[0].file
+ ".gz";
461 Desc
.Description
= Description
+ " " + available_patches
[0].file
+ string(".pdiff");
462 DestFile
= _config
->FindDir("Dir::State::lists") + "partial/";
463 DestFile
+= URItoFileName(RealURI
+ ".diff/" + available_patches
[0].file
);
466 std::clog
<< "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc
.URI
<< std::endl
;
475 void pkgAcqIndexDiffs::Done(string Message
,unsigned long Size
,string Md5Hash
,
476 pkgAcquire::MethodConfig
*Cnf
)
479 std::clog
<< "pkgAcqIndexDiffs::Done(): " << Desc
.URI
<< std::endl
;
481 Item::Done(Message
,Size
,Md5Hash
,Cnf
);
484 FinalFile
= _config
->FindDir("Dir::State::lists")+URItoFileName(RealURI
);
486 // sucess in downloading a diff, enter ApplyDiff state
487 if(State
== StateFetchDiff
)
491 std::clog
<< "Sending to gzip method: " << FinalFile
<< std::endl
;
493 string FileName
= LookupTag(Message
,"Filename");
494 State
= StateUnzipDiff
;
496 Desc
.URI
= "gzip:" + FileName
;
497 DestFile
+= ".decomp";
503 // sucess in downloading a diff, enter ApplyDiff state
504 if(State
== StateUnzipDiff
)
507 // rred excepts the patch as $FinalFile.ed
508 Rename(DestFile
,FinalFile
+".ed");
511 std::clog
<< "Sending to rred method: " << FinalFile
<< std::endl
;
513 State
= StateApplyDiff
;
515 Desc
.URI
= "rred:" + FinalFile
;
522 // success in download/apply a diff, queue next (if needed)
523 if(State
== StateApplyDiff
)
525 // remove the just applied patch
526 available_patches
.erase(available_patches
.begin());
531 std::clog
<< "Moving patched file in place: " << std::endl
532 << DestFile
<< " -> " << FinalFile
<< std::endl
;
534 Rename(DestFile
,FinalFile
);
535 chmod(FinalFile
.c_str(),0644);
537 // see if there is more to download
538 if(available_patches
.size() > 0) {
539 new pkgAcqIndexDiffs(Owner
, RealURI
, Description
, Desc
.ShortDesc
,
540 ExpectedHash
, available_patches
);
548 // AcqIndex::AcqIndex - Constructor /*{{{*/
549 // ---------------------------------------------------------------------
550 /* The package file is added to the queue and a second class is
551 instantiated to fetch the revision file */
552 pkgAcqIndex::pkgAcqIndex(pkgAcquire
*Owner
,
553 string URI
,string URIDesc
,string ShortDesc
,
554 HashString ExpectedHash
, string comprExt
)
555 : Item(Owner
), RealURI(URI
), ExpectedHash(ExpectedHash
)
557 Decompression
= false;
560 DestFile
= _config
->FindDir("Dir::State::lists") + "partial/";
561 DestFile
+= URItoFileName(URI
);
565 // autoselect the compression method
566 if(FileExists("/bin/bzip2"))
567 CompressionExtension
= ".bz2";
569 CompressionExtension
= ".gz";
571 CompressionExtension
= comprExt
;
573 Desc
.URI
= URI
+ CompressionExtension
;
575 Desc
.Description
= URIDesc
;
577 Desc
.ShortDesc
= ShortDesc
;
582 // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/
583 // ---------------------------------------------------------------------
584 /* The only header we use is the last-modified header. */
585 string
pkgAcqIndex::Custom600Headers()
587 string Final
= _config
->FindDir("Dir::State::lists");
588 Final
+= URItoFileName(RealURI
);
591 if (stat(Final
.c_str(),&Buf
) != 0)
592 return "\nIndex-File: true";
594 return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf
.st_mtime
);
598 void pkgAcqIndex::Failed(string Message
,pkgAcquire::MethodConfig
*Cnf
)
600 // no .bz2 found, retry with .gz
601 if(Desc
.URI
.substr(Desc
.URI
.size()-3) == "bz2") {
602 Desc
.URI
= Desc
.URI
.substr(0,Desc
.URI
.size()-3) + "gz";
604 // retry with a gzip one
605 new pkgAcqIndex(Owner
, RealURI
, Desc
.Description
,Desc
.ShortDesc
,
606 ExpectedHash
, string(".gz"));
613 // on decompression failure, remove bad versions in partial/
614 if(Decompression
&& Erase
) {
615 string s
= _config
->FindDir("Dir::State::lists") + "partial/";
616 s
+= URItoFileName(RealURI
);
620 Item::Failed(Message
,Cnf
);
624 // AcqIndex::Done - Finished a fetch /*{{{*/
625 // ---------------------------------------------------------------------
626 /* This goes through a number of states.. On the initial fetch the
627 method could possibly return an alternate filename which points
628 to the uncompressed version of the file. If this is so the file
629 is copied into the partial directory. In all other cases the file
630 is decompressed with a gzip uri. */
631 void pkgAcqIndex::Done(string Message
,unsigned long Size
,string Hash
,
632 pkgAcquire::MethodConfig
*Cfg
)
634 Item::Done(Message
,Size
,Hash
,Cfg
);
636 if (Decompression
== true)
638 if (_config
->FindB("Debug::pkgAcquire::Auth", false))
640 std::cerr
<< std::endl
<< RealURI
<< ": Computed Hash: " << Hash
;
641 std::cerr
<< " Expected Hash: " << ExpectedHash
.toStr() << std::endl
;
644 if (!ExpectedHash
.empty() && !ExpectedHash
.VerifyFile(DestFile
))
646 Status
= StatAuthError
;
647 ErrorText
= _("Hash Sum mismatch");
648 Rename(DestFile
,DestFile
+ ".FAILED");
651 // Done, move it into position
652 string FinalFile
= _config
->FindDir("Dir::State::lists");
653 FinalFile
+= URItoFileName(RealURI
);
654 Rename(DestFile
,FinalFile
);
655 chmod(FinalFile
.c_str(),0644);
657 /* We restore the original name to DestFile so that the clean operation
659 DestFile
= _config
->FindDir("Dir::State::lists") + "partial/";
660 DestFile
+= URItoFileName(RealURI
);
662 // Remove the compressed version.
664 unlink(DestFile
.c_str());
671 // Handle the unzipd case
672 string FileName
= LookupTag(Message
,"Alt-Filename");
673 if (FileName
.empty() == false)
675 // The files timestamp matches
676 if (StringToBool(LookupTag(Message
,"Alt-IMS-Hit"),false) == true)
679 Decompression
= true;
681 DestFile
+= ".decomp";
682 Desc
.URI
= "copy:" + FileName
;
688 FileName
= LookupTag(Message
,"Filename");
689 if (FileName
.empty() == true)
692 ErrorText
= "Method gave a blank filename";
695 // The files timestamp matches
696 if (StringToBool(LookupTag(Message
,"IMS-Hit"),false) == true)
699 if (FileName
== DestFile
)
704 string compExt
= Desc
.URI
.substr(Desc
.URI
.size()-3);
707 decompProg
= "bzip2";
708 else if(compExt
== ".gz")
711 _error
->Error("Unsupported extension: %s", compExt
.c_str());
715 Decompression
= true;
716 DestFile
+= ".decomp";
717 Desc
.URI
= string(decompProg
) + ":" + FileName
;
722 // AcqIndexTrans::pkgAcqIndexTrans - Constructor /*{{{*/
723 // ---------------------------------------------------------------------
724 /* The Translation file is added to the queue */
725 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire
*Owner
,
726 string URI
,string URIDesc
,string ShortDesc
)
727 : pkgAcqIndex(Owner
, URI
, URIDesc
, ShortDesc
, HashString(), "")
732 // AcqIndexTrans::Failed - Silence failure messages for missing files /*{{{*/
733 // ---------------------------------------------------------------------
735 void pkgAcqIndexTrans::Failed(string Message
,pkgAcquire::MethodConfig
*Cnf
)
737 if (Cnf
->LocalOnly
== true ||
738 StringToBool(LookupTag(Message
,"Transient-Failure"),false) == false)
747 Item::Failed(Message
,Cnf
);
751 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire
*Owner
,
752 string URI
,string URIDesc
,string ShortDesc
,
753 string MetaIndexURI
, string MetaIndexURIDesc
,
754 string MetaIndexShortDesc
,
755 const vector
<IndexTarget
*>* IndexTargets
,
756 indexRecords
* MetaIndexParser
) :
757 Item(Owner
), RealURI(URI
), MetaIndexURI(MetaIndexURI
),
758 MetaIndexURIDesc(MetaIndexURIDesc
), MetaIndexShortDesc(MetaIndexShortDesc
),
759 MetaIndexParser(MetaIndexParser
), IndexTargets(IndexTargets
)
761 DestFile
= _config
->FindDir("Dir::State::lists") + "partial/";
762 DestFile
+= URItoFileName(URI
);
764 // remove any partial downloaded sig-file in partial/.
765 // it may confuse proxies and is too small to warrant a
766 // partial download anyway
767 unlink(DestFile
.c_str());
770 Desc
.Description
= URIDesc
;
772 Desc
.ShortDesc
= ShortDesc
;
776 string Final
= _config
->FindDir("Dir::State::lists");
777 Final
+= URItoFileName(RealURI
);
779 if (stat(Final
.c_str(),&Buf
) == 0)
781 // File was already in place. It needs to be re-verified
782 // because Release might have changed, so Move it into partial
783 Rename(Final
,DestFile
);
789 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers /*{{{*/
790 // ---------------------------------------------------------------------
791 /* The only header we use is the last-modified header. */
792 string
pkgAcqMetaSig::Custom600Headers()
795 if (stat(DestFile
.c_str(),&Buf
) != 0)
796 return "\nIndex-File: true";
798 return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf
.st_mtime
);
801 void pkgAcqMetaSig::Done(string Message
,unsigned long Size
,string MD5
,
802 pkgAcquire::MethodConfig
*Cfg
)
804 Item::Done(Message
,Size
,MD5
,Cfg
);
806 string FileName
= LookupTag(Message
,"Filename");
807 if (FileName
.empty() == true)
810 ErrorText
= "Method gave a blank filename";
814 if (FileName
!= DestFile
)
816 // We have to copy it into place
818 Desc
.URI
= "copy:" + FileName
;
825 // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
826 new pkgAcqMetaIndex(Owner
, MetaIndexURI
, MetaIndexURIDesc
, MetaIndexShortDesc
,
827 DestFile
, IndexTargets
, MetaIndexParser
);
831 void pkgAcqMetaSig::Failed(string Message
,pkgAcquire::MethodConfig
*Cnf
)
833 string Final
= _config
->FindDir("Dir::State::lists") + URItoFileName(RealURI
);
835 // if we get a network error we fail gracefully
836 if(Status
== StatTransientNetworkError
)
838 Item::Failed(Message
,Cnf
);
839 // move the sigfile back on transient network failures
840 if(FileExists(DestFile
))
841 Rename(DestFile
,Final
);
843 // set the status back to , Item::Failed likes to reset it
844 Status
= pkgAcquire::Item::StatTransientNetworkError
;
848 // Delete any existing sigfile when the acquire failed
849 unlink(Final
.c_str());
851 // queue a pkgAcqMetaIndex with no sigfile
852 new pkgAcqMetaIndex(Owner
, MetaIndexURI
, MetaIndexURIDesc
, MetaIndexShortDesc
,
853 "", IndexTargets
, MetaIndexParser
);
855 if (Cnf
->LocalOnly
== true ||
856 StringToBool(LookupTag(Message
,"Transient-Failure"),false) == false)
865 Item::Failed(Message
,Cnf
);
868 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire
*Owner
,
869 string URI
,string URIDesc
,string ShortDesc
,
871 const vector
<struct IndexTarget
*>* IndexTargets
,
872 indexRecords
* MetaIndexParser
) :
873 Item(Owner
), RealURI(URI
), SigFile(SigFile
), IndexTargets(IndexTargets
),
874 MetaIndexParser(MetaIndexParser
), AuthPass(false), IMSHit(false)
876 DestFile
= _config
->FindDir("Dir::State::lists") + "partial/";
877 DestFile
+= URItoFileName(URI
);
880 Desc
.Description
= URIDesc
;
882 Desc
.ShortDesc
= ShortDesc
;
889 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers /*{{{*/
890 // ---------------------------------------------------------------------
891 /* The only header we use is the last-modified header. */
892 string
pkgAcqMetaIndex::Custom600Headers()
894 string Final
= _config
->FindDir("Dir::State::lists");
895 Final
+= URItoFileName(RealURI
);
898 if (stat(Final
.c_str(),&Buf
) != 0)
899 return "\nIndex-File: true";
901 return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf
.st_mtime
);
904 void pkgAcqMetaIndex::Done(string Message
,unsigned long Size
,string Hash
,
905 pkgAcquire::MethodConfig
*Cfg
)
907 Item::Done(Message
,Size
,Hash
,Cfg
);
909 // MetaIndexes are done in two passes: one to download the
910 // metaindex with an appropriate method, and a second to verify it
911 // with the gpgv method
913 if (AuthPass
== true)
919 RetrievalDone(Message
);
921 // Still more retrieving to do
926 // There was no signature file, so we are finished. Download
927 // the indexes without verification.
932 // There was a signature file, so pass it to gpgv for
935 if (_config
->FindB("Debug::pkgAcquire::Auth", false))
936 std::cerr
<< "Metaindex acquired, queueing gpg verification ("
937 << SigFile
<< "," << DestFile
<< ")\n";
939 Desc
.URI
= "gpgv:" + SigFile
;
946 void pkgAcqMetaIndex::RetrievalDone(string Message
)
948 // We have just finished downloading a Release file (it is not
951 string FileName
= LookupTag(Message
,"Filename");
952 if (FileName
.empty() == true)
955 ErrorText
= "Method gave a blank filename";
959 if (FileName
!= DestFile
)
962 Desc
.URI
= "copy:" + FileName
;
967 // see if the download was a IMSHit
968 IMSHit
= StringToBool(LookupTag(Message
,"IMS-Hit"),false);
972 string FinalFile
= _config
->FindDir("Dir::State::lists");
973 FinalFile
+= URItoFileName(RealURI
);
975 // The files timestamp matches
976 if (StringToBool(LookupTag(Message
,"IMS-Hit"),false) == false)
978 // Move it into position
979 Rename(DestFile
,FinalFile
);
981 chmod(FinalFile
.c_str(),0644);
982 DestFile
= FinalFile
;
985 void pkgAcqMetaIndex::AuthDone(string Message
)
987 // At this point, the gpgv method has succeeded, so there is a
988 // valid signature from a key in the trusted keyring. We
989 // perform additional verification of its contents, and use them
990 // to verify the indexes we are about to download
992 if (!MetaIndexParser
->Load(DestFile
))
994 Status
= StatAuthError
;
995 ErrorText
= MetaIndexParser
->ErrorText
;
999 if (!VerifyVendor(Message
))
1004 if (_config
->FindB("Debug::pkgAcquire::Auth", false))
1005 std::cerr
<< "Signature verification succeeded: "
1006 << DestFile
<< std::endl
;
1008 // Download further indexes with verification
1011 // Done, move signature file into position
1013 string VerifiedSigFile
= _config
->FindDir("Dir::State::lists") +
1014 URItoFileName(RealURI
) + ".gpg";
1015 Rename(SigFile
,VerifiedSigFile
);
1016 chmod(VerifiedSigFile
.c_str(),0644);
1019 void pkgAcqMetaIndex::QueueIndexes(bool verify
)
1021 for (vector
<struct IndexTarget
*>::const_iterator Target
= IndexTargets
->begin();
1022 Target
!= IndexTargets
->end();
1025 HashString ExpectedIndexHash
;
1028 const indexRecords::checkSum
*Record
= MetaIndexParser
->Lookup((*Target
)->MetaKey
);
1031 Status
= StatAuthError
;
1032 ErrorText
= "Unable to find expected entry "
1033 + (*Target
)->MetaKey
+ " in Meta-index file (malformed Release file?)";
1036 ExpectedIndexHash
= Record
->Hash
;
1037 if (_config
->FindB("Debug::pkgAcquire::Auth", false))
1039 std::cerr
<< "Queueing: " << (*Target
)->URI
<< std::endl
;
1040 std::cerr
<< "Expected Hash: " << ExpectedIndexHash
.toStr() << std::endl
;
1042 if (ExpectedIndexHash
.empty())
1044 Status
= StatAuthError
;
1045 ErrorText
= "Unable to find hash sum for "
1046 + (*Target
)->MetaKey
+ " in Meta-index file";
1051 // Queue Packages file (either diff or full packages files, depending
1052 // on the users option)
1053 if(_config
->FindB("Acquire::PDiffs",true) == true)
1054 new pkgAcqDiffIndex(Owner
, (*Target
)->URI
, (*Target
)->Description
,
1055 (*Target
)->ShortDesc
, ExpectedIndexHash
);
1057 new pkgAcqIndex(Owner
, (*Target
)->URI
, (*Target
)->Description
,
1058 (*Target
)->ShortDesc
, ExpectedIndexHash
);
1062 bool pkgAcqMetaIndex::VerifyVendor(string Message
)
1064 // // Maybe this should be made available from above so we don't have
1065 // // to read and parse it every time?
1066 // pkgVendorList List;
1067 // List.ReadMainList();
1069 // const Vendor* Vndr = NULL;
1070 // for (std::vector<string>::const_iterator I = GPGVOutput.begin(); I != GPGVOutput.end(); I++)
1072 // string::size_type pos = (*I).find("VALIDSIG ");
1073 // if (_config->FindB("Debug::Vendor", false))
1074 // std::cerr << "Looking for VALIDSIG in \"" << (*I) << "\": pos " << pos
1076 // if (pos != std::string::npos)
1078 // string Fingerprint = (*I).substr(pos+sizeof("VALIDSIG"));
1079 // if (_config->FindB("Debug::Vendor", false))
1080 // std::cerr << "Looking for \"" << Fingerprint << "\" in vendor..." <<
1082 // Vndr = List.FindVendor(Fingerprint) != "";
1083 // if (Vndr != NULL);
1087 string::size_type pos
;
1089 // check for missing sigs (that where not fatal because otherwise we had
1092 string msg
= _("There is no public key available for the "
1093 "following key IDs:\n");
1094 pos
= Message
.find("NO_PUBKEY ");
1095 if (pos
!= std::string::npos
)
1097 string::size_type start
= pos
+strlen("NO_PUBKEY ");
1098 string Fingerprint
= Message
.substr(start
, Message
.find("\n")-start
);
1099 missingkeys
+= (Fingerprint
);
1101 if(!missingkeys
.empty())
1102 _error
->Warning("%s", string(msg
+missingkeys
).c_str());
1104 string Transformed
= MetaIndexParser
->GetExpectedDist();
1106 if (Transformed
== "../project/experimental")
1108 Transformed
= "experimental";
1111 pos
= Transformed
.rfind('/');
1112 if (pos
!= string::npos
)
1114 Transformed
= Transformed
.substr(0, pos
);
1117 if (Transformed
== ".")
1122 if (_config
->FindB("Debug::pkgAcquire::Auth", false))
1124 std::cerr
<< "Got Codename: " << MetaIndexParser
->GetDist() << std::endl
;
1125 std::cerr
<< "Expecting Dist: " << MetaIndexParser
->GetExpectedDist() << std::endl
;
1126 std::cerr
<< "Transformed Dist: " << Transformed
<< std::endl
;
1129 if (MetaIndexParser
->CheckDist(Transformed
) == false)
1131 // This might become fatal one day
1132 // Status = StatAuthError;
1133 // ErrorText = "Conflicting distribution; expected "
1134 // + MetaIndexParser->GetExpectedDist() + " but got "
1135 // + MetaIndexParser->GetDist();
1137 if (!Transformed
.empty())
1139 _error
->Warning("Conflicting distribution: %s (expected %s but got %s)",
1140 Desc
.Description
.c_str(),
1141 Transformed
.c_str(),
1142 MetaIndexParser
->GetDist().c_str());
1149 // pkgAcqMetaIndex::Failed - no Release file present or no signature
1150 // file present /*{{{*/
1151 // ---------------------------------------------------------------------
1153 void pkgAcqMetaIndex::Failed(string Message
,pkgAcquire::MethodConfig
*Cnf
)
1155 if (AuthPass
== true)
1157 // if we fail the authentication but got the file via a IMS-Hit
1158 // this means that the file wasn't downloaded and that it might be
1159 // just stale (server problem, proxy etc). we delete what we have
1160 // queue it again without i-m-s
1161 // alternatively we could just unlink the file and let the user try again
1167 unlink(DestFile
.c_str());
1169 DestFile
= _config
->FindDir("Dir::State::lists") + "partial/";
1170 DestFile
+= URItoFileName(RealURI
);
1176 // gpgv method failed
1177 _error
->Warning("GPG error: %s: %s",
1178 Desc
.Description
.c_str(),
1179 LookupTag(Message
,"Message").c_str());
1183 // No Release file was present, or verification failed, so fall
1184 // back to queueing Packages files without verification
1185 QueueIndexes(false);
1190 // AcqArchive::AcqArchive - Constructor /*{{{*/
1191 // ---------------------------------------------------------------------
1192 /* This just sets up the initial fetch environment and queues the first
1194 pkgAcqArchive::pkgAcqArchive(pkgAcquire
*Owner
,pkgSourceList
*Sources
,
1195 pkgRecords
*Recs
,pkgCache::VerIterator
const &Version
,
1196 string
&StoreFilename
) :
1197 Item(Owner
), Version(Version
), Sources(Sources
), Recs(Recs
),
1198 StoreFilename(StoreFilename
), Vf(Version
.FileList()),
1201 Retries
= _config
->FindI("Acquire::Retries",0);
1203 if (Version
.Arch() == 0)
1205 _error
->Error(_("I wasn't able to locate a file for the %s package. "
1206 "This might mean you need to manually fix this package. "
1207 "(due to missing arch)"),
1208 Version
.ParentPkg().Name());
1212 /* We need to find a filename to determine the extension. We make the
1213 assumption here that all the available sources for this version share
1214 the same extension.. */
1215 // Skip not source sources, they do not have file fields.
1216 for (; Vf
.end() == false; Vf
++)
1218 if ((Vf
.File()->Flags
& pkgCache::Flag::NotSource
) != 0)
1223 // Does not really matter here.. we are going to fail out below
1224 if (Vf
.end() != true)
1226 // If this fails to get a file name we will bomb out below.
1227 pkgRecords::Parser
&Parse
= Recs
->Lookup(Vf
);
1228 if (_error
->PendingError() == true)
1231 // Generate the final file name as: package_version_arch.foo
1232 StoreFilename
= QuoteString(Version
.ParentPkg().Name(),"_:") + '_' +
1233 QuoteString(Version
.VerStr(),"_:") + '_' +
1234 QuoteString(Version
.Arch(),"_:.") +
1235 "." + flExtension(Parse
.FileName());
1238 // check if we have one trusted source for the package. if so, switch
1239 // to "TrustedOnly" mode
1240 for (pkgCache::VerFileIterator i
= Version
.FileList(); i
.end() == false; i
++)
1242 pkgIndexFile
*Index
;
1243 if (Sources
->FindIndex(i
.File(),Index
) == false)
1245 if (_config
->FindB("Debug::pkgAcquire::Auth", false))
1247 std::cerr
<< "Checking index: " << Index
->Describe()
1248 << "(Trusted=" << Index
->IsTrusted() << ")\n";
1250 if (Index
->IsTrusted()) {
1256 // "allow-unauthenticated" restores apts old fetching behaviour
1257 // that means that e.g. unauthenticated file:// uris are higher
1258 // priority than authenticated http:// uris
1259 if (_config
->FindB("APT::Get::AllowUnauthenticated",false) == true)
1263 if (QueueNext() == false && _error
->PendingError() == false)
1264 _error
->Error(_("I wasn't able to locate file for the %s package. "
1265 "This might mean you need to manually fix this package."),
1266 Version
.ParentPkg().Name());
1269 // AcqArchive::QueueNext - Queue the next file source /*{{{*/
1270 // ---------------------------------------------------------------------
1271 /* This queues the next available file version for download. It checks if
1272 the archive is already available in the cache and stashs the MD5 for
1274 bool pkgAcqArchive::QueueNext()
1276 for (; Vf
.end() == false; Vf
++)
1278 // Ignore not source sources
1279 if ((Vf
.File()->Flags
& pkgCache::Flag::NotSource
) != 0)
1282 // Try to cross match against the source list
1283 pkgIndexFile
*Index
;
1284 if (Sources
->FindIndex(Vf
.File(),Index
) == false)
1287 // only try to get a trusted package from another source if that source
1289 if(Trusted
&& !Index
->IsTrusted())
1292 // Grab the text package record
1293 pkgRecords::Parser
&Parse
= Recs
->Lookup(Vf
);
1294 if (_error
->PendingError() == true)
1297 string PkgFile
= Parse
.FileName();
1298 if(Parse
.SHA256Hash() != "")
1299 hash
= HashString("SHA256", Parse
.SHA256Hash());
1300 else if (Parse
.SHA1Hash() != "")
1301 hash
= HashString("SHA1", Parse
.SHA1Hash());
1303 hash
= HashString("MD5Sum", Parse
.MD5Hash());
1304 if (PkgFile
.empty() == true)
1305 return _error
->Error(_("The package index files are corrupted. No Filename: "
1306 "field for package %s."),
1307 Version
.ParentPkg().Name());
1309 Desc
.URI
= Index
->ArchiveURI(PkgFile
);
1310 Desc
.Description
= Index
->ArchiveInfo(Version
);
1312 Desc
.ShortDesc
= Version
.ParentPkg().Name();
1314 // See if we already have the file. (Legacy filenames)
1315 FileSize
= Version
->Size
;
1316 string FinalFile
= _config
->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile
);
1318 if (stat(FinalFile
.c_str(),&Buf
) == 0)
1320 // Make sure the size matches
1321 if ((unsigned)Buf
.st_size
== Version
->Size
)
1326 StoreFilename
= DestFile
= FinalFile
;
1330 /* Hmm, we have a file and its size does not match, this means it is
1331 an old style mismatched arch */
1332 unlink(FinalFile
.c_str());
1335 // Check it again using the new style output filenames
1336 FinalFile
= _config
->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename
);
1337 if (stat(FinalFile
.c_str(),&Buf
) == 0)
1339 // Make sure the size matches
1340 if ((unsigned)Buf
.st_size
== Version
->Size
)
1345 StoreFilename
= DestFile
= FinalFile
;
1349 /* Hmm, we have a file and its size does not match, this shouldnt
1351 unlink(FinalFile
.c_str());
1354 DestFile
= _config
->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename
);
1356 // Check the destination file
1357 if (stat(DestFile
.c_str(),&Buf
) == 0)
1359 // Hmm, the partial file is too big, erase it
1360 if ((unsigned)Buf
.st_size
> Version
->Size
)
1361 unlink(DestFile
.c_str());
1363 PartialSize
= Buf
.st_size
;
1368 Desc
.URI
= Index
->ArchiveURI(PkgFile
);
1369 Desc
.Description
= Index
->ArchiveInfo(Version
);
1371 Desc
.ShortDesc
= Version
.ParentPkg().Name();
1380 // AcqArchive::Done - Finished fetching /*{{{*/
1381 // ---------------------------------------------------------------------
1383 void pkgAcqArchive::Done(string Message
,unsigned long Size
,string CalcHash
,
1384 pkgAcquire::MethodConfig
*Cfg
)
1386 Item::Done(Message
,Size
,CalcHash
,Cfg
);
1389 if (Size
!= Version
->Size
)
1392 ErrorText
= _("Size mismatch");
1397 if(!hash
.VerifyFile(DestFile
))
1400 ErrorText
= _("Hash Sum mismatch");
1401 if(FileExists(DestFile
))
1402 Rename(DestFile
,DestFile
+ ".FAILED");
1406 // Grab the output filename
1407 string FileName
= LookupTag(Message
,"Filename");
1408 if (FileName
.empty() == true)
1411 ErrorText
= "Method gave a blank filename";
1417 // Reference filename
1418 if (FileName
!= DestFile
)
1420 StoreFilename
= DestFile
= FileName
;
1425 // Done, move it into position
1426 string FinalFile
= _config
->FindDir("Dir::Cache::Archives");
1427 FinalFile
+= flNotDir(StoreFilename
);
1428 Rename(DestFile
,FinalFile
);
1430 StoreFilename
= DestFile
= FinalFile
;
1434 // AcqArchive::Failed - Failure handler /*{{{*/
1435 // ---------------------------------------------------------------------
1436 /* Here we try other sources */
1437 void pkgAcqArchive::Failed(string Message
,pkgAcquire::MethodConfig
*Cnf
)
1439 ErrorText
= LookupTag(Message
,"Message");
1441 /* We don't really want to retry on failed media swaps, this prevents
1442 that. An interesting observation is that permanent failures are not
1444 if (Cnf
->Removable
== true &&
1445 StringToBool(LookupTag(Message
,"Transient-Failure"),false) == true)
1447 // Vf = Version.FileList();
1448 while (Vf
.end() == false) Vf
++;
1449 StoreFilename
= string();
1450 Item::Failed(Message
,Cnf
);
1454 if (QueueNext() == false)
1456 // This is the retry counter
1458 Cnf
->LocalOnly
== false &&
1459 StringToBool(LookupTag(Message
,"Transient-Failure"),false) == true)
1462 Vf
= Version
.FileList();
1463 if (QueueNext() == true)
1467 StoreFilename
= string();
1468 Item::Failed(Message
,Cnf
);
1472 // AcqArchive::IsTrusted - Determine whether this archive comes from a
1473 // trusted source /*{{{*/
1474 // ---------------------------------------------------------------------
1475 bool pkgAcqArchive::IsTrusted()
1480 // AcqArchive::Finished - Fetching has finished, tidy up /*{{{*/
1481 // ---------------------------------------------------------------------
1483 void pkgAcqArchive::Finished()
1485 if (Status
== pkgAcquire::Item::StatDone
&&
1488 StoreFilename
= string();
1492 // AcqFile::pkgAcqFile - Constructor /*{{{*/
1493 // ---------------------------------------------------------------------
1494 /* The file is added to the queue */
1495 pkgAcqFile::pkgAcqFile(pkgAcquire
*Owner
,string URI
,string MD5
,
1496 unsigned long Size
,string Dsc
,string ShortDesc
,
1497 const string
&DestDir
, const string
&DestFilename
) :
1498 Item(Owner
), hash(MD5
)
1500 Retries
= _config
->FindI("Acquire::Retries",0);
1502 if(!DestFilename
.empty())
1503 DestFile
= DestFilename
;
1504 else if(!DestDir
.empty())
1505 DestFile
= DestDir
+ "/" + flNotDir(URI
);
1507 DestFile
= flNotDir(URI
);
1511 Desc
.Description
= Dsc
;
1514 // Set the short description to the archive component
1515 Desc
.ShortDesc
= ShortDesc
;
1517 // Get the transfer sizes
1520 if (stat(DestFile
.c_str(),&Buf
) == 0)
1522 // Hmm, the partial file is too big, erase it
1523 if ((unsigned)Buf
.st_size
> Size
)
1524 unlink(DestFile
.c_str());
1526 PartialSize
= Buf
.st_size
;
1532 // AcqFile::Done - Item downloaded OK /*{{{*/
1533 // ---------------------------------------------------------------------
1535 void pkgAcqFile::Done(string Message
,unsigned long Size
,string CalcHash
,
1536 pkgAcquire::MethodConfig
*Cnf
)
1538 Item::Done(Message
,Size
,CalcHash
,Cnf
);
1541 if(!hash
.VerifyFile(DestFile
))
1544 ErrorText
= "Hash Sum mismatch";
1545 Rename(DestFile
,DestFile
+ ".FAILED");
1549 string FileName
= LookupTag(Message
,"Filename");
1550 if (FileName
.empty() == true)
1553 ErrorText
= "Method gave a blank filename";
1559 // The files timestamp matches
1560 if (StringToBool(LookupTag(Message
,"IMS-Hit"),false) == true)
1563 // We have to copy it into place
1564 if (FileName
!= DestFile
)
1567 if (_config
->FindB("Acquire::Source-Symlinks",true) == false ||
1568 Cnf
->Removable
== true)
1570 Desc
.URI
= "copy:" + FileName
;
1575 // Erase the file if it is a symlink so we can overwrite it
1577 if (lstat(DestFile
.c_str(),&St
) == 0)
1579 if (S_ISLNK(St
.st_mode
) != 0)
1580 unlink(DestFile
.c_str());
1584 if (symlink(FileName
.c_str(),DestFile
.c_str()) != 0)
1586 ErrorText
= "Link to " + DestFile
+ " failure ";
1593 // AcqFile::Failed - Failure handler /*{{{*/
1594 // ---------------------------------------------------------------------
1595 /* Here we try other sources */
1596 void pkgAcqFile::Failed(string Message
,pkgAcquire::MethodConfig
*Cnf
)
1598 ErrorText
= LookupTag(Message
,"Message");
1600 // This is the retry counter
1602 Cnf
->LocalOnly
== false &&
1603 StringToBool(LookupTag(Message
,"Transient-Failure"),false) == true)
1610 Item::Failed(Message
,Cnf
);