1 // -*- mode: cpp; mode: fold -*-
3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
4 /* ######################################################################
6 Acquire Item - Item to acquire
8 Each item can download to exactly one file at a time. This means you
9 cannot create an item that fetches two uri's to two files at the same
10 time. The pkgAcqIndex class creates a second class upon instantiation
11 to fetch the other index files because of this.
13 ##################################################################### */
15 // Include Files /*{{{*/
16 #include <apt-pkg/acquire-item.h>
17 #include <apt-pkg/configuration.h>
18 #include <apt-pkg/sourcelist.h>
19 #include <apt-pkg/vendorlist.h>
20 #include <apt-pkg/error.h>
21 #include <apt-pkg/strutl.h>
22 #include <apt-pkg/fileutl.h>
23 #include <apt-pkg/md5.h>
24 #include <apt-pkg/sha1.h>
25 #include <apt-pkg/tagfile.h>
39 // Acquire::Item::Item - Constructor /*{{{*/
40 // ---------------------------------------------------------------------
42 pkgAcquire::Item::Item(pkgAcquire
*Owner
) : Owner(Owner
), FileSize(0),
43 PartialSize(0), Mode(0), ID(0), Complete(false),
44 Local(false), QueueCounter(0)
50 // Acquire::Item::~Item - Destructor /*{{{*/
51 // ---------------------------------------------------------------------
53 pkgAcquire::Item::~Item()
58 // Acquire::Item::Failed - Item failed to download /*{{{*/
59 // ---------------------------------------------------------------------
60 /* We return to an idle state if there are still other queues that could
62 void pkgAcquire::Item::Failed(string Message
,pkgAcquire::MethodConfig
*Cnf
)
65 ErrorText
= LookupTag(Message
,"Message");
66 if (QueueCounter
<= 1)
68 /* This indicates that the file is not available right now but might
69 be sometime later. If we do a retry cycle then this should be
71 if (Cnf
->LocalOnly
== true &&
72 StringToBool(LookupTag(Message
,"Transient-Failure"),false) == true)
84 // Acquire::Item::Start - Item has begun to download /*{{{*/
85 // ---------------------------------------------------------------------
86 /* Stash status and the file size. Note that setting Complete means
87 sub-phases of the acquire process such as decompresion are operating */
88 void pkgAcquire::Item::Start(string
/*Message*/,unsigned long Size
)
90 Status
= StatFetching
;
91 if (FileSize
== 0 && Complete
== false)
95 // Acquire::Item::Done - Item downloaded OK /*{{{*/
96 // ---------------------------------------------------------------------
98 void pkgAcquire::Item::Done(string Message
,unsigned long Size
,string
,
99 pkgAcquire::MethodConfig
*Cnf
)
101 // We just downloaded something..
102 string FileName
= LookupTag(Message
,"Filename");
103 // we only inform the Log class if it was actually not a local thing
104 if (Complete
== false && !Local
&& FileName
== DestFile
)
107 Owner
->Log
->Fetched(Size
,atoi(LookupTag(Message
,"Resume-Point","0").c_str()));
114 ErrorText
= string();
115 Owner
->Dequeue(this);
118 // Acquire::Item::Rename - Rename a file /*{{{*/
119 // ---------------------------------------------------------------------
120 /* This helper function is used by alot of item methods as thier final
122 void pkgAcquire::Item::Rename(string From
,string To
)
124 if (rename(From
.c_str(),To
.c_str()) != 0)
127 snprintf(S
,sizeof(S
),_("rename failed, %s (%s -> %s)."),strerror(errno
),
128 From
.c_str(),To
.c_str());
136 // AcqDiffIndex::AcqDiffIndex - Constructor
137 // ---------------------------------------------------------------------
138 /* Get the DiffIndex file first and see if there are patches availabe
139 * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
140 * patches. If anything goes wrong in that process, it will fall back to
141 * the original packages file
143 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire
*Owner
,
144 string URI
,string URIDesc
,string ShortDesc
,
146 : Item(Owner
), RealURI(URI
), ExpectedMD5(ExpectedMD5
), Description(URIDesc
)
149 Debug
= _config
->FindB("Debug::pkgAcquire::Diffs",false);
151 Desc
.Description
= URIDesc
+ "/DiffIndex";
153 Desc
.ShortDesc
= ShortDesc
;
154 Desc
.URI
= URI
+ ".diff/Index";
156 DestFile
= _config
->FindDir("Dir::State::lists") + "partial/";
157 DestFile
+= URItoFileName(URI
) + string(".DiffIndex");
160 std::clog
<< "pkgAcqDiffIndex: " << Desc
.URI
<< std::endl
;
162 // look for the current package file
163 CurrentPackagesFile
= _config
->FindDir("Dir::State::lists");
164 CurrentPackagesFile
+= URItoFileName(RealURI
);
166 // FIXME: this file:/ check is a hack to prevent fetching
167 // from local sources. this is really silly, and
168 // should be fixed cleanly as soon as possible
169 if(!FileExists(CurrentPackagesFile
) ||
170 Desc
.URI
.substr(0,strlen("file:/")) == "file:/")
172 // we don't have a pkg file or we don't want to queue
174 std::clog
<< "No index file, local or canceld by user" << std::endl
;
180 std::clog
<< "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): "
181 << CurrentPackagesFile
<< std::endl
;
187 // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/
188 // ---------------------------------------------------------------------
189 /* The only header we use is the last-modified header. */
190 string
pkgAcqDiffIndex::Custom600Headers()
192 string Final
= _config
->FindDir("Dir::State::lists");
193 Final
+= URItoFileName(RealURI
) + string(".IndexDiff");
196 std::clog
<< "Custom600Header-IMS: " << Final
<< std::endl
;
199 if (stat(Final
.c_str(),&Buf
) != 0)
200 return "\nIndex-File: true";
202 return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf
.st_mtime
);
206 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile
)
209 std::clog
<< "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile
214 vector
<DiffInfo
> available_patches
;
216 FileFd
Fd(IndexDiffFile
,FileFd::ReadOnly
);
218 if (_error
->PendingError() == true)
221 if(TF
.Step(Tags
) == true)
228 string tmp
= Tags
.FindS("SHA1-Current");
229 std::stringstream
ss(tmp
);
232 FileFd
fd(CurrentPackagesFile
, FileFd::ReadOnly
);
234 SHA1
.AddFD(fd
.Fd(), fd
.Size());
235 local_sha1
= string(SHA1
.Result());
237 if(local_sha1
== ServerSha1
)
239 // we have the same sha1 as the server
241 std::clog
<< "Package file is up-to-date" << std::endl
;
242 // set found to true, this will queue a pkgAcqIndexDiffs with
243 // a empty availabe_patches
249 std::clog
<< "SHA1-Current: " << ServerSha1
<< std::endl
;
251 // check the historie and see what patches we need
252 string history
= Tags
.FindS("SHA1-History");
253 std::stringstream
hist(history
);
254 while(hist
>> d
.sha1
>> size
>> d
.file
)
256 d
.size
= atoi(size
.c_str());
257 // read until the first match is found
258 if(d
.sha1
== local_sha1
)
260 // from that point on, we probably need all diffs
264 std::clog
<< "Need to get diff: " << d
.file
<< std::endl
;
265 available_patches
.push_back(d
);
270 // we have something, queue the next diff
274 unsigned int last_space
= Description
.rfind(" ");
275 if(last_space
!= string::npos
)
276 Description
.erase(last_space
, Description
.size()-last_space
);
277 new pkgAcqIndexDiffs(Owner
, RealURI
, Description
, Desc
.ShortDesc
,
278 ExpectedMD5
, available_patches
);
286 // Nothing found, report and return false
287 // Failing here is ok, if we return false later, the full
288 // IndexFile is queued
290 std::clog
<< "Can't find a patch in the index file" << std::endl
;
294 void pkgAcqDiffIndex::Failed(string Message
,pkgAcquire::MethodConfig
*Cnf
)
297 std::clog
<< "pkgAcqDiffIndex failed: " << Desc
.URI
<< std::endl
298 << "Falling back to normal index file aquire" << std::endl
;
300 new pkgAcqIndex(Owner
, RealURI
, Description
, Desc
.ShortDesc
,
308 void pkgAcqDiffIndex::Done(string Message
,unsigned long Size
,string Md5Hash
,
309 pkgAcquire::MethodConfig
*Cnf
)
312 std::clog
<< "pkgAcqDiffIndex::Done(): " << Desc
.URI
<< std::endl
;
314 Item::Done(Message
,Size
,Md5Hash
,Cnf
);
317 FinalFile
= _config
->FindDir("Dir::State::lists")+URItoFileName(RealURI
);
319 // sucess in downloading the index
321 FinalFile
+= string(".IndexDiff");
323 std::clog
<< "Renaming: " << DestFile
<< " -> " << FinalFile
325 Rename(DestFile
,FinalFile
);
326 chmod(FinalFile
.c_str(),0644);
327 DestFile
= FinalFile
;
329 if(!ParseDiffIndex(DestFile
))
330 return Failed("", NULL
);
340 // AcqIndexDiffs::AcqIndexDiffs - Constructor
341 // ---------------------------------------------------------------------
342 /* The package diff is added to the queue. one object is constructed
343 * for each diff and the index
345 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire
*Owner
,
346 string URI
,string URIDesc
,string ShortDesc
,
347 string ExpectedMD5
, vector
<DiffInfo
> diffs
)
348 : Item(Owner
), RealURI(URI
), ExpectedMD5(ExpectedMD5
),
349 available_patches(diffs
)
352 DestFile
= _config
->FindDir("Dir::State::lists") + "partial/";
353 DestFile
+= URItoFileName(URI
);
355 Debug
= _config
->FindB("Debug::pkgAcquire::Diffs",false);
357 Description
= URIDesc
;
359 Desc
.ShortDesc
= ShortDesc
;
361 if(available_patches
.size() == 0)
363 // we are done (yeah!)
369 State
= StateFetchDiff
;
375 void pkgAcqIndexDiffs::Failed(string Message
,pkgAcquire::MethodConfig
*Cnf
)
378 std::clog
<< "pkgAcqIndexDiffs failed: " << Desc
.URI
<< std::endl
379 << "Falling back to normal index file aquire" << std::endl
;
380 new pkgAcqIndex(Owner
, RealURI
, Description
,Desc
.ShortDesc
,
386 // helper that cleans the item out of the fetcher queue
387 void pkgAcqIndexDiffs::Finish(bool allDone
)
389 // we restore the original name, this is required, otherwise
390 // the file will be cleaned
393 DestFile
= _config
->FindDir("Dir::State::lists");
394 DestFile
+= URItoFileName(RealURI
);
396 // do the final md5sum checking
398 FileFd
Fd(DestFile
, FileFd::ReadOnly
);
399 sum
.AddFD(Fd
.Fd(), Fd
.Size());
401 string MD5
= (string
)sum
.Result();
403 if (!ExpectedMD5
.empty() && MD5
!= ExpectedMD5
)
405 Status
= StatAuthError
;
406 ErrorText
= _("MD5Sum mismatch");
407 Rename(DestFile
,DestFile
+ ".FAILED");
412 // this is for the "real" finish
417 std::clog
<< "\n\nallDone: " << DestFile
<< "\n" << std::endl
;
422 std::clog
<< "Finishing: " << Desc
.URI
<< std::endl
;
431 bool pkgAcqIndexDiffs::QueueNextDiff()
434 // calc sha1 of the just patched file
435 string FinalFile
= _config
->FindDir("Dir::State::lists");
436 FinalFile
+= URItoFileName(RealURI
);
438 FileFd
fd(FinalFile
, FileFd::ReadOnly
);
440 SHA1
.AddFD(fd
.Fd(), fd
.Size());
441 string local_sha1
= string(SHA1
.Result());
443 std::clog
<< "QueueNextDiff: "
444 << FinalFile
<< " (" << local_sha1
<< ")"<<std::endl
;
446 // remove all patches until the next matching patch is found
447 // this requires the Index file to be ordered
448 for(vector
<DiffInfo
>::iterator I
=available_patches
.begin();
449 available_patches
.size() > 0 &&
450 I
!= available_patches
.end() &&
451 (*I
).sha1
!= local_sha1
;
454 available_patches
.erase(I
);
457 // error checking and falling back if no patch was found
458 if(available_patches
.size() == 0)
464 // queue the right diff
465 Desc
.URI
= string(RealURI
) + ".diff/" + available_patches
[0].file
+ ".gz";
466 Desc
.Description
= Description
+ " " + available_patches
[0].file
+ string(".pdiff");
467 DestFile
= _config
->FindDir("Dir::State::lists") + "partial/";
468 DestFile
+= URItoFileName(RealURI
+ ".diff/" + available_patches
[0].file
);
471 std::clog
<< "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc
.URI
<< std::endl
;
480 void pkgAcqIndexDiffs::Done(string Message
,unsigned long Size
,string Md5Hash
,
481 pkgAcquire::MethodConfig
*Cnf
)
484 std::clog
<< "pkgAcqIndexDiffs::Done(): " << Desc
.URI
<< std::endl
;
486 Item::Done(Message
,Size
,Md5Hash
,Cnf
);
489 FinalFile
= _config
->FindDir("Dir::State::lists")+URItoFileName(RealURI
);
491 // sucess in downloading a diff, enter ApplyDiff state
492 if(State
== StateFetchDiff
)
496 std::clog
<< "Sending to gzip method: " << FinalFile
<< std::endl
;
498 string FileName
= LookupTag(Message
,"Filename");
499 State
= StateUnzipDiff
;
501 Desc
.URI
= "gzip:" + FileName
;
502 DestFile
+= ".decomp";
508 // sucess in downloading a diff, enter ApplyDiff state
509 if(State
== StateUnzipDiff
)
512 // rred excepts the patch as $FinalFile.ed
513 Rename(DestFile
,FinalFile
+".ed");
516 std::clog
<< "Sending to rred method: " << FinalFile
<< std::endl
;
518 State
= StateApplyDiff
;
520 Desc
.URI
= "rred:" + FinalFile
;
527 // success in download/apply a diff, queue next (if needed)
528 if(State
== StateApplyDiff
)
530 // remove the just applied patch
531 available_patches
.erase(available_patches
.begin());
536 std::clog
<< "Moving patched file in place: " << std::endl
537 << DestFile
<< " -> " << FinalFile
<< std::endl
;
539 Rename(DestFile
,FinalFile
);
540 chmod(FinalFile
.c_str(),0644);
542 // see if there is more to download
543 if(available_patches
.size() > 0) {
544 new pkgAcqIndexDiffs(Owner
, RealURI
, Description
, Desc
.ShortDesc
,
545 ExpectedMD5
, available_patches
);
553 // AcqIndex::AcqIndex - Constructor /*{{{*/
554 // ---------------------------------------------------------------------
555 /* The package file is added to the queue and a second class is
556 instantiated to fetch the revision file */
557 pkgAcqIndex::pkgAcqIndex(pkgAcquire
*Owner
,
558 string URI
,string URIDesc
,string ShortDesc
,
559 string ExpectedMD5
, string comprExt
)
560 : Item(Owner
), RealURI(URI
), ExpectedMD5(ExpectedMD5
)
562 Decompression
= false;
565 DestFile
= _config
->FindDir("Dir::State::lists") + "partial/";
566 DestFile
+= URItoFileName(URI
);
570 // autoselect the compression method
571 if(FileExists("/bin/bzip2"))
572 CompressionExtension
= ".bz2";
574 CompressionExtension
= ".gz";
576 CompressionExtension
= comprExt
;
578 Desc
.URI
= URI
+ CompressionExtension
;
580 Desc
.Description
= URIDesc
;
582 Desc
.ShortDesc
= ShortDesc
;
587 // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/
588 // ---------------------------------------------------------------------
589 /* The only header we use is the last-modified header. */
590 string
pkgAcqIndex::Custom600Headers()
592 string Final
= _config
->FindDir("Dir::State::lists");
593 Final
+= URItoFileName(RealURI
);
596 if (stat(Final
.c_str(),&Buf
) != 0)
597 return "\nIndex-File: true";
599 return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf
.st_mtime
);
603 void pkgAcqIndex::Failed(string Message
,pkgAcquire::MethodConfig
*Cnf
)
605 // no .bz2 found, retry with .gz
606 if(Desc
.URI
.substr(Desc
.URI
.size()-3) == "bz2") {
607 Desc
.URI
= Desc
.URI
.substr(0,Desc
.URI
.size()-3) + "gz";
609 // retry with a gzip one
610 new pkgAcqIndex(Owner
, RealURI
, Desc
.Description
,Desc
.ShortDesc
,
611 ExpectedMD5
, string(".gz"));
618 // on decompression failure, remove bad versions in partial/
619 if(Decompression
&& Erase
) {
620 string s
= _config
->FindDir("Dir::State::lists") + "partial/";
621 s
+= URItoFileName(RealURI
);
625 Item::Failed(Message
,Cnf
);
629 // AcqIndex::Done - Finished a fetch /*{{{*/
630 // ---------------------------------------------------------------------
631 /* This goes through a number of states.. On the initial fetch the
632 method could possibly return an alternate filename which points
633 to the uncompressed version of the file. If this is so the file
634 is copied into the partial directory. In all other cases the file
635 is decompressed with a gzip uri. */
636 void pkgAcqIndex::Done(string Message
,unsigned long Size
,string MD5
,
637 pkgAcquire::MethodConfig
*Cfg
)
639 Item::Done(Message
,Size
,MD5
,Cfg
);
641 if (Decompression
== true)
643 if (_config
->FindB("Debug::pkgAcquire::Auth", false))
645 std::cerr
<< std::endl
<< RealURI
<< ": Computed MD5: " << MD5
;
646 std::cerr
<< " Expected MD5: " << ExpectedMD5
<< std::endl
;
652 FileFd
Fd(DestFile
, FileFd::ReadOnly
);
653 sum
.AddFD(Fd
.Fd(), Fd
.Size());
655 MD5
= (string
)sum
.Result();
658 if (!ExpectedMD5
.empty() && MD5
!= ExpectedMD5
)
660 Status
= StatAuthError
;
661 ErrorText
= _("MD5Sum mismatch");
662 Rename(DestFile
,DestFile
+ ".FAILED");
665 // Done, move it into position
666 string FinalFile
= _config
->FindDir("Dir::State::lists");
667 FinalFile
+= URItoFileName(RealURI
);
668 Rename(DestFile
,FinalFile
);
669 chmod(FinalFile
.c_str(),0644);
671 /* We restore the original name to DestFile so that the clean operation
673 DestFile
= _config
->FindDir("Dir::State::lists") + "partial/";
674 DestFile
+= URItoFileName(RealURI
);
676 // Remove the compressed version.
678 unlink(DestFile
.c_str());
685 // Handle the unzipd case
686 string FileName
= LookupTag(Message
,"Alt-Filename");
687 if (FileName
.empty() == false)
689 // The files timestamp matches
690 if (StringToBool(LookupTag(Message
,"Alt-IMS-Hit"),false) == true)
692 unlink(FileName
.c_str());
695 Decompression
= true;
697 DestFile
+= ".decomp";
698 Desc
.URI
= "copy:" + FileName
;
704 FileName
= LookupTag(Message
,"Filename");
705 if (FileName
.empty() == true)
708 ErrorText
= "Method gave a blank filename";
711 // The files timestamp matches
712 if (StringToBool(LookupTag(Message
,"IMS-Hit"),false) == true)
714 unlink(FileName
.c_str());
718 if (FileName
== DestFile
)
723 string compExt
= Desc
.URI
.substr(Desc
.URI
.size()-3);
726 decompProg
= "bzip2";
727 else if(compExt
== ".gz")
730 _error
->Error("Unsupported extension: %s", compExt
.c_str());
734 Decompression
= true;
735 DestFile
+= ".decomp";
736 Desc
.URI
= string(decompProg
) + ":" + FileName
;
741 // AcqIndexTrans::pkgAcqIndexTrans - Constructor /*{{{*/
742 // ---------------------------------------------------------------------
743 /* The Translation file is added to the queue */
744 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire
*Owner
,
745 string URI
,string URIDesc
,string ShortDesc
) :
746 pkgAcqIndex(Owner
, URI
, URIDesc
, ShortDesc
, "", "")
751 // AcqIndexTrans::Failed - Silence failure messages for missing files /*{{{*/
752 // ---------------------------------------------------------------------
754 void pkgAcqIndexTrans::Failed(string Message
,pkgAcquire::MethodConfig
*Cnf
)
756 if (Cnf
->LocalOnly
== true ||
757 StringToBool(LookupTag(Message
,"Transient-Failure"),false) == false)
766 Item::Failed(Message
,Cnf
);
770 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire
*Owner
,
771 string URI
,string URIDesc
,string ShortDesc
,
772 string MetaIndexURI
, string MetaIndexURIDesc
,
773 string MetaIndexShortDesc
,
774 const vector
<IndexTarget
*>* IndexTargets
,
775 indexRecords
* MetaIndexParser
) :
776 Item(Owner
), RealURI(URI
), MetaIndexURI(MetaIndexURI
),
777 MetaIndexURIDesc(MetaIndexURIDesc
), MetaIndexShortDesc(MetaIndexShortDesc
),
778 MetaIndexParser(MetaIndexParser
), IndexTargets(IndexTargets
)
780 DestFile
= _config
->FindDir("Dir::State::lists") + "partial/";
781 DestFile
+= URItoFileName(URI
);
783 // remove any partial downloaded sig-file in partial/.
784 // it may confuse proxies and is too small to warrant a
785 // partial download anyway
786 unlink(DestFile
.c_str());
789 Desc
.Description
= URIDesc
;
791 Desc
.ShortDesc
= ShortDesc
;
795 string Final
= _config
->FindDir("Dir::State::lists");
796 Final
+= URItoFileName(RealURI
);
798 if (stat(Final
.c_str(),&Buf
) == 0)
800 // File was already in place. It needs to be re-verified
801 // because Release might have changed, so Move it into partial
802 Rename(Final
,DestFile
);
808 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers /*{{{*/
809 // ---------------------------------------------------------------------
810 /* The only header we use is the last-modified header. */
811 string
pkgAcqMetaSig::Custom600Headers()
814 if (stat(DestFile
.c_str(),&Buf
) != 0)
815 return "\nIndex-File: true";
817 return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf
.st_mtime
);
820 void pkgAcqMetaSig::Done(string Message
,unsigned long Size
,string MD5
,
821 pkgAcquire::MethodConfig
*Cfg
)
823 Item::Done(Message
,Size
,MD5
,Cfg
);
825 string FileName
= LookupTag(Message
,"Filename");
826 if (FileName
.empty() == true)
829 ErrorText
= "Method gave a blank filename";
833 if (FileName
!= DestFile
)
835 // We have to copy it into place
837 Desc
.URI
= "copy:" + FileName
;
844 // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
845 new pkgAcqMetaIndex(Owner
, MetaIndexURI
, MetaIndexURIDesc
, MetaIndexShortDesc
,
846 DestFile
, IndexTargets
, MetaIndexParser
);
850 void pkgAcqMetaSig::Failed(string Message
,pkgAcquire::MethodConfig
*Cnf
)
852 string Final
= _config
->FindDir("Dir::State::lists") + URItoFileName(RealURI
);
854 // if we get a network error we fail gracefully
855 if(Status
== StatTransientNetworkError
)
857 Item::Failed(Message
,Cnf
);
858 // move the sigfile back on transient network failures
859 if(FileExists(DestFile
))
860 Rename(DestFile
,Final
);
862 // set the status back to , Item::Failed likes to reset it
863 Status
= pkgAcquire::Item::StatTransientNetworkError
;
867 // Delete any existing sigfile when the acquire failed
868 unlink(Final
.c_str());
870 // queue a pkgAcqMetaIndex with no sigfile
871 new pkgAcqMetaIndex(Owner
, MetaIndexURI
, MetaIndexURIDesc
, MetaIndexShortDesc
,
872 "", IndexTargets
, MetaIndexParser
);
874 if (Cnf
->LocalOnly
== true ||
875 StringToBool(LookupTag(Message
,"Transient-Failure"),false) == false)
884 Item::Failed(Message
,Cnf
);
887 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire
*Owner
,
888 string URI
,string URIDesc
,string ShortDesc
,
890 const vector
<struct IndexTarget
*>* IndexTargets
,
891 indexRecords
* MetaIndexParser
) :
892 Item(Owner
), RealURI(URI
), SigFile(SigFile
), IndexTargets(IndexTargets
),
893 MetaIndexParser(MetaIndexParser
), AuthPass(false), IMSHit(false)
895 DestFile
= _config
->FindDir("Dir::State::lists") + "partial/";
896 DestFile
+= URItoFileName(URI
);
899 Desc
.Description
= URIDesc
;
901 Desc
.ShortDesc
= ShortDesc
;
908 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers /*{{{*/
909 // ---------------------------------------------------------------------
910 /* The only header we use is the last-modified header. */
911 string
pkgAcqMetaIndex::Custom600Headers()
913 string Final
= _config
->FindDir("Dir::State::lists");
914 Final
+= URItoFileName(RealURI
);
917 if (stat(Final
.c_str(),&Buf
) != 0)
918 return "\nIndex-File: true";
920 return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf
.st_mtime
);
923 void pkgAcqMetaIndex::Done(string Message
,unsigned long Size
,string MD5
,
924 pkgAcquire::MethodConfig
*Cfg
)
926 Item::Done(Message
,Size
,MD5
,Cfg
);
928 // MetaIndexes are done in two passes: one to download the
929 // metaindex with an appropriate method, and a second to verify it
930 // with the gpgv method
932 if (AuthPass
== true)
938 RetrievalDone(Message
);
940 // Still more retrieving to do
945 // There was no signature file, so we are finished. Download
946 // the indexes without verification.
951 // There was a signature file, so pass it to gpgv for
954 if (_config
->FindB("Debug::pkgAcquire::Auth", false))
955 std::cerr
<< "Metaindex acquired, queueing gpg verification ("
956 << SigFile
<< "," << DestFile
<< ")\n";
958 Desc
.URI
= "gpgv:" + SigFile
;
965 void pkgAcqMetaIndex::RetrievalDone(string Message
)
967 // We have just finished downloading a Release file (it is not
970 string FileName
= LookupTag(Message
,"Filename");
971 if (FileName
.empty() == true)
974 ErrorText
= "Method gave a blank filename";
978 if (FileName
!= DestFile
)
981 Desc
.URI
= "copy:" + FileName
;
986 // see if the download was a IMSHit
987 IMSHit
= StringToBool(LookupTag(Message
,"IMS-Hit"),false);
990 string FinalFile
= _config
->FindDir("Dir::State::lists");
991 FinalFile
+= URItoFileName(RealURI
);
993 // If we get a IMS hit we can remove the empty file in partial
994 // othersie we move the file in place
996 unlink(DestFile
.c_str());
998 Rename(DestFile
,FinalFile
);
1000 chmod(FinalFile
.c_str(),0644);
1001 DestFile
= FinalFile
;
1004 void pkgAcqMetaIndex::AuthDone(string Message
)
1006 // At this point, the gpgv method has succeeded, so there is a
1007 // valid signature from a key in the trusted keyring. We
1008 // perform additional verification of its contents, and use them
1009 // to verify the indexes we are about to download
1011 if (!MetaIndexParser
->Load(DestFile
))
1013 Status
= StatAuthError
;
1014 ErrorText
= MetaIndexParser
->ErrorText
;
1018 if (!VerifyVendor(Message
))
1023 if (_config
->FindB("Debug::pkgAcquire::Auth", false))
1024 std::cerr
<< "Signature verification succeeded: "
1025 << DestFile
<< std::endl
;
1027 // Download further indexes with verification
1030 // Done, move signature file into position
1032 string VerifiedSigFile
= _config
->FindDir("Dir::State::lists") +
1033 URItoFileName(RealURI
) + ".gpg";
1034 Rename(SigFile
,VerifiedSigFile
);
1035 chmod(VerifiedSigFile
.c_str(),0644);
1038 void pkgAcqMetaIndex::QueueIndexes(bool verify
)
1040 for (vector
<struct IndexTarget
*>::const_iterator Target
= IndexTargets
->begin();
1041 Target
!= IndexTargets
->end();
1044 string ExpectedIndexMD5
;
1047 const indexRecords::checkSum
*Record
= MetaIndexParser
->Lookup((*Target
)->MetaKey
);
1050 Status
= StatAuthError
;
1051 ErrorText
= "Unable to find expected entry "
1052 + (*Target
)->MetaKey
+ " in Meta-index file (malformed Release file?)";
1055 ExpectedIndexMD5
= Record
->MD5Hash
;
1056 if (_config
->FindB("Debug::pkgAcquire::Auth", false))
1058 std::cerr
<< "Queueing: " << (*Target
)->URI
<< std::endl
;
1059 std::cerr
<< "Expected MD5: " << ExpectedIndexMD5
<< std::endl
;
1061 if (ExpectedIndexMD5
.empty())
1063 Status
= StatAuthError
;
1064 ErrorText
= "Unable to find MD5 sum for "
1065 + (*Target
)->MetaKey
+ " in Meta-index file";
1070 // Queue Packages file (either diff or full packages files, depending
1071 // on the users option)
1072 if(_config
->FindB("Acquire::PDiffs",true) == true)
1073 new pkgAcqDiffIndex(Owner
, (*Target
)->URI
, (*Target
)->Description
,
1074 (*Target
)->ShortDesc
, ExpectedIndexMD5
);
1076 new pkgAcqIndex(Owner
, (*Target
)->URI
, (*Target
)->Description
,
1077 (*Target
)->ShortDesc
, ExpectedIndexMD5
);
1081 bool pkgAcqMetaIndex::VerifyVendor(string Message
)
1083 // // Maybe this should be made available from above so we don't have
1084 // // to read and parse it every time?
1085 // pkgVendorList List;
1086 // List.ReadMainList();
1088 // const Vendor* Vndr = NULL;
1089 // for (std::vector<string>::const_iterator I = GPGVOutput.begin(); I != GPGVOutput.end(); I++)
1091 // string::size_type pos = (*I).find("VALIDSIG ");
1092 // if (_config->FindB("Debug::Vendor", false))
1093 // std::cerr << "Looking for VALIDSIG in \"" << (*I) << "\": pos " << pos
1095 // if (pos != std::string::npos)
1097 // string Fingerprint = (*I).substr(pos+sizeof("VALIDSIG"));
1098 // if (_config->FindB("Debug::Vendor", false))
1099 // std::cerr << "Looking for \"" << Fingerprint << "\" in vendor..." <<
1101 // Vndr = List.FindVendor(Fingerprint) != "";
1102 // if (Vndr != NULL);
1106 string::size_type pos
;
1108 // check for missing sigs (that where not fatal because otherwise we had
1111 string msg
= _("There is no public key available for the "
1112 "following key IDs:\n");
1113 pos
= Message
.find("NO_PUBKEY ");
1114 if (pos
!= std::string::npos
)
1116 string::size_type start
= pos
+strlen("NO_PUBKEY ");
1117 string Fingerprint
= Message
.substr(start
, Message
.find("\n")-start
);
1118 missingkeys
+= (Fingerprint
);
1120 if(!missingkeys
.empty())
1121 _error
->Warning("%s", string(msg
+missingkeys
).c_str());
1123 string Transformed
= MetaIndexParser
->GetExpectedDist();
1125 if (Transformed
== "../project/experimental")
1127 Transformed
= "experimental";
1130 pos
= Transformed
.rfind('/');
1131 if (pos
!= string::npos
)
1133 Transformed
= Transformed
.substr(0, pos
);
1136 if (Transformed
== ".")
1141 if (_config
->FindB("Debug::pkgAcquire::Auth", false))
1143 std::cerr
<< "Got Codename: " << MetaIndexParser
->GetDist() << std::endl
;
1144 std::cerr
<< "Expecting Dist: " << MetaIndexParser
->GetExpectedDist() << std::endl
;
1145 std::cerr
<< "Transformed Dist: " << Transformed
<< std::endl
;
1148 if (MetaIndexParser
->CheckDist(Transformed
) == false)
1150 // This might become fatal one day
1151 // Status = StatAuthError;
1152 // ErrorText = "Conflicting distribution; expected "
1153 // + MetaIndexParser->GetExpectedDist() + " but got "
1154 // + MetaIndexParser->GetDist();
1156 if (!Transformed
.empty())
1158 _error
->Warning("Conflicting distribution: %s (expected %s but got %s)",
1159 Desc
.Description
.c_str(),
1160 Transformed
.c_str(),
1161 MetaIndexParser
->GetDist().c_str());
1168 // pkgAcqMetaIndex::Failed - no Release file present or no signature
1169 // file present /*{{{*/
1170 // ---------------------------------------------------------------------
1172 void pkgAcqMetaIndex::Failed(string Message
,pkgAcquire::MethodConfig
*Cnf
)
1174 if (AuthPass
== true)
1176 // if we fail the authentication but got the file via a IMS-Hit
1177 // this means that the file wasn't downloaded and that it might be
1178 // just stale (server problem, proxy etc). we delete what we have
1179 // queue it again without i-m-s
1180 // alternatively we could just unlink the file and let the user try again
1186 unlink(DestFile
.c_str());
1188 DestFile
= _config
->FindDir("Dir::State::lists") + "partial/";
1189 DestFile
+= URItoFileName(RealURI
);
1195 // gpgv method failed
1196 _error
->Warning("GPG error: %s: %s",
1197 Desc
.Description
.c_str(),
1198 LookupTag(Message
,"Message").c_str());
1202 // No Release file was present, or verification failed, so fall
1203 // back to queueing Packages files without verification
1204 QueueIndexes(false);
1209 // AcqArchive::AcqArchive - Constructor /*{{{*/
1210 // ---------------------------------------------------------------------
1211 /* This just sets up the initial fetch environment and queues the first
1213 pkgAcqArchive::pkgAcqArchive(pkgAcquire
*Owner
,pkgSourceList
*Sources
,
1214 pkgRecords
*Recs
,pkgCache::VerIterator
const &Version
,
1215 string
&StoreFilename
) :
1216 Item(Owner
), Version(Version
), Sources(Sources
), Recs(Recs
),
1217 StoreFilename(StoreFilename
), Vf(Version
.FileList()),
1220 Retries
= _config
->FindI("Acquire::Retries",0);
1222 if (Version
.Arch() == 0)
1224 _error
->Error(_("I wasn't able to locate a file for the %s package. "
1225 "This might mean you need to manually fix this package. "
1226 "(due to missing arch)"),
1227 Version
.ParentPkg().Name());
1231 /* We need to find a filename to determine the extension. We make the
1232 assumption here that all the available sources for this version share
1233 the same extension.. */
1234 // Skip not source sources, they do not have file fields.
1235 for (; Vf
.end() == false; Vf
++)
1237 if ((Vf
.File()->Flags
& pkgCache::Flag::NotSource
) != 0)
1242 // Does not really matter here.. we are going to fail out below
1243 if (Vf
.end() != true)
1245 // If this fails to get a file name we will bomb out below.
1246 pkgRecords::Parser
&Parse
= Recs
->Lookup(Vf
);
1247 if (_error
->PendingError() == true)
1250 // Generate the final file name as: package_version_arch.foo
1251 StoreFilename
= QuoteString(Version
.ParentPkg().Name(),"_:") + '_' +
1252 QuoteString(Version
.VerStr(),"_:") + '_' +
1253 QuoteString(Version
.Arch(),"_:.") +
1254 "." + flExtension(Parse
.FileName());
1257 // check if we have one trusted source for the package. if so, switch
1258 // to "TrustedOnly" mode
1259 for (pkgCache::VerFileIterator i
= Version
.FileList(); i
.end() == false; i
++)
1261 pkgIndexFile
*Index
;
1262 if (Sources
->FindIndex(i
.File(),Index
) == false)
1264 if (_config
->FindB("Debug::pkgAcquire::Auth", false))
1266 std::cerr
<< "Checking index: " << Index
->Describe()
1267 << "(Trusted=" << Index
->IsTrusted() << ")\n";
1269 if (Index
->IsTrusted()) {
1275 // "allow-unauthenticated" restores apts old fetching behaviour
1276 // that means that e.g. unauthenticated file:// uris are higher
1277 // priority than authenticated http:// uris
1278 if (_config
->FindB("APT::Get::AllowUnauthenticated",false) == true)
1282 if (QueueNext() == false && _error
->PendingError() == false)
1283 _error
->Error(_("I wasn't able to locate file for the %s package. "
1284 "This might mean you need to manually fix this package."),
1285 Version
.ParentPkg().Name());
1288 // AcqArchive::QueueNext - Queue the next file source /*{{{*/
1289 // ---------------------------------------------------------------------
1290 /* This queues the next available file version for download. It checks if
1291 the archive is already available in the cache and stashs the MD5 for
1293 bool pkgAcqArchive::QueueNext()
1295 for (; Vf
.end() == false; Vf
++)
1297 // Ignore not source sources
1298 if ((Vf
.File()->Flags
& pkgCache::Flag::NotSource
) != 0)
1301 // Try to cross match against the source list
1302 pkgIndexFile
*Index
;
1303 if (Sources
->FindIndex(Vf
.File(),Index
) == false)
1306 // only try to get a trusted package from another source if that source
1308 if(Trusted
&& !Index
->IsTrusted())
1311 // Grab the text package record
1312 pkgRecords::Parser
&Parse
= Recs
->Lookup(Vf
);
1313 if (_error
->PendingError() == true)
1316 string PkgFile
= Parse
.FileName();
1317 MD5
= Parse
.MD5Hash();
1318 if (PkgFile
.empty() == true)
1319 return _error
->Error(_("The package index files are corrupted. No Filename: "
1320 "field for package %s."),
1321 Version
.ParentPkg().Name());
1323 Desc
.URI
= Index
->ArchiveURI(PkgFile
);
1324 Desc
.Description
= Index
->ArchiveInfo(Version
);
1326 Desc
.ShortDesc
= Version
.ParentPkg().Name();
1328 // See if we already have the file. (Legacy filenames)
1329 FileSize
= Version
->Size
;
1330 string FinalFile
= _config
->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile
);
1332 if (stat(FinalFile
.c_str(),&Buf
) == 0)
1334 // Make sure the size matches
1335 if ((unsigned)Buf
.st_size
== Version
->Size
)
1340 StoreFilename
= DestFile
= FinalFile
;
1344 /* Hmm, we have a file and its size does not match, this means it is
1345 an old style mismatched arch */
1346 unlink(FinalFile
.c_str());
1349 // Check it again using the new style output filenames
1350 FinalFile
= _config
->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename
);
1351 if (stat(FinalFile
.c_str(),&Buf
) == 0)
1353 // Make sure the size matches
1354 if ((unsigned)Buf
.st_size
== Version
->Size
)
1359 StoreFilename
= DestFile
= FinalFile
;
1363 /* Hmm, we have a file and its size does not match, this shouldnt
1365 unlink(FinalFile
.c_str());
1368 DestFile
= _config
->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename
);
1370 // Check the destination file
1371 if (stat(DestFile
.c_str(),&Buf
) == 0)
1373 // Hmm, the partial file is too big, erase it
1374 if ((unsigned)Buf
.st_size
> Version
->Size
)
1375 unlink(DestFile
.c_str());
1377 PartialSize
= Buf
.st_size
;
1382 Desc
.URI
= Index
->ArchiveURI(PkgFile
);
1383 Desc
.Description
= Index
->ArchiveInfo(Version
);
1385 Desc
.ShortDesc
= Version
.ParentPkg().Name();
1394 // AcqArchive::Done - Finished fetching /*{{{*/
1395 // ---------------------------------------------------------------------
1397 void pkgAcqArchive::Done(string Message
,unsigned long Size
,string Md5Hash
,
1398 pkgAcquire::MethodConfig
*Cfg
)
1400 Item::Done(Message
,Size
,Md5Hash
,Cfg
);
1403 if (Size
!= Version
->Size
)
1406 ErrorText
= _("Size mismatch");
1411 if (Md5Hash
.empty() == false && MD5
.empty() == false)
1416 ErrorText
= _("MD5Sum mismatch");
1417 if(FileExists(DestFile
))
1418 Rename(DestFile
,DestFile
+ ".FAILED");
1423 // Grab the output filename
1424 string FileName
= LookupTag(Message
,"Filename");
1425 if (FileName
.empty() == true)
1428 ErrorText
= "Method gave a blank filename";
1434 // Reference filename
1435 if (FileName
!= DestFile
)
1437 StoreFilename
= DestFile
= FileName
;
1442 // Done, move it into position
1443 string FinalFile
= _config
->FindDir("Dir::Cache::Archives");
1444 FinalFile
+= flNotDir(StoreFilename
);
1445 Rename(DestFile
,FinalFile
);
1447 StoreFilename
= DestFile
= FinalFile
;
1451 // AcqArchive::Failed - Failure handler /*{{{*/
1452 // ---------------------------------------------------------------------
1453 /* Here we try other sources */
1454 void pkgAcqArchive::Failed(string Message
,pkgAcquire::MethodConfig
*Cnf
)
1456 ErrorText
= LookupTag(Message
,"Message");
1458 /* We don't really want to retry on failed media swaps, this prevents
1459 that. An interesting observation is that permanent failures are not
1461 if (Cnf
->Removable
== true &&
1462 StringToBool(LookupTag(Message
,"Transient-Failure"),false) == true)
1464 // Vf = Version.FileList();
1465 while (Vf
.end() == false) Vf
++;
1466 StoreFilename
= string();
1467 Item::Failed(Message
,Cnf
);
1471 if (QueueNext() == false)
1473 // This is the retry counter
1475 Cnf
->LocalOnly
== false &&
1476 StringToBool(LookupTag(Message
,"Transient-Failure"),false) == true)
1479 Vf
= Version
.FileList();
1480 if (QueueNext() == true)
1484 StoreFilename
= string();
1485 Item::Failed(Message
,Cnf
);
1489 // AcqArchive::IsTrusted - Determine whether this archive comes from a
1490 // trusted source /*{{{*/
1491 // ---------------------------------------------------------------------
1492 bool pkgAcqArchive::IsTrusted()
1497 // AcqArchive::Finished - Fetching has finished, tidy up /*{{{*/
1498 // ---------------------------------------------------------------------
1500 void pkgAcqArchive::Finished()
1502 if (Status
== pkgAcquire::Item::StatDone
&&
1505 StoreFilename
= string();
1509 // AcqFile::pkgAcqFile - Constructor /*{{{*/
1510 // ---------------------------------------------------------------------
1511 /* The file is added to the queue */
1512 pkgAcqFile::pkgAcqFile(pkgAcquire
*Owner
,string URI
,string MD5
,
1513 unsigned long Size
,string Dsc
,string ShortDesc
,
1514 const string
&DestDir
, const string
&DestFilename
) :
1515 Item(Owner
), Md5Hash(MD5
)
1517 Retries
= _config
->FindI("Acquire::Retries",0);
1519 if(!DestFilename
.empty())
1520 DestFile
= DestFilename
;
1521 else if(!DestDir
.empty())
1522 DestFile
= DestDir
+ "/" + flNotDir(URI
);
1524 DestFile
= flNotDir(URI
);
1528 Desc
.Description
= Dsc
;
1531 // Set the short description to the archive component
1532 Desc
.ShortDesc
= ShortDesc
;
1534 // Get the transfer sizes
1537 if (stat(DestFile
.c_str(),&Buf
) == 0)
1539 // Hmm, the partial file is too big, erase it
1540 if ((unsigned)Buf
.st_size
> Size
)
1541 unlink(DestFile
.c_str());
1543 PartialSize
= Buf
.st_size
;
1549 // AcqFile::Done - Item downloaded OK /*{{{*/
1550 // ---------------------------------------------------------------------
1552 void pkgAcqFile::Done(string Message
,unsigned long Size
,string MD5
,
1553 pkgAcquire::MethodConfig
*Cnf
)
1556 if (Md5Hash
.empty() == false && MD5
.empty() == false)
1561 ErrorText
= "MD5Sum mismatch";
1562 Rename(DestFile
,DestFile
+ ".FAILED");
1567 Item::Done(Message
,Size
,MD5
,Cnf
);
1569 string FileName
= LookupTag(Message
,"Filename");
1570 if (FileName
.empty() == true)
1573 ErrorText
= "Method gave a blank filename";
1579 // The files timestamp matches
1580 if (StringToBool(LookupTag(Message
,"IMS-Hit"),false) == true)
1583 // We have to copy it into place
1584 if (FileName
!= DestFile
)
1587 if (_config
->FindB("Acquire::Source-Symlinks",true) == false ||
1588 Cnf
->Removable
== true)
1590 Desc
.URI
= "copy:" + FileName
;
1595 // Erase the file if it is a symlink so we can overwrite it
1597 if (lstat(DestFile
.c_str(),&St
) == 0)
1599 if (S_ISLNK(St
.st_mode
) != 0)
1600 unlink(DestFile
.c_str());
1604 if (symlink(FileName
.c_str(),DestFile
.c_str()) != 0)
1606 ErrorText
= "Link to " + DestFile
+ " failure ";
1613 // AcqFile::Failed - Failure handler /*{{{*/
1614 // ---------------------------------------------------------------------
1615 /* Here we try other sources */
1616 void pkgAcqFile::Failed(string Message
,pkgAcquire::MethodConfig
*Cnf
)
1618 ErrorText
= LookupTag(Message
,"Message");
1620 // This is the retry counter
1622 Cnf
->LocalOnly
== false &&
1623 StringToBool(LookupTag(Message
,"Transient-Failure"),false) == true)
1630 Item::Failed(Message
,Cnf
);