From: Michael Vogt <egon@bottom>
Date: Sun, 2 Apr 2006 11:12:05 +0000 (+0200)
Subject: * merged the pdiff stuff
X-Git-Tag: 0.7.21~334
X-Git-Url: https://git.saurik.com/apt.git/commitdiff_plain/376091144df4991f161f068f2c297b80e94d36db?hp=-c

* merged the pdiff stuff
---

376091144df4991f161f068f2c297b80e94d36db
diff --combined apt-pkg/acquire-item.cc
index 1fa929aad,a29063e1a..ef0c4c1a8
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@@ -24,6 -24,8 +24,8 @@@
  #include <apt-pkg/strutl.h>
  #include <apt-pkg/fileutl.h>
  #include <apt-pkg/md5.h>
+ #include <apt-pkg/sha1.h>
+ #include <apt-pkg/tagfile.h>
  
  #include <apti18n.h>
      
@@@ -31,6 -33,7 +33,7 @@@
  #include <unistd.h>
  #include <errno.h>
  #include <string>
+ #include <sstream>
  #include <stdio.h>
  									/*}}}*/
  
@@@ -100,7 -103,8 +103,8 @@@ void pkgAcquire::Item::Done(string Mess
  {
     // We just downloaded something..
     string FileName = LookupTag(Message,"Filename");
-    if (Complete == false && FileName == DestFile)
+    // we only inform the Log class if it was actually not a local thing
+    if (Complete == false && !Local && FileName == DestFile)
     {
        if (Owner->Log != 0)
  	 Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
@@@ -131,14 -135,429 +135,429 @@@ void pkgAcquire::Item::Rename(string Fr
  }
  									/*}}}*/
  
+ 
+ // AcqDiffIndex::AcqDiffIndex - Constructor			
+ // ---------------------------------------------------------------------
+ /* Get the DiffIndex file first and see if there are patches availabe 
+  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
+  * patches. If anything goes wrong in that process, it will fall back to
+  * the original packages file
+  */
+ pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
+ 				 string URI,string URIDesc,string ShortDesc,
+ 				 string ExpectedMD5)
+    : Item(Owner), RealURI(URI), ExpectedMD5(ExpectedMD5), Description(URIDesc)
+ {
+    
+    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
+ 
+    Desc.Description = URIDesc + "/DiffIndex";
+    Desc.Owner = this;
+    Desc.ShortDesc = ShortDesc;
+    Desc.URI = URI + ".diff/Index";
+ 
+    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
+    DestFile += URItoFileName(URI) + string(".DiffIndex");
+ 
+    if(Debug)
+       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
+ 
+    // look for the current package file
+    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
+    CurrentPackagesFile += URItoFileName(RealURI);
+ 
+    // FIXME: this file:/ check is a hack to prevent fetching
+    //        from local sources. this is really silly, and
+    //        should be fixed cleanly as soon as possible
+    if(!FileExists(CurrentPackagesFile) || 
+       Desc.URI.substr(0,strlen("file:/")) == "file:/")
+    {
+       // we don't have a pkg file or we don't want to queue
+       if(Debug)
+ 	 std::clog << "No index file, local or canceld by user" << std::endl;
+       Failed("", NULL);
+       return;
+    }
+ 
+    if(Debug) 
+       std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): " 
+ 		<< CurrentPackagesFile << std::endl;
+    
+    QueueURI(Desc);
+ 
+ }
+ 
+ // AcqIndex::Custom600Headers - Insert custom request headers		/*{{{*/
+ // ---------------------------------------------------------------------
+ /* The only header we use is the last-modified header. */
+ string pkgAcqDiffIndex::Custom600Headers()
+ {
+    string Final = _config->FindDir("Dir::State::lists");
+    Final += URItoFileName(RealURI) + string(".IndexDiff");
+    
+    if(Debug)
+       std::clog << "Custom600Header-IMS: " << Final << std::endl;
+ 
+    struct stat Buf;
+    if (stat(Final.c_str(),&Buf) != 0)
+       return "\nIndex-File: true";
+    
+    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
+ }
+ 
+ 
+ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)
+ {
+    if(Debug)
+       std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile 
+ 		<< std::endl;
+ 
+    pkgTagSection Tags;
+    string ServerSha1;
+    vector<DiffInfo> available_patches;
+    
+    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
+    pkgTagFile TF(&Fd);
+    if (_error->PendingError() == true)
+       return false;
+ 
+    if(TF.Step(Tags) == true)
+    {
+       string local_sha1;
+       bool found = false;
+       DiffInfo d;
+       string size;
+ 
+       string tmp = Tags.FindS("SHA1-Current");
+       std::stringstream ss(tmp);
+       ss >> ServerSha1;
+ 
+       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
+       SHA1Summation SHA1;
+       SHA1.AddFD(fd.Fd(), fd.Size());
+       local_sha1 = string(SHA1.Result());
+ 
+       if(local_sha1 == ServerSha1) 
+       {
+ 	 // we have the same sha1 as the server
+ 	 if(Debug)
+ 	    std::clog << "Package file is up-to-date" << std::endl;
+ 	 // set found to true, this will queue a pkgAcqIndexDiffs with
+ 	 // a empty availabe_patches
+ 	 found = true;
+       } 
+       else 
+       {
+ 	 if(Debug)
+ 	    std::clog << "SHA1-Current: " << ServerSha1 << std::endl;
+ 
+ 	 // check the historie and see what patches we need
+ 	 string history = Tags.FindS("SHA1-History");     
+ 	 std::stringstream hist(history);
+ 	 while(hist >> d.sha1 >> size >> d.file) 
+ 	 {
+ 	    d.size = atoi(size.c_str());
+ 	    // read until the first match is found
+ 	    if(d.sha1 == local_sha1) 
+ 	       found=true;
+ 	    // from that point on, we probably need all diffs
+ 	    if(found) 
+ 	    {
+ 	       if(Debug)
+ 		  std::clog << "Need to get diff: " << d.file << std::endl;
+ 	       available_patches.push_back(d);
+ 	    }
+ 	 }
+       }
+ 
+       // no information how to get the patches, bail out
+       if(!found) 
+       {
+ 	 if(Debug)
+ 	    std::clog << "Can't find a patch in the index file" << std::endl;
+ 	 // Failed will queue a big package file
+ 	 Failed("", NULL);
+       } 
+       else 
+       {
+ 	 // queue the diffs
+ 	 new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
+ 			      ExpectedMD5, available_patches);
+ 	 Complete = false;
+ 	 Status = StatDone;
+ 	 Dequeue();
+ 	 return true;
+       }
+    }
+ 
+    return false;
+ }
+ 
+ void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
+ {
+    if(Debug)
+       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << std::endl
+ 		<< "Falling back to normal index file aquire" << std::endl;
+ 
+    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc, 
+ 		   ExpectedMD5);
+ 
+    Complete = false;
+    Status = StatDone;
+    Dequeue();
+ }
+ 
+ void pkgAcqDiffIndex::Done(string Message,unsigned long Size,string Md5Hash,
+ 			   pkgAcquire::MethodConfig *Cnf)
+ {
+    if(Debug)
+       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
+ 
+    Item::Done(Message,Size,Md5Hash,Cnf);
+ 
+    string FinalFile;
+    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
+ 
+    // sucess in downloading the index
+    // rename the index
+    FinalFile += string(".IndexDiff");
+    if(Debug)
+       std::clog << "Renaming: " << DestFile << " -> " << FinalFile 
+ 		<< std::endl;
+    Rename(DestFile,FinalFile);
+    chmod(FinalFile.c_str(),0644);
+    DestFile = FinalFile;
+ 
+    if(!ParseDiffIndex(DestFile))
+       return Failed("", NULL);
+ 
+    Complete = true;
+    Status = StatDone;
+    Dequeue();
+    return;
+ }
+ 
+ 
+ 
+ // AcqIndexDiffs::AcqIndexDiffs - Constructor			
+ // ---------------------------------------------------------------------
+ /* The package diff is added to the queue. one object is constructed
+  * for each diff and the index
+  */
+ pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
+ 				   string URI,string URIDesc,string ShortDesc,
+ 				   string ExpectedMD5, vector<DiffInfo> diffs)
+    : Item(Owner), RealURI(URI), ExpectedMD5(ExpectedMD5), 
+      available_patches(diffs)
+ {
+    
+    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
+    DestFile += URItoFileName(URI);
+ 
+    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
+ 
+    Desc.Description = URIDesc;
+    Desc.Owner = this;
+    Desc.ShortDesc = ShortDesc;
+ 
+    if(available_patches.size() == 0) 
+    {
+       // we are done (yeah!)
+       Finish(true);
+    }
+    else
+    {
+       // get the next diff
+       State = StateFetchDiff;
+       QueueNextDiff();
+    }
+ }
+ 
+ 
+ void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
+ {
+    if(Debug)
+       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl
+ 		<< "Falling back to normal index file aquire" << std::endl;
+    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc, 
+ 		   ExpectedMD5);
+    Finish();
+ }
+ 
+ 
+ // helper that cleans the item out of the fetcher queue
+ void pkgAcqIndexDiffs::Finish(bool allDone)
+ {
+    // we restore the original name, this is required, otherwise
+    // the file will be cleaned
+    if(allDone) 
+    {
+       DestFile = _config->FindDir("Dir::State::lists");
+       DestFile += URItoFileName(RealURI);
+ 
+       // do the final md5sum checking
+       MD5Summation sum;
+       FileFd Fd(DestFile, FileFd::ReadOnly);
+       sum.AddFD(Fd.Fd(), Fd.Size());
+       Fd.Close();
+       string MD5 = (string)sum.Result();
+ 
+       if (!ExpectedMD5.empty() && MD5 != ExpectedMD5)
+       {
+ 	 Status = StatAuthError;
+ 	 ErrorText = _("MD5Sum mismatch");
+ 	 Rename(DestFile,DestFile + ".FAILED");
+ 	 Dequeue();
+ 	 return;
+       }
+ 
+       // this is for the "real" finish
+       Complete = true;
+       Status = StatDone;
+       Dequeue();
+       if(Debug)
+ 	 std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
+       return;
+    }
+ 
+    if(Debug)
+       std::clog << "Finishing: " << Desc.URI << std::endl;
+    Complete = false;
+    Status = StatDone;
+    Dequeue();
+    return;
+ }
+ 
+ 
+ 
+ bool pkgAcqIndexDiffs::QueueNextDiff()
+ {
+ 
+    // calc sha1 of the just patched file
+    string FinalFile = _config->FindDir("Dir::State::lists");
+    FinalFile += URItoFileName(RealURI);
+ 
+    FileFd fd(FinalFile, FileFd::ReadOnly);
+    SHA1Summation SHA1;
+    SHA1.AddFD(fd.Fd(), fd.Size());
+    string local_sha1 = string(SHA1.Result());
+    if(Debug)
+       std::clog << "QueueNextDiff: " 
+ 		<< FinalFile << " (" << local_sha1 << ")"<<std::endl;
+ 
+    // remove all patches until the next matching patch is found
+    // this requires the Index file to be ordered
+    for(vector<DiffInfo>::iterator I=available_patches.begin();
+        available_patches.size() > 0 && 
+ 	  I != available_patches.end() &&
+ 	  (*I).sha1 != local_sha1; 
+        I++) 
+    {
+       available_patches.erase(I);
+    }
+ 
+    // error checking and falling back if no patch was found
+    if(available_patches.size() == 0) 
+    { 
+       Failed("", NULL);
+       return false;
+    }
+ 
+    // queue the right diff
+    Desc.URI = string(RealURI) + ".diff/" + available_patches[0].file + ".gz";
+    Desc.Description = available_patches[0].file + string(".pdiff");
+ 
+    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
+    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
+ 
+    if(Debug)
+       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
+    
+    QueueURI(Desc);
+ 
+    return true;
+ }
+ 
+ 
+ 
+ void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash,
+ 			    pkgAcquire::MethodConfig *Cnf)
+ {
+    if(Debug)
+       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
+ 
+    Item::Done(Message,Size,Md5Hash,Cnf);
+ 
+    string FinalFile;
+    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
+ 
+    // sucess in downloading a diff, enter ApplyDiff state
+    if(State == StateFetchDiff) 
+    {
+ 
+       if(Debug)
+ 	 std::clog << "Sending to gzip method: " << FinalFile << std::endl;
+ 
+       string FileName = LookupTag(Message,"Filename");
+       State = StateUnzipDiff;
+       Desc.URI = "gzip:" + FileName;
+       DestFile += ".decomp";
+       QueueURI(Desc);
+       Mode = "gzip";
+       return;
+    } 
+ 
+    // sucess in downloading a diff, enter ApplyDiff state
+    if(State == StateUnzipDiff) 
+    {
+ 
+       // rred excepts the patch as $FinalFile.ed
+       Rename(DestFile,FinalFile+".ed");
+ 
+       if(Debug)
+ 	 std::clog << "Sending to rred method: " << FinalFile << std::endl;
+ 
+       State = StateApplyDiff;
+       Desc.URI = "rred:" + FinalFile;
+       QueueURI(Desc);
+       Mode = "rred";
+       return;
+    } 
+ 
+ 
+    // success in download/apply a diff, queue next (if needed)
+    if(State == StateApplyDiff)
+    {
+       // remove the just applied patch
+       available_patches.erase(available_patches.begin());
+ 
+       // move into place
+       if(Debug) 
+       {
+ 	 std::clog << "Moving patched file in place: " << std::endl
+ 		   << DestFile << " -> " << FinalFile << std::endl;
+       }
+       Rename(DestFile,FinalFile);
+ 
+       // see if there is more to download
+       if(available_patches.size() > 0) {
+ 	 new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
+ 			      ExpectedMD5, available_patches);
+ 	 return Finish();
+       } else 
+ 	 return Finish(true);
+    }
+ }
+ 
+ 
  // AcqIndex::AcqIndex - Constructor					/*{{{*/
  // ---------------------------------------------------------------------
  /* The package file is added to the queue and a second class is 
     instantiated to fetch the revision file */   
  pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
  			 string URI,string URIDesc,string ShortDesc,
- 			 string ExpectedMD5, string comprExt) :
-    Item(Owner), RealURI(URI), ExpectedMD5(ExpectedMD5)
+ 			 string ExpectedMD5, string comprExt)
+    : Item(Owner), RealURI(URI), ExpectedMD5(ExpectedMD5)
  {
     Decompression = false;
     Erase = false;
@@@ -184,7 -603,7 +603,7 @@@ string pkgAcqIndex::Custom600Headers(
  void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  {
     // no .bz2 found, retry with .gz
 -   if(Desc.URI.substr(Desc.URI.size()-3,Desc.URI.size()-1) == "bz2") {
 +   if(Desc.URI.substr(Desc.URI.size()-3) == "bz2") {
        Desc.URI = Desc.URI.substr(0,Desc.URI.size()-3) + "gz"; 
  
        // retry with a gzip one 
@@@ -290,7 -709,7 +709,7 @@@ void pkgAcqIndex::Done(string Message,u
     else
        Local = true;
     
 -   string compExt = Desc.URI.substr(Desc.URI.size()-3,Desc.URI.size()-1);
 +   string compExt = Desc.URI.substr(Desc.URI.size()-3);
     char *decompProg;
     if(compExt == "bz2") 
        decompProg = "bzip2";
@@@ -315,9 -734,10 +734,9 @@@ pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquir
  			     const vector<IndexTarget*>* IndexTargets,
  			     indexRecords* MetaIndexParser) :
     Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
 -   MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc)
 +   MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
 +   MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
  {
 -   this->MetaIndexParser = MetaIndexParser;
 -   this->IndexTargets = IndexTargets;
     DestFile = _config->FindDir("Dir::State::lists") + "partial/";
     DestFile += URItoFileName(URI);
  
@@@ -340,6 -760,12 +759,6 @@@
        // File was already in place.  It needs to be re-verified
        // because Release might have changed, so Move it into partial
        Rename(Final,DestFile);
 -      // unlink the file and do not try to use I-M-S and Last-Modified
 -      // if the users proxy is broken
 -      if(_config->FindB("Acquire::BrokenProxy", false) == true) {
 -	 std::cerr << "forcing re-get of the signature file as requested" << std::endl;
 -	 unlink(DestFile.c_str());
 -      }
     }
  
     QueueURI(Desc);
@@@ -389,19 -815,18 +808,19 @@@ void pkgAcqMetaSig::Done(string Message
  									/*}}}*/
  void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  {
 -   // Delete any existing sigfile, so that this source isn't
 -   // mistakenly trusted
 -   string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 -   unlink(Final.c_str());
  
 -   // if we get a timeout if fail
 +   // if we get a network error we fail gracefully
     if(LookupTag(Message,"FailReason") == "Timeout" || 
 -      LookupTag(Message,"FailReason") == "TmpResolveFailure") {
 +      LookupTag(Message,"FailReason") == "TmpResolveFailure" ||
 +      LookupTag(Message,"FailReason") == "ConnectionRefused") {
        Item::Failed(Message,Cnf);
        return;
     }
  
 +   // Delete any existing sigfile when the acquire failed
 +   string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 +   unlink(Final.c_str());
 +
     // queue a pkgAcqMetaIndex with no sigfile
     new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
  		       "", IndexTargets, MetaIndexParser);
@@@ -424,9 -849,11 +843,9 @@@ pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcq
  				 string SigFile,
  				 const vector<struct IndexTarget*>* IndexTargets,
  				 indexRecords* MetaIndexParser) :
 -  Item(Owner), RealURI(URI), SigFile(SigFile)
 +   Item(Owner), RealURI(URI), SigFile(SigFile), AuthPass(false),
 +   MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets), IMSHit(false)
  {
 -   this->AuthPass = false;
 -   this->MetaIndexParser = MetaIndexParser;
 -   this->IndexTargets = IndexTargets;
     DestFile = _config->FindDir("Dir::State::lists") + "partial/";
     DestFile += URItoFileName(URI);
  
@@@ -518,9 -945,6 +937,9 @@@ void pkgAcqMetaIndex::RetrievalDone(str
        return;
     }
  
 +   // see if the download was a IMSHit
 +   IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
 +
     Complete = true;
  
     string FinalFile = _config->FindDir("Dir::State::lists");
@@@ -549,7 -973,7 +968,7 @@@ void pkgAcqMetaIndex::AuthDone(string M
        return;
     }
  
 -   if (!VerifyVendor())
 +   if (!VerifyVendor(Message))
     {
        return;
     }
@@@ -601,13 -1025,18 +1020,18 @@@ void pkgAcqMetaIndex::QueueIndexes(boo
           }
        }
        
-       // Queue Packages file
-       new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description,
-                       (*Target)->ShortDesc, ExpectedIndexMD5);
+       // Queue Packages file (either diff or full packages files, depending
+       // on the users option)
+       if(_config->FindB("Acquire::PDiffs",false) == false) 
+ 	 new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
+ 			     (*Target)->ShortDesc, ExpectedIndexMD5);
+       else 
+ 	 new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description,
+ 			    (*Target)->ShortDesc, ExpectedIndexMD5);
     }
  }
  
 -bool pkgAcqMetaIndex::VerifyVendor()
 +bool pkgAcqMetaIndex::VerifyVendor(string Message)
  {
  //    // Maybe this should be made available from above so we don't have
  //    // to read and parse it every time?
@@@ -632,22 -1061,6 +1056,22 @@@
  //          break;
  //       }
  //    }
 +   string::size_type pos;
 +
 +   // check for missing sigs (that where not fatal because otherwise we had
 +   // bombed earlier)
 +   string missingkeys;
 +   string msg = _("There are no public key available for the "
 +		  "following key IDs:\n");
 +   pos = Message.find("NO_PUBKEY ");
 +   if (pos != std::string::npos)
 +   {
 +      string::size_type start = pos+strlen("NO_PUBKEY ");
 +      string Fingerprint = Message.substr(start, Message.find("\n")-start);
 +      missingkeys += (Fingerprint);
 +   }
 +   if(!missingkeys.empty())
 +      _error->Warning("%s", string(msg+missingkeys).c_str());
  
     string Transformed = MetaIndexParser->GetExpectedDist();
  
@@@ -656,7 -1069,7 +1080,7 @@@
        Transformed = "experimental";
     }
  
 -   string::size_type pos = Transformed.rfind('/');
 +   pos = Transformed.rfind('/');
     if (pos != string::npos)
     {
        Transformed = Transformed.substr(0, pos);
@@@ -702,30 -1115,10 +1126,30 @@@ void pkgAcqMetaIndex::Failed(string Mes
  {
     if (AuthPass == true)
     {
 -      // gpgv method failed
 +      // if we fail the authentication but got the file via a IMS-Hit 
 +      // this means that the file wasn't downloaded and that it might be
 +      // just stale (server problem, proxy etc). we delete what we have
 +      // queue it again without i-m-s 
 +      // alternatively we could just unlink the file and let the user try again
 +      if (IMSHit)
 +      {
 +	 Complete = false;
 +	 Local = false;
 +	 AuthPass = false;
 +	 unlink(DestFile.c_str());
 +
 +	 DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 +	 DestFile += URItoFileName(RealURI);
 +	 Desc.URI = RealURI;
 +	 QueueURI(Desc);
 +	 return;
 +      }
 +
 +      // gpgv method failed 
        _error->Warning("GPG error: %s: %s",
                        Desc.Description.c_str(),
                        LookupTag(Message,"Message").c_str());
 +
     }
  
     // No Release file was present, or verification failed, so fall
@@@ -801,12 -1194,6 +1225,12 @@@ pkgAcqArchive::pkgAcqArchive(pkgAcquir
        }
     }
  
 +   // "allow-unauthenticated" restores apts old fetching behaviour
 +   // that means that e.g. unauthenticated file:// uris are higher
 +   // priority than authenticated http:// uris
 +   if (_config->FindB("APT::Get::AllowUnauthenticated",false) == true)
 +      Trusted = false;
 +
     // Select a source
     if (QueueNext() == false && _error->PendingError() == false)
        _error->Error(_("I wasn't able to locate file for the %s package. "
@@@ -1039,19 -1426,13 +1463,19 @@@ void pkgAcqArchive::Finished(
  // ---------------------------------------------------------------------
  /* The file is added to the queue */
  pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string MD5,
 -		       unsigned long Size,string Dsc,string ShortDesc) :
 +		       unsigned long Size,string Dsc,string ShortDesc,
 +		       const string &DestDir, const string &DestFilename) :
                         Item(Owner), Md5Hash(MD5)
  {
     Retries = _config->FindI("Acquire::Retries",0);
     
 -   DestFile = flNotDir(URI);
 -   
 +   if(!DestFilename.empty())
 +      DestFile = DestFilename;
 +   else if(!DestDir.empty())
 +      DestFile = DestDir + "/" + flNotDir(URI);
 +   else
 +      DestFile = flNotDir(URI);
 +
     // Create the item
     Desc.URI = URI;
     Desc.Description = Dsc;
@@@ -1071,7 -1452,7 +1495,7 @@@
        else
  	 PartialSize = Buf.st_size;
     }
 -   
 +
     QueueURI(Desc);
  }
  									/*}}}*/
diff --combined apt-pkg/acquire-item.h
index da1bea801,3a6e93b1a..3649d7a03
--- a/apt-pkg/acquire-item.h
+++ b/apt-pkg/acquire-item.h
@@@ -82,6 -82,70 +82,70 @@@ class pkgAcquire::Ite
     virtual ~Item();
  };
  
+ // item for index diffs
+ 
+ struct DiffInfo {
+    string file;
+    string sha1;
+    unsigned long size;
+ };
+ 
+ class pkgAcqDiffIndex : public pkgAcquire::Item
+ {
+  protected:
+    bool Debug;
+    pkgAcquire::ItemDesc Desc;
+    string RealURI;
+    string ExpectedMD5;
+    string CurrentPackagesFile;
+    string Description;
+ 
+  public:
+    // Specialized action members
+    virtual void Failed(string Message,pkgAcquire::MethodConfig *Cnf);
+    virtual void Done(string Message,unsigned long Size,string Md5Hash,
+ 		     pkgAcquire::MethodConfig *Cnf);
+    virtual string DescURI() {return RealURI + "Index";};
+    virtual string Custom600Headers();
+ 
+    // helpers
+    bool ParseDiffIndex(string IndexDiffFile);
+    
+    pkgAcqDiffIndex(pkgAcquire *Owner,string URI,string URIDesc,
+ 		   string ShortDesct, string ExpectedMD5);
+ };
+ 
+ class pkgAcqIndexDiffs : public pkgAcquire::Item
+ {
+    protected:
+    bool Debug;
+    pkgAcquire::ItemDesc Desc;
+    string RealURI;
+    string ExpectedMD5;
+ 
+    // this is the SHA-1 sum we expect after the patching
+    string Description;
+    vector<DiffInfo> available_patches;
+    enum {StateFetchIndex,StateFetchDiff,StateUnzipDiff,StateApplyDiff} State;
+ 
+    public:
+    
+    // Specialized action members
+    virtual void Failed(string Message,pkgAcquire::MethodConfig *Cnf);
+    virtual void Done(string Message,unsigned long Size,string Md5Hash,
+ 		     pkgAcquire::MethodConfig *Cnf);
+    virtual string DescURI() {return RealURI + "Index";};
+ 
+    // various helpers
+    bool QueueNextDiff();
+    bool ApplyDiff(string PatchFile);
+    void Finish(bool allDone=false);
+ 
+    pkgAcqIndexDiffs(pkgAcquire *Owner,string URI,string URIDesc,
+ 		    string ShortDesct, string ExpectedMD5,
+ 		    vector<DiffInfo> diffs=vector<DiffInfo>());
+ };
+ 
  // Item class for index files
  class pkgAcqIndex : public pkgAcquire::Item
  {
@@@ -151,10 -215,8 +215,10 @@@ class pkgAcqMetaIndex : public pkgAcqui
     const vector<struct IndexTarget*>* IndexTargets;
     indexRecords* MetaIndexParser;
     bool AuthPass;
 +   // required to deal gracefully with problems caused by incorrect ims hits
 +   bool IMSHit; 
  
 -   bool VerifyVendor();
 +   bool VerifyVendor(string Message);
     void RetrievalDone(string Message);
     void AuthDone(string Message);
     void QueueIndexes(bool verify);
@@@ -226,14 -288,9 +290,14 @@@ class pkgAcqFile : public pkgAcquire::I
  		     pkgAcquire::MethodConfig *Cnf);
     virtual string MD5Sum() {return Md5Hash;};
     virtual string DescURI() {return Desc.URI;};
 -   
 -   pkgAcqFile(pkgAcquire *Owner,string URI,string MD5,unsigned long Size,
 -		  string Desc,string ShortDesc);
 +
 +   // If DestFilename is empty, download to DestDir/<basename> if
 +   // DestDir is non-empty, $CWD/<basename> otherwise.  If
 +   // DestFilename is NOT empty, DestDir is ignored and DestFilename
 +   // is the absolute name to which the file should be downloaded.
 +   pkgAcqFile(pkgAcquire *Owner, string URI, string MD5, unsigned long Size,
 +	      string Desc, string ShortDesc,
 +	      const string &DestDir="", const string &DestFilename="");
  };
  
  #endif
diff --combined configure.in
index 8d4b99904,d9c3289ff..4c5583a0f
--- a/configure.in
+++ b/configure.in
@@@ -18,7 -18,7 +18,7 @@@ AC_CONFIG_AUX_DIR(buildlib
  AC_CONFIG_HEADER(include/config.h:buildlib/config.h.in include/apti18n.h:buildlib/apti18n.h.in)
  
  dnl -- SET THIS TO THE RELEASE VERSION --
- AC_DEFINE_UNQUOTED(VERSION,"0.6.43.3")
 -AC_DEFINE_UNQUOTED(VERSION,"0.6.41")
++AC_DEFINE_UNQUOTED(VERSION,"0.6.43.4")
  PACKAGE="apt"
  AC_DEFINE_UNQUOTED(PACKAGE,"$PACKAGE")
  AC_SUBST(PACKAGE)
diff --combined debian/changelog
index b8633d9cb,cc3da8f84..89322ae31
--- a/debian/changelog
+++ b/debian/changelog
@@@ -1,186 -1,3 +1,187 @@@
 +apt (0.6.43.4) unstable; urgency=low
 +
 +  * apt-pkg/acquire.cc: don't show ETA if it is 0 or absurdely large
 +  * apt-pkg/contrib/sha256.{cc,h},hashes.{cc,h}: support for sha256 
 +    (thanks to Anthony Towns)
 +  * ftparchive/cachedb.{cc,h},writer.{cc,h}: optimizations 
 +    (thanks to Anthony Towns)
++  * apt pdiff support from experimental merged
 +
-  --
++ -- Michael Vogt <mvo@debian.org>  Sun,  2 Apr 2006 13:05:24 +0200
 +
 +apt (0.6.43.3) unstable; urgency=low
 +
 +  * Merge bubulle@debian.org--2005/apt--main--0 up to patch-186:
 +    * ca.po: Completed to 512t. Closes: #351592
 +    * eu.po: Completed to 512t. Closes: #350483
 +    * ja.po: Completed to 512t. Closes: #349806
 +    * pl.po: Completed to 512t. Closes: #349514
 +    * sk.po: Completed to 512t. Closes: #349474
 +    * gl.po: Completed to 512 strings Closes: #349407
 +    * sv.po: Completed to 512 strings Closes: #349210
 +    * ru.po: Completed to 512 strings Closes: #349154
 +    * da.po: Completed to 512 strings Closes: #349084
 +    * fr.po: Completed to 512 strings
 +    * vi.po: Completed to 511 strings  Closes: #348968
 +    * zh_CN.po: Completed to 512t. Closes: #353936
 +    * it.po: Completed to 512t. Closes: #352803
 +    * pt_BR.po: Completed to 512t. Closes: #352419
 +    * LINGUAS: Add Welsh
 +    * *.po: Updated from sources (512 strings)
 +  * apt-pkg/deb/deblistparser.cc:
 +    - don't explode on a DepCompareOp in a Provides line, but warn about
 +      it and ignore it otherwise (thanks to James Troup for reporting it)
 +  * cmdline/apt-get.cc:
 +    - don't lock the lists directory in DoInstall, breaks --print-uri 
 +      (thanks to James Troup for reporting it)
 +  * debian/apt.dirs: create /etc/apt/sources.list.d 
 +  * make apt-cache madison work without deb-src entries (#352583)
 +  * cmdline/apt-get.cc: only run the list-cleaner if a update was 
 +    successfull
 +
 + -- Michael Vogt <mvo@debian.org>  Wed, 22 Feb 2006 10:13:04 +0100
 +
 +apt (0.6.43.2) unstable; urgency=low
 +
 +  * Merge bubulle@debian.org--2005/apt--main--0 up to patch-166:
 +    - en_GB.po, de.po: fix spaces errors in "Ign " translations Closes: #347258
 +    - makefile: make update-po a pre-requisite of clean target so
 +    	        that POT and PO files are always up-to-date
 +    - sv.po: Completed to 511t. Closes: #346450
 +    - sk.po: Completed to 511t. Closes: #346369
 +    - fr.po: Completed to 511t
 +    - *.po: Updated from sources (511 strings)
 +    - el.po: Completed to 511 strings Closes: #344642
 +    - da.po: Completed to 511 strings Closes: #348574
 +    - es.po: Updated to 510t1f Closes: #348158
 +    - gl.po: Completed to 511 strings Closes: #347729
 +    - it.po: Yet another update Closes: #347435
 +  * added debian-archive-keyring to the Recommends (closes: #347970)
 +  * fixed message in apt-key to install debian-archive-keyring 
 +  * typos fixed in apt-cache.8 (closes: #348348, #347349)
 +  * add patch to fix http download corruption problem (thanks to
 +    Petr Vandrovec, closes: #280844, #290694)
 +
 + -- Michael Vogt <mvo@debian.org>  Thu, 19 Jan 2006 00:06:33 +0100
 +
 +apt (0.6.43.1) unstable; urgency=low
 +
 +  * Merge bubulle@debian.org--2005/apt--main--0 up to patch-148:
 +    * fr.po: Completed to 510 strings
 +    * it.po: Completed to 510t
 +    * en_GB.po: Completed to 510t
 +    * cs.po: Completed to 510t
 +    * zh_CN.po: Completed to 510t
 +    * el.po: Updated to 510t
 +    * vi.po: Updated to 383t93f34u
 +    * tl.po: Completed to 510 strings (Closes: #344306)
 +    * sv.po: Completed to 510 strings (Closes: #344056)
 +    * LINGUAS: disabled Hebrew translation. (Closes: #313283)
 +    * eu.po: Completed to 510 strings (Closes: #342091)
 +  * apt-get source won't download already downloaded files again
 +    (closes: #79277)
 +  * share/debian-archive.gpg: new 2006 ftp-archive signing key added
 +    (#345891)
 +  * redownload the Release file if IMS-Hit and gpg failure
 +  * deal with multiple signatures on a Release file
 +
 + -- Michael Vogt <mvo@debian.org>  Fri,  6 Jan 2006 01:17:08 +0100
 +
 +apt (0.6.43) unstable; urgency=medium
 +
 +  * Merge bubulle@debian.org--2005/apt--main--0 up to patch-132:  
 +    * zh_CN.po: Completed to 510 strings(Closes: #338267)
 +    * gl.po: Completed to 510 strings (Closes: #338356)
 +  * added support for "/etc/apt/sources.list.d" directory 
 +    (closes: #66325)
 +  * make pkgDirStream (a bit) more complete
 +  * fix bug in pkgCache::VerIterator::end() (thanks to Daniel Burrows)
 +    (closes: #339533)
 +  * pkgAcqFile is more flexible now (closes: #57091)
 +  * support a download rate limit for http (closes: #146877)
 +  * included lots of the speedup changes from #319377
 +  * add stdint.h to contrib/md5.h (closes: #340448)
 +  * ABI change, library name changed (closes: #339147)
 +  * Fix GNU/kFreeBSD crash on non-existing server file (closes: #317718)
 +  * switch to libdb4.3 in build-depends
 +  
 + -- Michael Vogt <mvo@debian.org>  Tue, 29 Nov 2005 00:17:07 +0100
 +
 +apt (0.6.42.3) unstable; urgency=low
 +
 +  * Merge bubulle@debian.org--2005/apt--main--0 up to patch-129:
 +    - patch-118: Russian translation update by Yuri Kozlov (closes: #335164)
 +    - patch-119: add update-po as a pre-req for binary (closes: #329910)
 +    - patch-121: Complete French translation
 +    - patch-125: Fixed localization of y/n questions in German translation 
 +                 (closes: #337078)
 +    - patch-126: Swedish translation update (closes: #337163)
 +    - patch-127: Complete Tagalog translation (closes: #337306)
 +    - patch-128: Danish translation update (closes: #337949)
 +    - patch-129: Basque translation update (closes: #338101)
 +  * cmdline/apt-get.cc:
 +    - bufix in FindSrc  (closes: #335213, #337910)
 +  * added armeb to archtable (closes: #333599)
 +  * with --allow-unauthenticated use the old fallback behaviour for
 +    sources (closes: #335112)
 +   
 + -- Michael Vogt <mvo@debian.org>  Wed,  9 Nov 2005 07:22:31 +0100
 +
 +apt (0.6.42.2) unstable; urgency=high
 +
 +  * NMU (approved by maintainer)
 +  * Add AMD64 archive signing key to debian-archive.gpg (closes: #336500).
 +  * Add big-endian arm (armeb) support (closes: #333599).
 +  * Priority high to get the AMD key into testing ASAP.
 +
 + -- Frans Pop <fjp@debian.org>  Sun, 30 Oct 2005 21:29:11 +0100
 + 
 +apt (0.6.42.1) unstable; urgency=low
 +
 +  * fix a incorrect example in the apt_prefrences man page
 +    (thanks to Filipus Klutiero, closes: #282918)
 +  * apt-pkg/pkgrecords.cc:
 +    - revert patch from last version, it causes trouble on alpha 
 +      and ia64 (closes: #335102, #335103)
 +  * cmdline/apt-get.cc:
 +    - be extra carefull in FindSrc (closes: #335213)
 +
 + -- Michael Vogt <mvo@debian.org>  Sat, 22 Oct 2005 23:44:35 +0200
 +
 +apt (0.6.42) unstable; urgency=low
 +
 +  * apt-pkg/cdrom.cc:
 +    - unmount the cdrom when apt failed to locate any package files
 +  * allow cdrom failures and fallback to other sources in that case
 +    (closes: #44135)
 +  * better error text when dpkg-source fails 
 +  * Merge bubulle@debian.org--2005/apt--main--0 up to patch-115:
 +    - patch-99: Added Galician translation
 +    - patch-100: Completed Danish translation (Closes: #325686)
 +    - patch-104: French translation completed
 +    - patch-109: Italian translation completed
 +    - patch-112: Swedish translation update 
 +    - patch-115: Basque translation completed (Closes: #333299)
 +  * applied french man-page update (thanks to Philippe Batailler)
 +    (closes: #316638, #327456)
 +  * fix leak in the mmap code, thanks to Daniel Burrows for the
 +    patch (closes: #250583)
 +  * support for apt-get [build-dep|source] -t (closes: #152129)
 +  * added "APT::Authentication::TrustCDROM" option to make the life
 +    for the installer people easier (closes: #334656)
 +  * fix crash in apt-ftparchive (thanks to Bastian Blank for the patch)
 +    (closes: #334671)
 +  * apt-pkg/contrib/md5.cc:
 +    - fix a alignment problem on sparc64 that gives random bus errors
 +      (thanks to Fabbione for providing a test-case)
 +  * init the default ScreenWidth to 79 columns by default 
 +    (Closes: #324921)
 +  * cmdline/apt-cdrom.cc: 
 +    - fix some missing gettext() calls (closes: #334539)
 +  * doc/apt-cache.8.xml: fix typo (closes: #334714)
 +  
 + -- Michael Vogt <mvo@debian.org>  Wed, 19 Oct 2005 22:02:09 +0200
 +
  apt (0.6.41) unstable; urgency=low
  
    * improved the support for "error" and "conffile" reporting from
diff --combined doc/examples/configure-index
index 9e851d753,965a09759..7346ba9bb
--- a/doc/examples/configure-index
+++ b/doc/examples/configure-index
@@@ -72,11 -72,6 +72,11 @@@ AP
       NoAct "false";
    };
  
 +  Authentication
 +  {
 +     TrustCDROM "false";            // consider the CDROM always trusted
 +  };
 +
    GPGV
    {
       TrustedKeyring "/etc/apt/trusted.gpg";
@@@ -104,6 -99,8 +104,8 @@@ Acquir
    Queue-Mode "host";       // host|access
    Retries "0";
    Source-Symlinks "true";
+ 
+   PDiffs "true";     // try to get the IndexFile diffs
    
    // HTTP method configuration
    http 
@@@ -117,7 -114,6 +119,7 @@@
      No-Cache "false";
      Max-Age "86400";     // 1 Day age on index files
      No-Store "false";    // Prevent the cache from storing archives    
 +    Dl-Limit "7";        // 7Kb/sec maximum download rate
    };
  
    ftp
diff --combined methods/http.cc
index 341de94e3,e2337a9ba..cb63ada49
--- a/methods/http.cc
+++ b/methods/http.cc
@@@ -58,12 -58,7 +58,11 @@@ unsigned long PipelineDepth = 10
  unsigned long TimeOut = 120;
  bool Debug = false;
  
--
 +unsigned long CircleBuf::BwReadLimit=0;
 +unsigned long CircleBuf::BwTickReadData=0;
 +struct timeval CircleBuf::BwReadTick={0,0};
 +const unsigned int CircleBuf::BW_HZ=10;
 +  
  // CircleBuf::CircleBuf - Circular input buffer				/*{{{*/
  // ---------------------------------------------------------------------
  /* */
@@@ -71,8 -66,6 +70,8 @@@ CircleBuf::CircleBuf(unsigned long Size
  {
     Buf = new unsigned char[Size];
     Reset();
 +
 +   CircleBuf::BwReadLimit = _config->FindI("Acquire::http::Dl-Limit",0)*1024;
  }
  									/*}}}*/
  // CircleBuf::Reset - Reset to the default state			/*{{{*/
@@@ -98,45 -91,16 +97,45 @@@ void CircleBuf::Reset(
     is non-blocking.. */
  bool CircleBuf::Read(int Fd)
  {
 +   unsigned long BwReadMax;
 +
     while (1)
     {
        // Woops, buffer is full
        if (InP - OutP == Size)
  	 return true;
  
 +      // what's left to read in this tick
 +      BwReadMax = CircleBuf::BwReadLimit/BW_HZ;
 +
 +      if(CircleBuf::BwReadLimit) {
 +	 struct timeval now;
 +	 gettimeofday(&now,0);
 +
 +	 unsigned long d = (now.tv_sec-CircleBuf::BwReadTick.tv_sec)*1000000 +
 +	    now.tv_usec-CircleBuf::BwReadTick.tv_usec;
 +	 if(d > 1000000/BW_HZ) {
 +	    CircleBuf::BwReadTick = now;
 +	    CircleBuf::BwTickReadData = 0;
 +	 } 
 +	 
 +	 if(CircleBuf::BwTickReadData >= BwReadMax) {
 +	    usleep(1000000/BW_HZ);
 +	    return true;
 +	 }
 +      }
 +
        // Write the buffer segment
        int Res;
 -      Res = read(Fd,Buf + (InP%Size),LeftRead());
 +      if(CircleBuf::BwReadLimit) {
 +	 Res = read(Fd,Buf + (InP%Size), 
 +		    BwReadMax > LeftRead() ? LeftRead() : BwReadMax);
 +      } else
 +	 Res = read(Fd,Buf + (InP%Size),LeftRead());
        
 +      if(Res > 0 && BwReadLimit > 0) 
 +	 CircleBuf::BwTickReadData += Res;
 +    
        if (Res == 0)
  	 return false;
        if (Res < 0)
@@@ -241,23 -205,28 +240,23 @@@ bool CircleBuf::WriteTillEl(string &Dat
        if (Buf[I%Size] != '\n')
  	 continue;
        ++I;
 -      if (I < InP  && Buf[I%Size] == '\r')
 -         ++I;
        
        if (Single == false)
        {
 -	 if (Buf[I%Size] != '\n')
 -	    continue;
 -         ++I;
           if (I < InP  && Buf[I%Size] == '\r')
              ++I;
 +         if (I >= InP || Buf[I%Size] != '\n')
 +            continue;
 +         ++I;
        }
        
 -      if (I > InP)
 -	 I = InP;
 -      
        Data = "";
        while (OutP < I)
        {
  	 unsigned long Sz = LeftWrite();
  	 if (Sz == 0)
  	    return false;
 -	 if (I - OutP < LeftWrite())
 +	 if (I - OutP < Sz)
  	    Sz = I - OutP;
  	 Data += string((char *)(Buf + (OutP%Size)),Sz);
  	 OutP += Sz;
@@@ -819,10 -788,7 +818,10 @@@ bool HttpMethod::Flush(ServerState *Srv
  {
     if (File != 0)
     {
 -      SetNonBlock(File->Fd(),false);
 +      // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
 +      // can't be set
 +      if (File->Name() != "/dev/null")
 +	 SetNonBlock(File->Fd(),false);
        if (Srv->In.WriteSpace() == false)
  	 return true;
        
@@@ -850,10 -816,7 +849,10 @@@ bool HttpMethod::ServerDie(ServerState 
     // Dump the buffer to the file
     if (Srv->State == ServerState::Data)
     {
 -      SetNonBlock(File->Fd(),false);
 +      // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
 +      // can't be set
 +      if (File->Name() != "/dev/null")
 +	 SetNonBlock(File->Fd(),false);
        while (Srv->In.WriteSpace() == true)
        {
  	 if (Srv->In.Write(File->Fd()) == false)