* apt-pkg/sourcelist.cc:

[apt.git] / apt-pkg / acquire-item.cc

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index 39ae327cbb64853d8cb869f3e6de049c44053fac..916fca71eeaaf0e1722129c71d90010e36c2cc23 100644 (file)
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -15,6 +15,7 @@
 // Include Files                                                       /*{{{*/
 #include <apt-pkg/acquire-item.h>
 #include <apt-pkg/configuration.h>
 // Include Files                                                       /*{{{*/
 #include <apt-pkg/acquire-item.h>
 #include <apt-pkg/configuration.h>

+#include <apt-pkg/aptconfiguration.h>

 #include <apt-pkg/sourcelist.h>
 #include <apt-pkg/vendorlist.h>
 #include <apt-pkg/error.h>
 #include <apt-pkg/sourcelist.h>
 #include <apt-pkg/vendorlist.h>
 #include <apt-pkg/error.h>


@@ -218,19 +219,19 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)                /*{{{*/
 
    if(TF.Step(Tags) == true)
    {
 
    if(TF.Step(Tags) == true)
    {

-      string local_sha1;

       bool found = false;
       DiffInfo d;
       string size;
 
       bool found = false;
       DiffInfo d;
       string size;
 

-      string tmp = Tags.FindS("SHA1-Current");
+      string const tmp = Tags.FindS("SHA1-Current");

       std::stringstream ss(tmp);
       std::stringstream ss(tmp);

-      ss >> ServerSha1;
+      ss >> ServerSha1 >> size;
+      unsigned long const ServerSize = atol(size.c_str());

 
       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
       SHA1Summation SHA1;
       SHA1.AddFD(fd.Fd(), fd.Size());
 
       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
       SHA1Summation SHA1;
       SHA1.AddFD(fd.Fd(), fd.Size());

-      local_sha1 = string(SHA1.Result());
+      string const local_sha1 = SHA1.Result();

 
       if(local_sha1 == ServerSha1) 
       {
 
       if(local_sha1 == ServerSha1) 
       {


@@ -247,20 +248,56 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)                /*{{{*/
            std::clog << "SHA1-Current: " << ServerSha1 << std::endl;
 
         // check the historie and see what patches we need
            std::clog << "SHA1-Current: " << ServerSha1 << std::endl;
 
         // check the historie and see what patches we need

-        string history = Tags.FindS("SHA1-History");     
+        string const history = Tags.FindS("SHA1-History");

         std::stringstream hist(history);
         std::stringstream hist(history);

-        while(hist >> d.sha1 >> size >> d.file) 
+        while(hist >> d.sha1 >> size >> d.file)

         {
         {

-           d.size = atoi(size.c_str());

            // read until the first match is found
            // read until the first match is found

+           // from that point on, we probably need all diffs

            if(d.sha1 == local_sha1) 
               found=true;
            if(d.sha1 == local_sha1) 
               found=true;

-           // from that point on, we probably need all diffs
-           if(found) 
+           else if (found == false)
+              continue;
+
+           if(Debug)
+              std::clog << "Need to get diff: " << d.file << std::endl;
+           available_patches.push_back(d);
+        }
+
+        if (available_patches.empty() == false)
+        {
+           // patching with too many files is rather slow compared to a fast download
+           unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
+           if (fileLimit != 0 && fileLimit < available_patches.size())

            {
            {

-              if(Debug)
-                 std::clog << "Need to get diff: " << d.file << std::endl;
-              available_patches.push_back(d);
+              if (Debug)
+                 std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
+                       << ") so fallback to complete download" << std::endl;
+              return false;
+           }
+
+           // see if the patches are too big
+           found = false; // it was true and it will be true again at the end
+           d = *available_patches.begin();
+           string const firstPatch = d.file;
+           unsigned long patchesSize = 0;
+           std::stringstream patches(Tags.FindS("SHA1-Patches"));
+           while(patches >> d.sha1 >> size >> d.file)
+           {
+              if (firstPatch == d.file)
+                 found = true;
+              else if (found == false)
+                 continue;
+
+              patchesSize += atol(size.c_str());
+           }
+           unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
+           if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
+           {
+              if (Debug)
+                 std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
+                       << ") so fallback to complete download" << std::endl;
+              return false;

            }
         }
       }
            }
         }
       }


@@ -269,11 +306,11 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)                /*{{{*/
       if(found) 
       {
         // queue the diffs
       if(found) 
       {
         // queue the diffs

-        string::size_type last_space = Description.rfind(" ");
+        string::size_type const last_space = Description.rfind(" ");

         if(last_space != string::npos)
            Description.erase(last_space, Description.size()-last_space);
         new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
         if(last_space != string::npos)
            Description.erase(last_space, Description.size()-last_space);
         new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,

-                             ExpectedHash, available_patches);
+                             ExpectedHash, ServerSha1, available_patches);

         Complete = false;
         Status = StatDone;
         Dequeue();
         Complete = false;
         Status = StatDone;
         Dequeue();


@@ -341,9 +378,10 @@ void pkgAcqDiffIndex::Done(string Message,unsigned long Size,string Md5Hash,       /*{
 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
                                   string URI,string URIDesc,string ShortDesc,
                                   HashString ExpectedHash, 
 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
                                   string URI,string URIDesc,string ShortDesc,
                                   HashString ExpectedHash, 

+                                  string ServerSha1,

                                   vector<DiffInfo> diffs)
    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash), 
                                   vector<DiffInfo> diffs)
    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash), 

-     available_patches(diffs)
+     available_patches(diffs), ServerSha1(ServerSha1)

 {
    
    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 {
    
    DestFile = _config->FindDir("Dir::State::lists") + "partial/";


@@ -429,6 +467,13 @@ bool pkgAcqIndexDiffs::QueueNextDiff()                                     /*{{{*/
       std::clog << "QueueNextDiff: " 
                << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 
       std::clog << "QueueNextDiff: " 
                << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 

+   // final file reached before all patches are applied
+   if(local_sha1 == ServerSha1)
+   {
+      Finish(true);
+      return true;
+   }
+

    // remove all patches until the next matching patch is found
    // this requires the Index file to be ordered
    for(vector<DiffInfo>::iterator I=available_patches.begin();
    // remove all patches until the next matching patch is found
    // this requires the Index file to be ordered
    for(vector<DiffInfo>::iterator I=available_patches.begin();


@@ -526,7 +571,7 @@ void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash,       /*
       // see if there is more to download
       if(available_patches.size() > 0) {
         new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
       // see if there is more to download
       if(available_patches.size() > 0) {
         new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,

-                             ExpectedHash, available_patches);
+                             ExpectedHash, ServerSha1, available_patches);

         return Finish();
       } else 
         return Finish(true);
         return Finish();
       } else 
         return Finish(true);


@@ -551,13 +596,14 @@ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
    if(comprExt.empty()) 
    {
       // autoselect the compression method
    if(comprExt.empty()) 
    {
       // autoselect the compression method

-      if(FileExists("/bin/bzip2")) 
-        CompressionExtension = ".bz2";
-      else 
-        CompressionExtension = ".gz";
-   } else {
-      CompressionExtension = (comprExt == "plain" ? "" : comprExt);
+      std::vector<std::string> types = APT::Configuration::getCompressionTypes();
+      if (types.empty() == true)
+        comprExt = "plain";
+      else
+        comprExt = "." + types[0];

    }
    }

+   CompressionExtension = ((comprExt == "plain" || comprExt == ".") ? "" : comprExt);
+

    Desc.URI = URI + CompressionExtension;
 
    Desc.Description = URIDesc;
    Desc.URI = URI + CompressionExtension;
 
    Desc.Description = URIDesc;


@@ -584,24 +630,27 @@ string pkgAcqIndex::Custom600Headers()
                                                                        /*}}}*/
 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/
 {
                                                                        /*}}}*/
 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/
 {

-   bool descChanged = false;
-   // no .bz2 found, retry with .gz
-   if(Desc.URI.substr(Desc.URI.size()-3) == "bz2") {
-      Desc.URI = Desc.URI.substr(0,Desc.URI.size()-3) + "gz";
-
-      new pkgAcqIndex(Owner, RealURI, Desc.Description,Desc.ShortDesc,
-                     ExpectedHash, string(".gz"));
-         descChanged = true;
-   }
-   // no .gz found, retry with uncompressed
-   else if(Desc.URI.substr(Desc.URI.size()-2) == "gz") {
-      Desc.URI = Desc.URI.substr(0,Desc.URI.size()-2);
+   std::vector<std::string> types = APT::Configuration::getCompressionTypes();

 
 

-      new pkgAcqIndex(Owner, RealURI, Desc.Description,Desc.ShortDesc,
-                     ExpectedHash, string("plain"));
-         descChanged = true;
-   }
-   if (descChanged) {
+   for (std::vector<std::string>::const_iterator t = types.begin();
+       t != types.end(); t++)
+   {
+      // jump over all already tried compression types
+      const unsigned int nameLen = Desc.URI.size() - (*t).size();
+      if(Desc.URI.substr(nameLen) != *t)
+        continue;
+
+      // we want to try it with the next extension (and make sure to 
+      // not skip over the end)
+      t++;
+      if (t == types.end())
+        break;
+
+      // queue new download
+      Desc.URI = Desc.URI.substr(0, nameLen) + *t;
+      new pkgAcqIndex(Owner, RealURI, Desc.Description, Desc.ShortDesc,
+      ExpectedHash, string(".").append(*t));
+      

       Status = StatDone;
       Complete = false;
       Dequeue();
       Status = StatDone;
       Complete = false;
       Dequeue();


@@ -698,11 +747,11 @@ void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash,
       Local = true;
    
    string compExt = flExtension(flNotDir(URI(Desc.URI).Path));
       Local = true;
    
    string compExt = flExtension(flNotDir(URI(Desc.URI).Path));

-   const char *decompProg;
-   if(compExt == "bz2") 
-      decompProg = "bzip2";
-   else if(compExt == "gz") 
-      decompProg = "gzip";
+   string decompProg;
+
+   // get the binary name for your used compression type
+   decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
+   if(decompProg.empty() == false);

    // flExtensions returns the full name if no extension is found
    // this is why we have this complicated compare operation here
    // FIMXE: add a new flJustExtension() that return "" if no
    // flExtensions returns the full name if no extension is found
    // this is why we have this complicated compare operation here
    // FIMXE: add a new flJustExtension() that return "" if no


@@ -717,9 +766,9 @@ void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash,
 
    Decompression = true;
    DestFile += ".decomp";
 
    Decompression = true;
    DestFile += ".decomp";

-   Desc.URI = string(decompProg) + ":" + FileName;
+   Desc.URI = decompProg + ":" + FileName;

    QueueURI(Desc);
    QueueURI(Desc);

-   Mode = decompProg;
+   Mode = decompProg.c_str();

 }
                                                                        /*}}}*/
 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                       /*{{{*/
 }
                                                                        /*}}}*/
 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                       /*{{{*/


@@ -1278,7 +1327,8 @@ pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
    the archive is already available in the cache and stashs the MD5 for
    checking later. */
 bool pkgAcqArchive::QueueNext()
    the archive is already available in the cache and stashs the MD5 for
    checking later. */
 bool pkgAcqArchive::QueueNext()

-{   
+{
+   string const ForceHash = _config->Find("Acquire::ForceHash");

    for (; Vf.end() == false; Vf++)
    {
       // Ignore not source sources
    for (; Vf.end() == false; Vf++)
    {
       // Ignore not source sources


@@ -1301,12 +1351,25 @@ bool pkgAcqArchive::QueueNext()
         return false;
       
       string PkgFile = Parse.FileName();
         return false;
       
       string PkgFile = Parse.FileName();

-      if(Parse.SHA256Hash() != "")
-        ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
-      else if (Parse.SHA1Hash() != "")
-        ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
-      else 
-        ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
+      if (ForceHash.empty() == false)
+      {
+        if(stringcasecmp(ForceHash, "sha256") == 0)
+           ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
+        else if (stringcasecmp(ForceHash, "sha1") == 0)
+           ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
+        else
+           ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
+      }
+      else
+      {
+        string Hash;
+        if ((Hash = Parse.SHA256Hash()).empty() == false)
+           ExpectedHash = HashString("SHA256", Hash);
+        else if ((Hash = Parse.SHA1Hash()).empty() == false)
+           ExpectedHash = HashString("SHA1", Hash);
+        else
+           ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
+      }

       if (PkgFile.empty() == true)
         return _error->Error(_("The package index files are corrupted. No Filename: "
                              "field for package %s."),
       if (PkgFile.empty() == true)
         return _error->Error(_("The package index files are corrupted. No Filename: "
                              "field for package %s."),