]> git.saurik.com Git - apt.git/commitdiff
Merge remote-tracking branch 'upstream/debian/experimental' into feature/acq-trans
authorMichael Vogt <mvo@ubuntu.com>
Tue, 7 Oct 2014 14:38:03 +0000 (16:38 +0200)
committerMichael Vogt <mvo@ubuntu.com>
Tue, 7 Oct 2014 14:38:03 +0000 (16:38 +0200)
Conflicts:
apt-pkg/acquire-item.cc

1  2 
apt-pkg/acquire-item.cc
apt-pkg/acquire-item.h
cmdline/apt-get.cc
debian/apt.postinst
test/integration/framework
test/integration/test-compressed-indexes
test/integration/test-pdiff-usage

index 9c1609048538de5b6e9cd644dfa8effce37966a8,779f828d3133a6806979704be4ec8543beb72f0e..5d0a00055501ca1235746209a7cb8ca76a5cd9c8
@@@ -330,7 -371,7 +330,7 @@@ pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcq
  
     RealURI = Target->URI;
     Desc.Owner = this;
--   Desc.Description = Target->Description + "/DiffIndex";
++   Desc.Description = Target->Description + ".diff/Index";
     Desc.ShortDesc = Target->ShortDesc;
     Desc.URI = Target->URI + ".diff/Index";
  
@@@ -371,7 -411,7 +369,7 @@@ string pkgAcqDiffIndex::Custom600Header
  {
     string Final = _config->FindDir("Dir::State::lists");
     Final += URItoFileName(Desc.URI);
--   
++
     if(Debug)
        std::clog << "Custom600Header-IMS: " << Final << std::endl;
  
@@@ -397,168 -435,256 +393,276 @@@ bool pkgAcqDiffIndex::ParseDiffIndex(st
     if (_error->PendingError() == true)
        return false;
  
-    if(TF.Step(Tags) == true)
-    {
-       bool found = false;
-       DiffInfo d;
-       string size;
+    pkgTagSection Tags;
+    if(unlikely(TF.Step(Tags) == false))
+       return false;
  
-       string const tmp = Tags.FindS("SHA1-Current");
+    HashStringList ServerHashes;
+    unsigned long long ServerSize = 0;
+    for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
+    {
+       std::string tagname = *type;
+       tagname.append("-Current");
+       std::string const tmp = Tags.FindS(tagname.c_str());
+       if (tmp.empty() == true)
+        continue;
+       string hash;
+       unsigned long long size;
        std::stringstream ss(tmp);
-       ss >> ServerSha1 >> size;
-       unsigned long const ServerSize = atol(size.c_str());
+       ss >> hash >> size;
+       if (unlikely(hash.empty() == true))
+        continue;
+       if (unlikely(ServerSize != 0 && ServerSize != size))
+        continue;
+       ServerHashes.push_back(HashString(*type, hash));
+       ServerSize = size;
+    }
  
-       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
-       SHA1Summation SHA1;
-       SHA1.AddFD(fd);
-       string const local_sha1 = SHA1.Result();
+    if (ServerHashes.usable() == false)
+    {
+       if (Debug == true)
+        std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": Did not find a good hashsum in the index" << std::endl;
+       return false;
+    }
  
-       if(local_sha1 == ServerSha1)
+    if (ServerHashes != HashSums())
+    {
+       if (Debug == true)
 +      {
-        // we have the same sha1 as the server so we are done here
-        if(Debug)
-           std::clog << "Package file is up-to-date" << std::endl;
-          // ensure we have no leftovers from previous runs
-          std::string Partial = _config->FindDir("Dir::State::lists");
-          Partial += "partial/" + URItoFileName(RealURI);
-          unlink(Partial.c_str());
-        // list cleanup needs to know that this file as well as the already
-        // present index is ours, so we create an empty diff to save it for us
-        new pkgAcqIndexDiffs(Owner, TransactionManager, Target, 
-                               ExpectedHashes, MetaIndexParser, 
-                               ServerSha1, available_patches);
-        return true;
+        std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": Index has different hashes than parser, probably older, so fail pdiffing" << std::endl;
++         printHashSumComparision(CurrentPackagesFile, ServerHashes, HashSums());
 +      }
-       else
+       return false;
+    }
+    if (ServerHashes.VerifyFile(CurrentPackagesFile) == true)
+    {
+       // we have the same sha1 as the server so we are done here
+       if(Debug)
 -       std::clog << "pkgAcqDiffIndex: Package file is up-to-date" << std::endl;
++       std::clog << "pkgAcqDiffIndex: Package file " << CurrentPackagesFile << " is up-to-date" << std::endl;
++
+       // list cleanup needs to know that this file as well as the already
+       // present index is ours, so we create an empty diff to save it for us
 -      new pkgAcqIndexDiffs(Owner, Target, ExpectedHashes, MetaIndexParser);
++      new pkgAcqIndexDiffs(Owner, TransactionManager, Target,
++                           ExpectedHashes, MetaIndexParser);
+       return true;
+    }
+    FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
+    Hashes LocalHashesCalc;
+    LocalHashesCalc.AddFD(fd);
+    HashStringList const LocalHashes = LocalHashesCalc.GetHashStringList();
+    if(Debug)
+       std::clog << "Server-Current: " << ServerHashes.find(NULL)->toStr() << " and we start at "
+        << fd.Name() << " " << fd.FileSize() << " " << LocalHashes.find(NULL)->toStr() << std::endl;
+    // parse all of (provided) history
+    vector<DiffInfo> available_patches;
+    bool firstAcceptedHashes = true;
+    for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
+    {
+       if (LocalHashes.find(*type) == NULL)
+        continue;
+       std::string tagname = *type;
+       tagname.append("-History");
+       std::string const tmp = Tags.FindS(tagname.c_str());
+       if (tmp.empty() == true)
+        continue;
+       string hash, filename;
+       unsigned long long size;
+       std::stringstream ss(tmp);
+       while (ss >> hash >> size >> filename)
        {
-        if(Debug)
-           std::clog << "SHA1-Current: " << ServerSha1 << " and we start at "<< fd.Name() << " " << fd.Size() << " " << local_sha1 << std::endl;
+        if (unlikely(hash.empty() == true || filename.empty() == true))
+           continue;
  
-        // check the historie and see what patches we need
-        string const history = Tags.FindS("SHA1-History");
-        std::stringstream hist(history);
-        while(hist >> d.sha1 >> size >> d.file)
+        // see if we have a record for this file already
+        std::vector<DiffInfo>::iterator cur = available_patches.begin();
+        for (; cur != available_patches.end(); ++cur)
         {
-           // read until the first match is found
-           // from that point on, we probably need all diffs
-           if(d.sha1 == local_sha1) 
-              found=true;
-           else if (found == false)
+           if (cur->file != filename || unlikely(cur->result_size != size))
               continue;
-           if(Debug)
-              std::clog << "Need to get diff: " << d.file << std::endl;
-           available_patches.push_back(d);
+           cur->result_hashes.push_back(HashString(*type, hash));
+           break;
         }
-        if (available_patches.empty() == false)
+        if (cur != available_patches.end())
+           continue;
+        if (firstAcceptedHashes == true)
+        {
+           DiffInfo next;
+           next.file = filename;
+           next.result_hashes.push_back(HashString(*type, hash));
+           next.result_size = size;
+           next.patch_size = 0;
+           available_patches.push_back(next);
+        }
+        else
         {
-           // patching with too many files is rather slow compared to a fast download
-           unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
-           if (fileLimit != 0 && fileLimit < available_patches.size())
-           {
-              if (Debug)
-                 std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
-                       << ") so fallback to complete download" << std::endl;
-              return false;
-           }
-           // see if the patches are too big
-           found = false; // it was true and it will be true again at the end
-           d = *available_patches.begin();
-           string const firstPatch = d.file;
-           unsigned long patchesSize = 0;
-           std::stringstream patches(Tags.FindS("SHA1-Patches"));
-           while(patches >> d.sha1 >> size >> d.file)
-           {
-              if (firstPatch == d.file)
-                 found = true;
-              else if (found == false)
-                 continue;
-              patchesSize += atol(size.c_str());
-           }
-           unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
-           if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
-           {
-              if (Debug)
-                 std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
-                       << ") so fallback to complete download" << std::endl;
-              return false;
-           }
+           if (Debug == true)
+              std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename
+                 << " wasn't in the list for the first parsed hash! (history)" << std::endl;
+           break;
         }
        }
+       firstAcceptedHashes = false;
+    }
+    if (unlikely(available_patches.empty() == true))
+    {
+       if (Debug)
+        std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": "
+           << "Couldn't find any patches for the patch series." << std::endl;
+       return false;
+    }
+    for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
+    {
+       if (LocalHashes.find(*type) == NULL)
+        continue;
+       std::string tagname = *type;
+       tagname.append("-Patches");
+       std::string const tmp = Tags.FindS(tagname.c_str());
+       if (tmp.empty() == true)
+        continue;
  
-       // we have something, queue the next diff
-       if(found)
+       string hash, filename;
+       unsigned long long size;
+       std::stringstream ss(tmp);
+       while (ss >> hash >> size >> filename)
        {
-          // FIXME: make this use the method
-          PackagesFileReadyInPartial = true;
-        std::string const Partial = GetPartialFileNameFromURI(RealURI);
-          FileFd From(CurrentPackagesFile, FileFd::ReadOnly);
-          FileFd To(Partial, FileFd::WriteEmpty);
-          if(CopyFile(From, To) == false)
-             return _error->Errno("CopyFile", "failed to copy");
-          if(Debug)
-             std::cerr << "Done copying " << CurrentPackagesFile
-                       << " -> " << Partial
-                       << std::endl;
-        // queue the diffs
-        string::size_type const last_space = Description.rfind(" ");
-        if(last_space != string::npos)
-           Description.erase(last_space, Description.size()-last_space);
-        /* decide if we should download patches one by one or in one go:
-           The first is good if the server merges patches, but many don't so client
-           based merging can be attempt in which case the second is better.
-           "bad things" will happen if patches are merged on the server,
-           but client side merging is attempt as well */
-        bool pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true);
-        if (pdiff_merge == true)
-        {
-           // reprepro adds this flag if it has merged patches on the server
-           std::string const precedence = Tags.FindS("X-Patch-Precedence");
-           pdiff_merge = (precedence != "merged");
-        }
+        if (unlikely(hash.empty() == true || filename.empty() == true))
+           continue;
  
-        if (pdiff_merge == false)
-          {
-           new pkgAcqIndexDiffs(Owner, TransactionManager, Target, ExpectedHashes, 
-                                  MetaIndexParser,
-                                  ServerSha1, available_patches);
-          }
-          else
+        // see if we have a record for this file already
+        std::vector<DiffInfo>::iterator cur = available_patches.begin();
+        for (; cur != available_patches.end(); ++cur)
         {
-           std::vector<pkgAcqIndexMergeDiffs*> *diffs = new std::vector<pkgAcqIndexMergeDiffs*>(available_patches.size());
-           for(size_t i = 0; i < available_patches.size(); ++i)
-              (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner,
-                                                        TransactionManager,
-                                                        Target,
-                                                        ExpectedHashes,
-                                                        MetaIndexParser,
-                                                        available_patches[i],
-                                                        diffs);
+           if (cur->file != filename)
+              continue;
+           if (unlikely(cur->patch_size != 0 && cur->patch_size != size))
+              continue;
+           cur->patch_hashes.push_back(HashString(*type, hash));
+           cur->patch_size = size;
+           break;
         }
-          Complete = false;
-          Status = StatDone;
-          Dequeue();
-          return true;
+        if (cur != available_patches.end())
+              continue;
+        if (Debug == true)
+           std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename
+              << " wasn't in the list for the first parsed hash! (patches)" << std::endl;
+        break;
        }
     }
-    // Nothing found, report and return false
-    // Failing here is ok, if we return false later, the full
-    // IndexFile is queued
+    bool foundStart = false;
+    for (std::vector<DiffInfo>::iterator cur = available_patches.begin();
+        cur != available_patches.end(); ++cur)
+    {
+       if (LocalHashes != cur->result_hashes)
+        continue;
+       available_patches.erase(available_patches.begin(), cur);
+       foundStart = true;
+       break;
+    }
+    if (foundStart == false || unlikely(available_patches.empty() == true))
+    {
+       if (Debug)
+        std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": "
+           << "Couldn't find the start of the patch series." << std::endl;
+       return false;
+    }
+    // patching with too many files is rather slow compared to a fast download
+    unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
+    if (fileLimit != 0 && fileLimit < available_patches.size())
+    {
+       if (Debug)
+        std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
+           << ") so fallback to complete download" << std::endl;
+       return false;
+    }
+    // calculate the size of all patches we have to get
+    // note that all sizes are uncompressed, while we download compressed files
+    unsigned long long patchesSize = 0;
+    for (std::vector<DiffInfo>::const_iterator cur = available_patches.begin();
+        cur != available_patches.end(); ++cur)
+       patchesSize += cur->patch_size;
+    unsigned long long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
+    if (false && sizeLimit > 0 && (sizeLimit/100) < patchesSize)
+    {
+       if (Debug)
+        std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
+           << ") so fallback to complete download" << std::endl;
+       return false;
+    }
++   // FIXME: make this use the method
++   PackagesFileReadyInPartial = true;
++   std::string const Partial = GetPartialFileNameFromURI(RealURI);
++   
++   FileFd From(CurrentPackagesFile, FileFd::ReadOnly);
++   FileFd To(Partial, FileFd::WriteEmpty);
++   if(CopyFile(From, To) == false)
++      return _error->Errno("CopyFile", "failed to copy");
 +   
-       std::clog << "Can't find a patch in the index file" << std::endl;
-    return false;
 +   if(Debug)
 -      new pkgAcqIndexDiffs(Owner, Target, ExpectedHashes, MetaIndexParser,
 -          available_patches);
++      std::cerr << "Done copying " << CurrentPackagesFile
++                << " -> " << Partial
++                << std::endl;
++
+    // we have something, queue the diffs
+    string::size_type const last_space = Description.rfind(" ");
+    if(last_space != string::npos)
+       Description.erase(last_space, Description.size()-last_space);
+    /* decide if we should download patches one by one or in one go:
+       The first is good if the server merges patches, but many don't so client
+       based merging can be attempt in which case the second is better.
+       "bad things" will happen if patches are merged on the server,
+       but client side merging is attempt as well */
+    bool pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true);
+    if (pdiff_merge == true)
+    {
+       // reprepro adds this flag if it has merged patches on the server
+       std::string const precedence = Tags.FindS("X-Patch-Precedence");
+       pdiff_merge = (precedence != "merged");
+    }
+    if (pdiff_merge == false)
+    {
 -       (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, Target,
++      new pkgAcqIndexDiffs(Owner, TransactionManager, Target, ExpectedHashes, 
++                           MetaIndexParser, available_patches);
+    }
+    else
+    {
+       std::vector<pkgAcqIndexMergeDiffs*> *diffs = new std::vector<pkgAcqIndexMergeDiffs*>(available_patches.size());
+       for(size_t i = 0; i < available_patches.size(); ++i)
++       (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, TransactionManager,
++               Target,
+              ExpectedHashes,
+              MetaIndexParser,
+              available_patches[i],
+              diffs);
+    }
+    Complete = false;
+    Status = StatDone;
+    Dequeue();
+    return true;
  }
                                                                        /*}}}*/
 -void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 +void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * Cnf)/*{{{*/
  {
     if(Debug)
        std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl
@@@ -578,29 -705,21 +682,33 @@@ void pkgAcqDiffIndex::Done(string Messa
  
     Item::Done(Message, Size, Hashes, Cnf);
  
 -   FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 +   // verify the index target
 +   if(Target && Target->MetaKey != "" && MetaIndexParser && Hashes.usable())
 +   {
 +      std::string IndexMetaKey  = Target->MetaKey + ".diff/Index";
 +      indexRecords::checkSum *Record = MetaIndexParser->Lookup(IndexMetaKey);
 +      if(Record && Record->Hashes.usable() && Hashes != Record->Hashes)
 +      {
 +         RenameOnError(HashSumMismatch);
 +         printHashSumComparision(RealURI, Record->Hashes, Hashes);
 +         Failed(Message, Cnf);
 +         return;
 +      }
 +
 +   }
 +
+    string FinalFile;
 -   // success in downloading the index
 -   // rename the index
 -   FinalFile += string(".IndexDiff");
 -   if(Debug)
 -      std::clog << "Renaming: " << DestFile << " -> " << FinalFile 
 -              << std::endl;
 -   Rename(DestFile,FinalFile);
 -   chmod(FinalFile.c_str(),0644);
 -   DestFile = FinalFile;
++   FinalFile = _config->FindDir("Dir::State::lists");
++   FinalFile += URItoFileName(Desc.URI);
++   if(StringToBool(LookupTag(Message,"IMS-Hit"),false))
++      DestFile = FinalFile;
     if(!ParseDiffIndex(DestFile))
 -      return Failed("Parsing pdiff Index failed", NULL);
 +      return Failed("Message: Couldn't parse pdiff index", Cnf);
 +
 +   // queue for final move
-    string FinalFile;
-    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
-    FinalFile += string(".IndexDiff");
 +   TransactionManager->TransactionStageCopy(this, DestFile, FinalFile);
  
     Complete = true;
     Status = StatDone;
@@@ -618,12 -736,12 +726,11 @@@ pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgA
                                     struct IndexTarget const * const Target,
                                     HashStringList const &ExpectedHashes,
                                     indexRecords *MetaIndexParser,
-                                  string ServerSha1,
                                   vector<DiffInfo> diffs)
 -   : pkgAcqBaseIndex(Owner, Target, ExpectedHashes, MetaIndexParser),
 +   : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser),
-      available_patches(diffs), ServerSha1(ServerSha1)
+      available_patches(diffs)
  {
 -   DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 -   DestFile += URItoFileName(Target->URI);
 +   DestFile = GetPartialFileNameFromURI(Target->URI);
  
     Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
  
@@@ -675,18 -789,6 +782,11 @@@ void pkgAcqIndexDiffs::Finish(bool allD
         return;
        }
  
-       PartialFile = GetPartialFileNameFromURI(RealURI);
-       DestFile = _config->FindDir("Dir::State::lists");
-       DestFile += URItoFileName(RealURI);
-       // this happens if we have a up-to-date indexfile
-       if(!FileExists(PartialFile))
-          PartialFile = DestFile;
-       TransactionManager->TransactionStageCopy(this, PartialFile, DestFile);
 +      // queue for copy
++      std::string FinalFile = _config->FindDir("Dir::State::lists");
++      FinalFile += URItoFileName(RealURI);
++      TransactionManager->TransactionStageCopy(this, DestFile, FinalFile);
 +
        // this is for the "real" finish
        Complete = true;
        Status = StatDone;
                                                                        /*}}}*/
  bool pkgAcqIndexDiffs::QueueNextDiff()                                        /*{{{*/
  {
 -
     // calc sha1 of the just patched file
 -   string FinalFile = _config->FindDir("Dir::State::lists");
 -   FinalFile += URItoFileName(RealURI);
 +   std::string const FinalFile = GetPartialFileNameFromURI(RealURI);
 +
 +   if(!FileExists(FinalFile))
 +   {
 +      Failed("Message: No FinalFile " + FinalFile + " available", NULL);
 +      return false;
 +   }
  
     FileFd fd(FinalFile, FileFd::ReadOnly);
-    SHA1Summation SHA1;
-    SHA1.AddFD(fd);
-    string local_sha1 = string(SHA1.Result());
+    Hashes LocalHashesCalc;
+    LocalHashesCalc.AddFD(fd);
+    HashStringList const LocalHashes = LocalHashesCalc.GetHashStringList();
     if(Debug)
-       std::clog << "QueueNextDiff: " 
-               << FinalFile << " (" << local_sha1 << ")"<<std::endl;
+       std::clog << "QueueNextDiff: " << FinalFile << " (" << LocalHashes.find(NULL)->toStr() << ")" << std::endl;
+    if (unlikely(LocalHashes.usable() == false || ExpectedHashes.usable() == false))
+    {
+       Failed("Local/Expected hashes are not usable", NULL);
+       return false;
+    }
  
 +
     // final file reached before all patches are applied
-    if(local_sha1 == ServerSha1)
+    if(LocalHashes == ExpectedHashes)
     {
        Finish(true);
        return true;
@@@ -810,13 -933,11 +927,13 @@@ void pkgAcqIndexDiffs::Done(string Mess
  
        // see if there is more to download
        if(available_patches.empty() == false) {
 -       new pkgAcqIndexDiffs(Owner, Target,
 +       new pkgAcqIndexDiffs(Owner, TransactionManager, Target,
                              ExpectedHashes, MetaIndexParser,
-                               ServerSha1, available_patches);
+                             available_patches);
         return Finish();
        } else 
 +         // update
 +         DestFile = FinalFile;
         return Finish(true);
     }
  }
index a3388ca3e09f2098df3ced4bf2873a804b629abe,18d72ca4097d4dd5a8335945bed5c1c5bc541b0f..0e7212fc55fd132eba5a7952a80538119471c3e3
@@@ -343,257 -328,54 +343,263 @@@ struct DiffInfo 
     /** The filename of the diff. */
     std::string file;
  
-    /** The sha1 hash of the diff. */
-    std::string sha1;
+    /** The hashes of the diff */
+    HashStringList result_hashes;
  
-    /** The size of the diff. */
-    unsigned long size;
+    /** The hashes of the file after the diff is applied */
+    HashStringList patch_hashes;
+    /** The size of the file after the diff is applied */
+    unsigned long long result_size;
+    /** The size of the diff itself */
+    unsigned long long patch_size;
  };
                                                                        /*}}}*/
 -/** \brief An item that is responsible for fetching a SubIndex                {{{
 - *
 - *  The MetaIndex file includes only records for important indexes
 - *  and records for these SubIndex files so these can carry records
 - *  for addition files like PDiffs and Translations
 - */
 -class pkgAcqSubIndex : public pkgAcquire::Item
 +                                                                      /*}}}*/
 +
 +class pkgAcqMetaBase  : public pkgAcquire::Item
  {
 +   void *d;
 +
   protected:
 -   /** \brief If \b true, debugging information will be written to std::clog. */
 -   bool Debug;
 +   std::vector<Item*> Transaction;
  
 - public:
 -   // Specialized action members
 -   virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf);
 -   virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes,
 -                   pkgAcquire::MethodConfig *Cnf);
 -   virtual std::string DescURI() {return Desc.URI;};
 -   virtual std::string Custom600Headers() const;
 -   virtual bool ParseIndex(std::string const &IndexFile);
 +   /** \brief A package-system-specific parser for the meta-index file. */
 +   indexRecords *MetaIndexParser;
  
 -   /** \brief Create a new pkgAcqSubIndex.
 +   /** \brief The index files which should be looked up in the meta-index
 +    *  and then downloaded.
 +    */
 +   const std::vector<IndexTarget*>* IndexTargets;
 +
 +   /** \brief If \b true, the index's signature is currently being verified.
 +    */
 +   bool AuthPass;
 +
 +   // required to deal gracefully with problems caused by incorrect ims hits
 +   bool IMSHit; 
 +
 +   /** \brief Starts downloading the individual index files.
      *
 -    *  \param Owner The Acquire object that owns this item.
 +    *  \param verify If \b true, only indices whose expected hashsum
 +    *  can be determined from the meta-index will be downloaded, and
 +    *  the hashsums of indices will be checked (reporting
 +    *  #StatAuthError if there is a mismatch).  If verify is \b false,
 +    *  no hashsum checking will be performed.
 +    */
 +   void QueueIndexes(bool verify);
 +
 +
 +   /** \brief Called when a file is finished being retrieved.
      *
 -    *  \param URI The URI of the list file to download.
 +    *  If the file was not downloaded to DestFile, a copy process is
 +    *  set up to copy it to DestFile; otherwise, Complete is set to \b
 +    *  true and the file is moved to its final location.
      *
 -    *  \param URIDesc A long description of the list file to download.
 +    *  \param Message The message block received from the fetch
 +    *  subprocess.
 +    */
 +   bool CheckDownloadDone(const std::string &Message,
 +                          const std::string &RealURI);
 +
 +   /** \brief Queue the downloaded Signature for verification */
 +   void QueueForSignatureVerify(const std::string &MetaIndexFile,
 +                                const std::string &MetaIndexFileSignature);
 +
 +   /** \brief Called when authentication succeeded.
      *
 -    *  \param ShortDesc A short description of the list file to download.
 +    *  Sanity-checks the authenticated file, queues up the individual
 +    *  index files for download, and saves the signature in the lists
 +    *  directory next to the authenticated list file.
      *
 -    *  \param ExpectedHashes The list file's hashsums which are expected.
 +    *  \param Message The message block received from the fetch
 +    *  subprocess.
 +    */
 +   bool CheckAuthDone(std::string Message, const std::string &RealURI);
 +
 +   /** Check if the current item should fail at this point */
 +   bool CheckStopAuthentication(const std::string &RealURI,
 +                                const std::string &Message);
 +
 +   /** \brief Check that the release file is a release file for the
 +    *  correct distribution.
 +    *
 +    *  \return \b true if no fatal errors were encountered.
      */
 -   pkgAcqSubIndex(pkgAcquire *Owner, std::string const &URI,std::string const &URIDesc,
 -                 std::string const &ShortDesc, HashStringList const &ExpectedHashes);
 +   bool VerifyVendor(std::string Message, const std::string &RealURI);
 +   
 + public:
 +   // transaction code
 +   void Add(Item *I);
 +   void AbortTransaction();
 +   bool TransactionHasError() APT_PURE;
 +   void CommitTransaction();
 +
 +   /** \brief Stage (queue) a copy action when the transaction is commited
 +    */
 +   void TransactionStageCopy(Item *I,
 +                             const std::string &From, 
 +                             const std::string &To);
 +   /** \brief Stage (queue) a removal action when the transaction is commited
 +    */
 +   void TransactionStageRemoval(Item *I, const std::string &FinalFile);
 +
 +   pkgAcqMetaBase(pkgAcquire *Owner,
 +                  const std::vector<IndexTarget*>* IndexTargets,
 +                  indexRecords* MetaIndexParser,
 +                  HashStringList const &ExpectedHashes=HashStringList(),
 +                  pkgAcqMetaBase *TransactionManager=NULL)
 +      : Item(Owner, ExpectedHashes, TransactionManager),
 +        MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets),
 +        AuthPass(false), IMSHit(false) {};
 +};
 +
 +/** \brief An acquire item that downloads the detached signature      {{{
 + *  of a meta-index (Release) file, then queues up the release
 + *  file itself.
 + *
 + *  \todo Why protected members?
 + *
 + *  \sa pkgAcqMetaIndex
 + */
 +class pkgAcqMetaSig : public pkgAcqMetaBase
 +{
 +   void *d;
 +
 +   protected:
 +
 +   /** \brief The URI of the signature file.  Unlike Desc.URI, this is
 +    *  never modified; it is used to determine the file that is being
 +    *  downloaded.
 +    */
 +   std::string RealURI;
 +
 +   /** \brief The file we need to verify */
 +   std::string MetaIndexFile;
 +
 +   /** \brief The file we use to verify the MetaIndexFile with */
 +   std::string MetaIndexFileSignature;
 +
 +   /** \brief Long URI description used in the acquire system */
 +   std::string URIDesc;
 +
 +   /** \brief Short URI description used in the acquire system */
 +   std::string ShortDesc;
 +
 +   public:
 +   
 +   // Specialized action members
 +   virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf);
 +   virtual void Done(std::string Message,unsigned long long Size,
 +                     HashStringList const &Hashes,
 +                   pkgAcquire::MethodConfig *Cnf);
 +   virtual std::string Custom600Headers() const;
 +   virtual std::string DescURI() {return RealURI; };
 +
 +   /** \brief Create a new pkgAcqMetaSig. */
 +   pkgAcqMetaSig(pkgAcquire *Owner,
 +                 pkgAcqMetaBase *TransactionManager,
 +                 std::string URI,std::string URIDesc, std::string ShortDesc,
 +                 std::string MetaIndexFile,
 +               const std::vector<IndexTarget*>* IndexTargets,
 +               indexRecords* MetaIndexParser);
 +   virtual ~pkgAcqMetaSig();
 +};
 +                                                                      /*}}}*/
 +
 +/** \brief An item that is responsible for downloading the meta-index {{{
 + *  file (i.e., Release) itself and verifying its signature.
 + *
 + *  Once the download and verification are complete, the downloads of
 + *  the individual index files are queued up using pkgAcqDiffIndex.
 + *  If the meta-index file had a valid signature, the expected hashsums
 + *  of the index files will be the md5sums listed in the meta-index;
 + *  otherwise, the expected hashsums will be "" (causing the
 + *  authentication of the index files to be bypassed).
 + */
 +class pkgAcqMetaIndex : public pkgAcqMetaBase
 +{
 +   void *d;
 +
 +   protected:
 +   /** \brief The URI that is actually being downloaded; never
 +    *  modified by pkgAcqMetaIndex.
 +    */
 +   std::string RealURI;
 +
 +   std::string URIDesc;
 +   std::string ShortDesc;
 +
 +   /** \brief The URI of the meta-index file for the detached signature */
 +   std::string MetaIndexSigURI;
 +
 +   /** \brief A "URI-style" description of the meta-index file */
 +   std::string MetaIndexSigURIDesc;
 +
 +   /** \brief A brief description of the meta-index file */
 +   std::string MetaIndexSigShortDesc;
 +
 +   /** \brief delayed constructor */
 +   void Init(std::string URIDesc, std::string ShortDesc);
 +   
 +   public:
 +
 +   // Specialized action members
 +   virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf);
 +   virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes,
 +                   pkgAcquire::MethodConfig *Cnf);
 +   virtual std::string Custom600Headers() const;
 +   virtual std::string DescURI() {return RealURI; };
 +   virtual void Finished();
 +
 +   /** \brief Create a new pkgAcqMetaIndex. */
 +   pkgAcqMetaIndex(pkgAcquire *Owner,
 +                   pkgAcqMetaBase *TransactionManager,
 +                 std::string URI,std::string URIDesc, std::string ShortDesc,
 +                   std::string MetaIndexSigURI, std::string MetaIndexSigURIDesc, std::string MetaIndexSigShortDesc,
 +                 const std::vector<IndexTarget*>* IndexTargets,
 +                 indexRecords* MetaIndexParser);
 +};
 +                                                                      /*}}}*/
 +/** \brief An item repsonsible for downloading clearsigned metaindexes        {{{*/
 +class pkgAcqMetaClearSig : public pkgAcqMetaIndex
 +{
 +   void *d;
 +
 +   /** \brief The URI of the meta-index file for the detached signature */
 +   std::string MetaIndexURI;
 +
 +   /** \brief A "URI-style" description of the meta-index file */
 +   std::string MetaIndexURIDesc;
 +
 +   /** \brief A brief description of the meta-index file */
 +   std::string MetaIndexShortDesc;
 +
 +   /** \brief The URI of the detached meta-signature file if the clearsigned one failed. */
 +   std::string MetaSigURI;
 +
 +   /** \brief A "URI-style" description of the meta-signature file */
 +   std::string MetaSigURIDesc;
 +
 +   /** \brief A brief description of the meta-signature file */
 +   std::string MetaSigShortDesc;
 +
 +public:
 +   virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf);
 +   virtual std::string Custom600Headers() const;
 +   virtual void Done(std::string Message,unsigned long long Size,
 +                     HashStringList const &Hashes,
 +                   pkgAcquire::MethodConfig *Cnf);
 +
 +   /** \brief Create a new pkgAcqMetaClearSig. */
 +   pkgAcqMetaClearSig(pkgAcquire *Owner,
 +              std::string const &URI, std::string const &URIDesc, std::string const &ShortDesc,
 +              std::string const &MetaIndexURI, std::string const &MetaIndexURIDesc, std::string const &MetaIndexShortDesc,
 +              std::string const &MetaSigURI, std::string const &MetaSigURIDesc, std::string const &MetaSigShortDesc,
 +              const std::vector<IndexTarget*>* IndexTargets,
 +              indexRecords* MetaIndexParser);
 +   virtual ~pkgAcqMetaClearSig();
  };
                                                                        /*}}}*/
  
Simple merge
Simple merge
index 29e5fafe62360b38dc8e328855c9c165f660aa6a,e83606fae22a93c5c6a3039ac395157608936619..0aa648fb657c734d213b76b9c7cdebdb63a92ced
@@@ -47,6 -47,6 +47,9 @@@ msgskip() { printf "${CWARNING}SKIP${CN
  msgfail() {
        if [ $# -gt 0 ]; then printf "${CFAIL}FAIL: $*${CNORMAL}\n" >&2;
        else printf "${CFAIL}FAIL${CNORMAL}\n" >&2; fi
++        if [ -n "$APT_DEBUG_TESTS" ]; then
++            bash
++        fi
        EXIT_CODE=$((EXIT_CODE+1));
  }
  
index 805ed5964c9298e1f2b78ea78c0bdec19d25a278,805ed5964c9298e1f2b78ea78c0bdec19d25a278..1beb5d8318f0b31a372fc94fa28f66c17843e8ea
@@@ -39,10 -39,10 +39,10 @@@ testrun() 
                test -e rootdir/var/lib/apt/lists/*_Translation-en.${COMPRESS} || F=1
                # there is no point in trying pdiff if we have compressed indexes
                # as we can't patch compressed files (well, we can, but what is the point?)
--              ! test -e rootdir/var/lib/apt/lists/*.IndexDiff || F=1
++              ! test -e rootdir/var/lib/apt/lists/*diff_Index || F=1
        else
                # clear the faked pdiff indexes so the glob below works
--              rm -f rootdir/var/lib/apt/lists/*.IndexDiff
++              rm -f rootdir/var/lib/apt/lists/*diff_Index
                test -e rootdir/var/lib/apt/lists/*_Packages || F=1
                test -e rootdir/var/lib/apt/lists/*_Sources || F=1
                test -e rootdir/var/lib/apt/lists/*_Translation-en || F=1
index e86963f2802960e1fda6a931fff3adc51adfc20d,0d52614291ab1f9fd8b813dcb2e62c3ae387afc4..5bad902147f4a94b4e68e56d0bee27aca6474d71
@@@ -87,7 -94,7 +94,7 @@@ SHA256-Patches
  " aptcache show apt newstuff
  
        msgmsg "Testcase: index is already up-to-date: $*"
--      find rootdir/var/lib/apt/lists -name '*.IndexDiff' -type f -delete
++      find rootdir/var/lib/apt/lists -name '*diff_Index' -type f -delete
        testsuccess aptget update "$@"
        testequal "$(cat ${PKGFILE}-new)
  " aptcache show apt newstuff