Merge remote-tracking branch 'mvo/bugfix/update-progress-reporting' into debian/exper...

diff --combined apt-pkg/acquire-item.cc
index e851432e9c8e4bd9f72609a4dfac8d093bb01234,1443380a171dcff38fd664f61a8b31aa0078bc91..07be72124ccb3b53b659d2dc7ce3eecf8d13fed1
--- 1/apt-pkg/acquire-item.cc
--- 2/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@@ -55,7 -55,8 +55,8 @@@ using namespace std
  /* */
  pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
                         PartialSize(0), Mode(0), ID(0), Complete(false), 
-                        Local(false), QueueCounter(0)
+                        Local(false), QueueCounter(0), 
+                        ExpectedAdditionalItems(0)
  {
     Owner->Add(this);
     Status = StatIdle;
@@@ -342,21 -343,24 +343,24 @@@ bool pkgAcqSubIndex::ParseIndex(string 
   * the original packages file
   */
  pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
-                                string URI,string URIDesc,string ShortDesc,
-                                HashString ExpectedHash)
-    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
-      Description(URIDesc)
+                                  IndexTarget const *Target,
+                                HashString ExpectedHash,
+                                  indexRecords *MetaIndexParser)
+    : Item(Owner), ExpectedHash(ExpectedHash), Target(Target),
+      MetaIndexParser(MetaIndexParser)
+      
  {
     
     Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
  
-    Desc.Description = URIDesc + "/DiffIndex";
+    RealURI = Target->URI;
     Desc.Owner = this;
-    Desc.ShortDesc = ShortDesc;
-    Desc.URI = URI + ".diff/Index";
+    Desc.Description = Target->Description + "/DiffIndex";
+    Desc.ShortDesc = Target->ShortDesc;
+    Desc.URI = Target->URI + ".diff/Index";
  
     DestFile = _config->FindDir("Dir::State::lists") + "partial/";
-    DestFile += URItoFileName(URI) + string(".DiffIndex");
+    DestFile += URItoFileName(Target->URI) + string(".DiffIndex");
  
     if(Debug)
        std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
@@@ -559,8 -563,7 +563,7 @@@ void pkgAcqDiffIndex::Failed(string Mes
        std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl
                << "Falling back to normal index file acquire" << std::endl;
  
-    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc, 
-                  ExpectedHash);
+    new pkgAcqIndex(Owner, Target, ExpectedHash, MetaIndexParser);
  
     Complete = false;
     Status = StatDone;
@@@ -919,7 -922,8 +922,8 @@@ void pkgAcqIndexMergeDiffs::Done(strin
  pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
                         string URI,string URIDesc,string ShortDesc,
                         HashString ExpectedHash, string comprExt)
-    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
+    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash), Target(0),
+      MetaIndexParser(0)
  {
     if(comprExt.empty() == true)
     {
@@@ -935,7 -939,7 +939,7 @@@
     Init(URI, URIDesc, ShortDesc);
  }
  pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target,
-                        HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
+                        HashString const &ExpectedHash, indexRecords *MetaIndexParser)
     : Item(Owner), RealURI(Target->URI), ExpectedHash(ExpectedHash)
  {
     // autoselect the compression method
@@@ -962,6 -966,10 +966,10 @@@
     else
       Verify = true;
  
+    // we need this in Init()
+    this->Target = Target;
+    this->MetaIndexParser = MetaIndexParser;
+ 
     Init(Target->URI, Target->Description, Target->ShortDesc);
  }
                                                                        /*}}}*/
@@@ -974,10 -982,27 +982,27 @@@ void pkgAcqIndex::Init(string const &UR
     DestFile += URItoFileName(URI);
  
     std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
+    std::string MetaKey;
     if (comprExt == "uncompressed")
+    {
        Desc.URI = URI;
+       if(Target)
+          MetaKey = string(Target->MetaKey);
+    }
     else
+    {
        Desc.URI = URI + '.' + comprExt;
+       if(Target)
+          MetaKey = string(Target->MetaKey) + '.' + comprExt;
+    }
+ 
+    // load the filesize
+    if(MetaIndexParser)
+    {
+       indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey);
+       if(Record)
+          FileSize = Record->Size;
+    }
  
     Desc.Description = URIDesc;
     Desc.Owner = this;
@@@ -1079,7 -1104,7 +1104,7 @@@ void pkgAcqIndex::Done(string Message,u
        FinalFile += URItoFileName(RealURI);
        Rename(DestFile,FinalFile);
        chmod(FinalFile.c_str(),0644);
-       
+ 
        /* We restore the original name to DestFile so that the clean operation
           will work OK */
        DestFile = _config->FindDir("Dir::State::lists") + "partial/";
@@@ -1088,6 -1113,7 +1113,7 @@@
        // Remove the compressed version.
        if (Erase == true)
         unlink(DestFile.c_str());
+ 
        return;
     }
  
@@@ -1175,9 -1201,13 +1201,13 @@@ pkgAcqIndexTrans::pkgAcqIndexTrans(pkgA
  {
  }
  pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const *Target,
-                        HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
+                        HashString const &ExpectedHash, indexRecords *MetaIndexParser)
    : pkgAcqIndex(Owner, Target, ExpectedHash, MetaIndexParser)
  {
+    // load the filesize
+    indexRecords::checkSum *Record = MetaIndexParser->Lookup(string(Target->MetaKey));
+    if(Record)
+       FileSize = Record->Size;
  }
                                                                        /*}}}*/
  // AcqIndexTrans::Custom600Headers - Insert custom request headers    /*{{{*/
@@@ -1257,6 -1287,9 +1287,9 @@@ pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquir
        Rename(Final,LastGoodSig);
     }
  
+    // we expect the indextargets + one additional Release file
+    ExpectedAdditionalItems = IndexTargets->size() + 1;
+ 
     QueueURI(Desc);
  }
                                                                        /*}}}*/
@@@ -1309,6 -1342,9 +1342,9 @@@ void pkgAcqMetaSig::Done(string Message
  
     Complete = true;
  
+    // at this point pkgAcqMetaIndex takes over
+    ExpectedAdditionalItems = 0;
+ 
     // put the last known good file back on i-m-s hit (it will
     // be re-verified again)
     // Else do nothing, we have the new file in DestFile then
@@@ -1326,6 -1362,9 +1362,9 @@@ void pkgAcqMetaSig::Failed(string Messa
  {
     string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
  
+    // at this point pkgAcqMetaIndex takes over
+    ExpectedAdditionalItems = 0;
+ 
     // if we get a network error we fail gracefully
     if(Status == StatTransientNetworkError)
     {
@@@ -1376,6 -1415,9 +1415,9 @@@ pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcq
     Desc.ShortDesc = ShortDesc;
     Desc.URI = URI;
  
+    // we expect more item
+    ExpectedAdditionalItems = IndexTargets->size();
+ 
     QueueURI(Desc);
  }
                                                                        /*}}}*/
@@@ -1426,20 -1468,9 +1468,20 @@@ void pkgAcqMetaIndex::Done(string Messa
        }
        else
        {
 +         // FIXME: move this into pkgAcqMetaClearSig::Done on the next
 +         //        ABI break
 +
 +         // if we expect a ClearTextSignature (InRelase), ensure that
 +         // this is what we get and if not fail to queue a 
 +         // Release/Release.gpg, see #346386
 +         if (SigFile == DestFile && !StartsWithGPGClearTextSignature(DestFile))
 +         {
 +            Failed(Message, Cfg);
 +            return;
 +         }
 +
           // There was a signature file, so pass it to gpgv for
           // verification
 -
           if (_config->FindB("Debug::pkgAcquire::Auth", false))
              std::cerr << "Metaindex acquired, queueing gpg verification ("
                        << SigFile << "," << DestFile << ")\n";
@@@ -1565,12 -1596,15 +1607,15 @@@ void pkgAcqMetaIndex::QueueIndexes(boo
         }
     }
  
+    // at this point the real Items are loaded in the fetcher
+    ExpectedAdditionalItems = 0;
+ 
     for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
          Target != IndexTargets->end();
          ++Target)
     {
        HashString ExpectedIndexHash;
-       const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
+       indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
        bool compressedAvailable = false;
        if (Record == NULL)
        {
@@@ -1617,8 -1651,7 +1662,7 @@@
         {
            if (_config->FindB("Acquire::PDiffs",true) == true && transInRelease == true &&
                MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true)
-              new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
-                                  (*Target)->ShortDesc, ExpectedIndexHash);
+              new pkgAcqDiffIndex(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
            else
               new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
         }
@@@ -1631,8 -1664,7 +1675,7 @@@
           instead, but passing the required info to it is to much hassle */
        if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
          MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true))
-        new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
-                            (*Target)->ShortDesc, ExpectedIndexHash);
+        new pkgAcqDiffIndex(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
        else
         new pkgAcqIndex(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
     }
@@@ -1795,6 -1827,10 +1838,10 @@@ pkgAcqMetaClearSig::pkgAcqMetaClearSig(
  {
     SigFile = DestFile;
  
+    // index targets + (worst case:) Release/Release.gpg
+    ExpectedAdditionalItems = IndexTargets->size() + 2;
+ 
+ 
     // keep the old InRelease around in case of transistent network errors
     string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
     if (RealFileExists(Final) == true)
@@@ -1837,6 -1873,9 +1884,9 @@@ string pkgAcqMetaClearSig::Custom600Hea
                                                                        /*}}}*/
  void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/
  {
+    // we failed, we will not get additional items from this method
+    ExpectedAdditionalItems = 0;
+ 
     if (AuthPass == false)
     {
        // Remove the 'old' InRelease file if we try Release.gpg now as otherwise
@@@ -2095,8 -2134,7 +2145,8 @@@ void pkgAcqArchive::Done(string Message
     }
     
     // Check the hash
 -   if(ExpectedHash.toStr() != CalcHash)
 +   // FIXME: could this empty() check impose *any* sort of security issue?
 +   if(ExpectedHash.empty() == false && ExpectedHash.toStr() != CalcHash)
     {
        RenameOnError(HashSumMismatch);
        return;