Merge branch 'debian/sid' into bugfix/update-progress-reporting

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index 30743addf0a85a0e1b8319b190f6ff10e818d2f4..1443380a171dcff38fd664f61a8b31aa0078bc91 100644 (file)
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -55,7 +55,8 @@ using namespace std;
 /* */
 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
                        PartialSize(0), Mode(0), ID(0), Complete(false), 
-                       Local(false), QueueCounter(0)
+                       Local(false), QueueCounter(0), 
+                       ExpectedAdditionalItems(0)
 {
    Owner->Add(this);
    Status = StatIdle;
@@ -342,21 +343,24 @@ bool pkgAcqSubIndex::ParseIndex(string const &IndexFile)          /*{{{*/
  * the original packages file
  */
 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
-                                string URI,string URIDesc,string ShortDesc,
-                                HashString ExpectedHash)
-   : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
-     Description(URIDesc)
+                                 IndexTarget const *Target,
+                                HashString ExpectedHash,
+                                 indexRecords *MetaIndexParser)
+   : Item(Owner), ExpectedHash(ExpectedHash), Target(Target),
+     MetaIndexParser(MetaIndexParser)
+     
 {
    
    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 
-   Desc.Description = URIDesc + "/DiffIndex";
+   RealURI = Target->URI;
    Desc.Owner = this;
-   Desc.ShortDesc = ShortDesc;
-   Desc.URI = URI + ".diff/Index";
+   Desc.Description = Target->Description + "/DiffIndex";
+   Desc.ShortDesc = Target->ShortDesc;
+   Desc.URI = Target->URI + ".diff/Index";
 
    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
-   DestFile += URItoFileName(URI) + string(".DiffIndex");
+   DestFile += URItoFileName(Target->URI) + string(".DiffIndex");
 
    if(Debug)
       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
@@ -559,8 +563,7 @@ void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/
       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl
                << "Falling back to normal index file acquire" << std::endl;
 
-   new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc, 
-                  ExpectedHash);
+   new pkgAcqIndex(Owner, Target, ExpectedHash, MetaIndexParser);
 
    Complete = false;
    Status = StatDone;
@@ -919,7 +922,8 @@ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,string M
 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
                         string URI,string URIDesc,string ShortDesc,
                         HashString ExpectedHash, string comprExt)
-   : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
+   : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash), Target(0),
+     MetaIndexParser(0)
 {
    if(comprExt.empty() == true)
    {
@@ -935,7 +939,7 @@ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
    Init(URI, URIDesc, ShortDesc);
 }
 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target,
-                        HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
+                        HashString const &ExpectedHash, indexRecords *MetaIndexParser)
    : Item(Owner), RealURI(Target->URI), ExpectedHash(ExpectedHash)
 {
    // autoselect the compression method
@@ -962,6 +966,10 @@ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target,
    else
      Verify = true;
 
+   // we need this in Init()
+   this->Target = Target;
+   this->MetaIndexParser = MetaIndexParser;
+
    Init(Target->URI, Target->Description, Target->ShortDesc);
 }
                                                                        /*}}}*/
@@ -974,10 +982,27 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &S
    DestFile += URItoFileName(URI);
 
    std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
+   std::string MetaKey;
    if (comprExt == "uncompressed")
+   {
       Desc.URI = URI;
+      if(Target)
+         MetaKey = string(Target->MetaKey);
+   }
    else
+   {
       Desc.URI = URI + '.' + comprExt;
+      if(Target)
+         MetaKey = string(Target->MetaKey) + '.' + comprExt;
+   }
+
+   // load the filesize
+   if(MetaIndexParser)
+   {
+      indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey);
+      if(Record)
+         FileSize = Record->Size;
+   }
 
    Desc.Description = URIDesc;
    Desc.Owner = this;
@@ -1079,7 +1104,7 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash,
       FinalFile += URItoFileName(RealURI);
       Rename(DestFile,FinalFile);
       chmod(FinalFile.c_str(),0644);
-      
+
       /* We restore the original name to DestFile so that the clean operation
          will work OK */
       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
@@ -1088,6 +1113,7 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash,
       // Remove the compressed version.
       if (Erase == true)
         unlink(DestFile.c_str());
+
       return;
    }
 
@@ -1175,9 +1201,13 @@ pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
 {
 }
 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const *Target,
-                        HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
+                        HashString const &ExpectedHash, indexRecords *MetaIndexParser)
   : pkgAcqIndex(Owner, Target, ExpectedHash, MetaIndexParser)
 {
+   // load the filesize
+   indexRecords::checkSum *Record = MetaIndexParser->Lookup(string(Target->MetaKey));
+   if(Record)
+      FileSize = Record->Size;
 }
                                                                        /*}}}*/
 // AcqIndexTrans::Custom600Headers - Insert custom request headers     /*{{{*/
@@ -1257,6 +1287,9 @@ pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                           /*{{{*/
       Rename(Final,LastGoodSig);
    }
 
+   // we expect the indextargets + one additional Release file
+   ExpectedAdditionalItems = IndexTargets->size() + 1;
+
    QueueURI(Desc);
 }
                                                                        /*}}}*/
@@ -1309,6 +1342,9 @@ void pkgAcqMetaSig::Done(string Message,unsigned long long Size,string MD5,
 
    Complete = true;
 
+   // at this point pkgAcqMetaIndex takes over
+   ExpectedAdditionalItems = 0;
+
    // put the last known good file back on i-m-s hit (it will
    // be re-verified again)
    // Else do nothing, we have the new file in DestFile then
@@ -1326,6 +1362,9 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
 {
    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 
+   // at this point pkgAcqMetaIndex takes over
+   ExpectedAdditionalItems = 0;
+
    // if we get a network error we fail gracefully
    if(Status == StatTransientNetworkError)
    {
@@ -1376,6 +1415,9 @@ pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                       /*{{{*/
    Desc.ShortDesc = ShortDesc;
    Desc.URI = URI;
 
+   // we expect more item
+   ExpectedAdditionalItems = IndexTargets->size();
+
    QueueURI(Desc);
 }
                                                                        /*}}}*/
@@ -1554,12 +1596,15 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
         }
    }
 
+   // at this point the real Items are loaded in the fetcher
+   ExpectedAdditionalItems = 0;
+
    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
         Target != IndexTargets->end();
         ++Target)
    {
       HashString ExpectedIndexHash;
-      const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
+      indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
       bool compressedAvailable = false;
       if (Record == NULL)
       {
@@ -1606,8 +1651,7 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify)                           /*{{{*/
         {
            if (_config->FindB("Acquire::PDiffs",true) == true && transInRelease == true &&
                MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true)
-              new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
-                                  (*Target)->ShortDesc, ExpectedIndexHash);
+              new pkgAcqDiffIndex(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
            else
               new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
         }
@@ -1620,8 +1664,7 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify)                           /*{{{*/
          instead, but passing the required info to it is to much hassle */
       if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
          MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true))
-        new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
-                            (*Target)->ShortDesc, ExpectedIndexHash);
+        new pkgAcqDiffIndex(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
       else
         new pkgAcqIndex(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
    }
@@ -1784,6 +1827,10 @@ pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner,                /*{{{*/
 {
    SigFile = DestFile;
 
+   // index targets + (worst case:) Release/Release.gpg
+   ExpectedAdditionalItems = IndexTargets->size() + 2;
+
+
    // keep the old InRelease around in case of transistent network errors
    string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
    if (RealFileExists(Final) == true)
@@ -1826,6 +1873,9 @@ string pkgAcqMetaClearSig::Custom600Headers()
                                                                        /*}}}*/
 void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/
 {
+   // we failed, we will not get additional items from this method
+   ExpectedAdditionalItems = 0;
+
    if (AuthPass == false)
    {
       // Remove the 'old' InRelease file if we try Release.gpg now as otherwise

diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h
index f48d2a0d79f1350b89867accab78300fd2fa750c..eab5bb2229e3a9c66d16742cfe3a5a5eddc00855 100644 (file)
--- a/apt-pkg/acquire-item.h
+++ b/apt-pkg/acquire-item.h
@@ -166,6 +166,16 @@ class pkgAcquire::Item : public WeakPointable
     *  \sa pkgAcquire
     */
    unsigned int QueueCounter;
+
+   /** \brief The number of additional fetch items that are expected
+    *  once this item is done.
+    *
+    *  Some items like pkgAcqMeta{Index,Sig} will queue additional
+    *  items. This variable can be set by the methods if it knows
+    *  in advance how many items to expect to get a more accurate
+    *  progress.
+    */
+   unsigned int ExpectedAdditionalItems;
    
    /** \brief The name of the file into which the retrieved object
     *  will be written.
@@ -395,6 +405,11 @@ class pkgAcqDiffIndex : public pkgAcquire::Item
     */
    std::string Description;
 
+   /** \brief Pointer to the IndexTarget data
+    */
+   const struct IndexTarget * Target;
+   indexRecords *MetaIndexParser;
+
  public:
    // Specialized action members
    virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf);
@@ -428,8 +443,10 @@ class pkgAcqDiffIndex : public pkgAcquire::Item
     *
     *  \param ExpectedHash The list file's MD5 signature.
     */
-   pkgAcqDiffIndex(pkgAcquire *Owner,std::string URI,std::string URIDesc,
-                  std::string ShortDesc, HashString ExpectedHash);
+   pkgAcqDiffIndex(pkgAcquire *Owner,
+                   struct IndexTarget const * const Target,
+                   HashString ExpectedHash,
+                   indexRecords *MetaIndexParser);
 };
                                                                        /*}}}*/
 /** \brief An item that is responsible for fetching client-merge patches {{{
@@ -713,6 +730,11 @@ class pkgAcqIndex : public pkgAcquire::Item
     */
    std::string CompressionExtension;
 
+   /** \brief Pointer to the IndexTarget data
+    */
+   const struct IndexTarget * Target;
+   indexRecords *MetaIndexParser;
+
    public:
    
    // Specialized action members
@@ -746,7 +768,7 @@ class pkgAcqIndex : public pkgAcquire::Item
               std::string ShortDesc, HashString ExpectedHash, 
               std::string compressExt="");
    pkgAcqIndex(pkgAcquire *Owner, struct IndexTarget const * const Target,
-                        HashString const &ExpectedHash, indexRecords const *MetaIndexParser);
+                        HashString const &ExpectedHash, indexRecords *MetaIndexParser);
    void Init(std::string const &URI, std::string const &URIDesc, std::string const &ShortDesc);
 };
                                                                        /*}}}*/
@@ -778,7 +800,7 @@ class pkgAcqIndexTrans : public pkgAcqIndex
    pkgAcqIndexTrans(pkgAcquire *Owner,std::string URI,std::string URIDesc,
                    std::string ShortDesc);
    pkgAcqIndexTrans(pkgAcquire *Owner, struct IndexTarget const * const Target,
-                   HashString const &ExpectedHash, indexRecords const *MetaIndexParser);
+                   HashString const &ExpectedHash, indexRecords *MetaIndexParser);
 };
                                                                        /*}}}*/
 /** \brief Information about an index file. */                         /*{{{*/

diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc
index a187a00ae9752fc900a2797302cf369780465816..37964c943685c9747d62a329be3e9d9c577ff647 100644 (file)
--- a/apt-pkg/acquire.cc
+++ b/apt-pkg/acquire.cc
@@ -31,6 +31,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
+#include <iomanip>
 
 #include <dirent.h>
 #include <sys/time.h>
@@ -832,6 +833,9 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner)
       if ((*I)->Local == true)
         continue;
 
+      // see if the method tells us to expect more
+      TotalItems += (*I)->ExpectedAdditionalItems;
+
       TotalBytes += (*I)->FileSize;
       if ((*I)->Complete == true)
         CurrentBytes += (*I)->FileSize;
@@ -843,6 +847,7 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner)
    unsigned long long ResumeSize = 0;
    for (pkgAcquire::Worker *I = Owner->WorkersBegin(); I != 0;
        I = Owner->WorkerStep(I))
+   {
       if (I->CurrentItem != 0 && I->CurrentItem->Owner->Complete == false)
       {
         CurrentBytes += I->CurrentSize;
@@ -853,6 +858,7 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner)
             I->CurrentItem->Owner->Complete == false)
            TotalBytes += I->CurrentSize;
       }
+   }
    
    // Normalize the figures and account for unknown size downloads
    if (TotalBytes <= 0)
@@ -863,6 +869,12 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner)
    // Wha?! Is not supposed to happen.
    if (CurrentBytes > TotalBytes)
       CurrentBytes = TotalBytes;
+
+   // debug
+   if (_config->FindB("Debug::acquire::progress", false) == true)
+      std::clog << " Bytes: " 
+                << SizeToStr(CurrentBytes) << " / " << SizeToStr(TotalBytes) 
+                << std::endl;
    
    // Compute the CPS
    struct timeval NewTime;
@@ -883,6 +895,15 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner)
       Time = NewTime;
    }
 
+   // calculate the percentage, if we have too little data assume 0%
+   // FIXME: the 5k is totally arbitrary 
+   if (TotalBytes < 5*1024)
+      Percent = 0;
+   else 
+      // use both files and bytes because bytes can be unreliable
+      Percent = (0.8 * (CurrentBytes/float(TotalBytes)*100.0) + 
+                 0.2 * (CurrentItems/float(TotalItems)*100.0));
+
    int fd = _config->FindI("APT::Status-Fd",-1);
    if(fd > 0) 
    {
@@ -900,13 +921,11 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner)
       else
         snprintf(msg,sizeof(msg), _("Retrieving file %li of %li"), i, TotalItems);
         
-
-
       // build the status str
       status << "dlstatus:" << i
-            << ":"  << (CurrentBytes/float(TotalBytes)*100.0) 
-            << ":" << msg 
-            << endl;
+             << ":"  << std::setprecision(3) << Percent
+             << ":" << msg 
+             << endl;
 
       std::string const dlstatus = status.str();
       FileFd::Write(fd, dlstatus.c_str(), dlstatus.size());

diff --git a/apt-pkg/acquire.h b/apt-pkg/acquire.h
index ef16d8556538bc2ab88bd88b71b0ec802db16d3c..0113021b21dce95c1a4cec534a61732c2e75d5c8 100644 (file)
--- a/apt-pkg/acquire.h
+++ b/apt-pkg/acquire.h
@@ -714,6 +714,10 @@ class pkgAcquireStatus
    /** \brief The number of items that have been successfully downloaded. */
    unsigned long CurrentItems;
    
+   /** \brief The estimated percentage of the download (0-100)
+    */
+   double Percent;
+
    public:
 
    /** \brief If \b true, the download scheduler should call Pulse()

diff --git a/apt-pkg/indexrecords.cc b/apt-pkg/indexrecords.cc
index 5353d109845cd6fe7ebf7584b6a56c3ee0c62906..68ebdce0886fb5065a3cec1e5297b49cbf160107 100644 (file)
--- a/apt-pkg/indexrecords.cc
+++ b/apt-pkg/indexrecords.cc
@@ -53,7 +53,7 @@ APT_PURE time_t indexRecords::GetValidUntil() const
    return this->ValidUntil;
 }
 
-APT_PURE const indexRecords::checkSum *indexRecords::Lookup(const string MetaKey)
+APT_PURE indexRecords::checkSum *indexRecords::Lookup(const string MetaKey)
 {
    std::map<std::string, indexRecords::checkSum* >::const_iterator sum = Entries.find(MetaKey);
    if (sum == Entries.end())

diff --git a/apt-pkg/indexrecords.h b/apt-pkg/indexrecords.h
index e31f889ad17ea384d8bf69e5322323def33d22f6..2260a4ae1edb2202480cb46a58362bbf4adff007 100644 (file)
--- a/apt-pkg/indexrecords.h
+++ b/apt-pkg/indexrecords.h
@@ -41,7 +41,7 @@ class indexRecords
    indexRecords(const std::string ExpectedDist);
 
    // Lookup function
-   virtual const checkSum *Lookup(const std::string MetaKey);
+   virtual checkSum *Lookup(const std::string MetaKey);
    /** \brief tests if a checksum for this file is available */
    bool Exists(std::string const &MetaKey) const;
    std::vector<std::string> MetaKeys();

diff --git a/apt-private/acqprogress.cc b/apt-private/acqprogress.cc
index 0f5b53e500666ff7d48ec48085cfa7fdb9480de3..ee6c4536f1c3ffb964dcb3d3c8d9a9e8af5b6aa1 100644 (file)
--- a/apt-private/acqprogress.cc
+++ b/apt-private/acqprogress.cc
@@ -170,7 +170,7 @@ bool AcqTextStatus::Pulse(pkgAcquire *Owner)
       ScreenWidth = sizeof(Buffer)-1;
 
    // Put in the percent done
-   sprintf(S,"%.0f%%",((CurrentBytes + CurrentItems)*100.0)/(TotalBytes+TotalItems));
+   sprintf(S,"%.0f%%", Percent);
 
    bool Shown = false;
    for (pkgAcquire::Worker *I = Owner->WorkersBegin(); I != 0;