delay progress until Release files are downloaded

Progress reporting used an "upper bound" on files we might get, expect
that this wasn't correct in case pdiff entered the picture. So instead
of calculating a value which is perhaps incorrect, we just accept that
we can't tell how many files we are going to download and just keep at
0% until we know. Additionally, if we have pdiffs we wait until we got
these (sub)index files, too.

That could all be done better by downloading all Release files first and
planing with them in hand accordingly, but one step at a time.

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index a5a3b564b5bcbad6f61786ee8247698313a92519..f4f2b80ecff8bb3e469b38d23c9608fddb85a40b 100644 (file)
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -1188,12 +1188,12 @@ bool pkgAcqMetaBase::CheckAuthDone(string const &Message)               /*{{{*/
                 << DestFile << std::endl;
 
    // Download further indexes with verification
                 << DestFile << std::endl;
 
    // Download further indexes with verification

-   QueueIndexes(true);
+   TransactionManager->QueueIndexes(true);

 
    return true;
 }
                                                                        /*}}}*/
 
    return true;
 }
                                                                        /*}}}*/

-void pkgAcqMetaBase::QueueIndexes(bool const verify)                   /*{{{*/
+void pkgAcqMetaClearSig::QueueIndexes(bool const verify)                       /*{{{*/

 {
    // at this point the real Items are loaded in the fetcher
    ExpectedAdditionalItems = 0;
 {
    // at this point the real Items are loaded in the fetcher
    ExpectedAdditionalItems = 0;


@@ -1455,7 +1455,7 @@ pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire * const Owner,  /*{{{*/
    MetaIndexParser(MetaIndexParser), LastMetaIndexParser(NULL)
 {
    // index targets + (worst case:) Release/Release.gpg
    MetaIndexParser(MetaIndexParser), LastMetaIndexParser(NULL)
 {
    // index targets + (worst case:) Release/Release.gpg

-   ExpectedAdditionalItems = IndexTargets.size() + 2;
+   ExpectedAdditionalItems = std::numeric_limits<decltype(ExpectedAdditionalItems)>::max();

    TransactionManager->Add(this);
 }
                                                                        /*}}}*/
    TransactionManager->Add(this);
 }
                                                                        /*}}}*/


@@ -1529,9 +1529,6 @@ void pkgAcqMetaClearSig::Failed(string const &Message,pkgAcquire::MethodConfig c
 {
    Item::Failed(Message, Cnf);
 
 {
    Item::Failed(Message, Cnf);
 

-   // we failed, we will not get additional items from this method
-   ExpectedAdditionalItems = 0;
-

    if (AuthPass == false)
    {
       if (Status == StatAuthError || Status == StatTransientNetworkError)
    if (AuthPass == false)
    {
       if (Status == StatAuthError || Status == StatTransientNetworkError)


@@ -1579,7 +1576,7 @@ void pkgAcqMetaClearSig::Failed(string const &Message,pkgAcquire::MethodConfig c
         if (TransactionManager->MetaIndexParser->Load(PartialRelease, &ErrorText) == false || VerifyVendor(Message) == false)
            /* expired Release files are still a problem you need extra force for */;
         else
         if (TransactionManager->MetaIndexParser->Load(PartialRelease, &ErrorText) == false || VerifyVendor(Message) == false)
            /* expired Release files are still a problem you need extra force for */;
         else

-           QueueIndexes(true);
+           TransactionManager->QueueIndexes(true);

       }
    }
 }
       }
    }
 }


@@ -1604,9 +1601,6 @@ pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire * const Owner,                /*{{{*/
    Desc.Owner = this;
    Desc.ShortDesc = DataTarget.ShortDesc;
    Desc.URI = DataTarget.URI;
    Desc.Owner = this;
    Desc.ShortDesc = DataTarget.ShortDesc;
    Desc.URI = DataTarget.URI;

-
-   // we expect more item
-   ExpectedAdditionalItems = IndexTargets.size();

    QueueURI(Desc);
 }
                                                                        /*}}}*/
    QueueURI(Desc);
 }
                                                                        /*}}}*/


@@ -1641,7 +1635,7 @@ void pkgAcqMetaIndex::Failed(string const &Message,
       TransactionManager->TransactionStageRemoval(this, GetFinalFilename());
 
       // queue without any kind of hashsum support
       TransactionManager->TransactionStageRemoval(this, GetFinalFilename());
 
       // queue without any kind of hashsum support

-      QueueIndexes(false);
+      TransactionManager->QueueIndexes(false);

    }
 }
                                                                        /*}}}*/
    }
 }
                                                                        /*}}}*/


@@ -1787,7 +1781,7 @@ void pkgAcqMetaSig::Failed(string const &Message,pkgAcquire::MethodConfig const
       if (MetaIndex->VerifyVendor(Message) == false)
         /* expired Release files are still a problem you need extra force for */;
       else
       if (MetaIndex->VerifyVendor(Message) == false)
         /* expired Release files are still a problem you need extra force for */;
       else

-        MetaIndex->QueueIndexes(GoodLoad);
+        TransactionManager->QueueIndexes(GoodLoad);

 
       TransactionManager->TransactionStageCopy(MetaIndex, MetaIndex->DestFile, MetaIndex->GetFinalFilename());
    }
 
       TransactionManager->TransactionStageCopy(MetaIndex, MetaIndex->DestFile, MetaIndex->GetFinalFilename());
    }


@@ -1840,6 +1834,9 @@ pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire * const Owner,
                                  IndexTarget const &Target)
    : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL), diffs(NULL)
 {
                                  IndexTarget const &Target)
    : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL), diffs(NULL)
 {

+   // FIXME: Magic number as an upper bound on pdiffs we will reasonably acquire
+   ExpectedAdditionalItems = 40;
+

    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 
    Desc.Owner = this;
    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 
    Desc.Owner = this;


@@ -1884,6 +1881,7 @@ void pkgAcqDiffIndex::QueueOnIMSHit() const                               /*{{{*/
                                                                        /*}}}*/
 bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)      /*{{{*/
 {
                                                                        /*}}}*/
 bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)      /*{{{*/
 {

+   ExpectedAdditionalItems = 0;

    // failing here is fine: our caller will take care of trying to
    // get the complete file if patching fails
    if(Debug)
    // failing here is fine: our caller will take care of trying to
    // get the complete file if patching fails
    if(Debug)


@@ -2267,6 +2265,7 @@ void pkgAcqDiffIndex::Failed(string const &Message,pkgAcquire::MethodConfig cons
 {
    pkgAcqBaseIndex::Failed(Message,Cnf);
    Status = StatDone;
 {
    pkgAcqBaseIndex::Failed(Message,Cnf);
    Status = StatDone;

+   ExpectedAdditionalItems = 0;

 
    if(Debug)
       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl
 
    if(Debug)
       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl

diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h
index 81b6d3944a313a97f1f87cf171294a02e8529943..8b47226006f4ab777297f5d3bb424ca2b34ddb84 100644 (file)
--- a/apt-pkg/acquire-item.h
+++ b/apt-pkg/acquire-item.h
@@ -416,16 +416,6 @@ class APT_HIDDEN pkgAcqMetaBase : public pkgAcqTransactionItem             /*{{{*/
     */
    bool AuthPass;
 
     */
    bool AuthPass;
 

-   /** \brief Starts downloading the individual index files.
-    *
-    *  \param verify If \b true, only indices whose expected hashsum
-    *  can be determined from the meta-index will be downloaded, and
-    *  the hashsums of indices will be checked (reporting
-    *  #StatAuthError if there is a mismatch).  If verify is \b false,
-    *  no hashsum checking will be performed.
-    */
-   void QueueIndexes(bool const verify);
-

    /** \brief Called when a file is finished being retrieved.
     *
     *  If the file was not downloaded to DestFile, a copy process is
    /** \brief Called when a file is finished being retrieved.
     *
     *  If the file was not downloaded to DestFile, a copy process is


@@ -592,6 +582,16 @@ class APT_HIDDEN pkgAcqMetaClearSig : public pkgAcqMetaIndex
                     pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE;
    virtual void Finished() APT_OVERRIDE;
 
                     pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE;
    virtual void Finished() APT_OVERRIDE;
 

+   /** \brief Starts downloading the individual index files.
+    *
+    *  \param verify If \b true, only indices whose expected hashsum
+    *  can be determined from the meta-index will be downloaded, and
+    *  the hashsums of indices will be checked (reporting
+    *  #StatAuthError if there is a mismatch).  If verify is \b false,
+    *  no hashsum checking will be performed.
+    */
+   void QueueIndexes(bool const verify);
+

    /** \brief Create a new pkgAcqMetaClearSig. */
    pkgAcqMetaClearSig(pkgAcquire * const Owner,
                IndexTarget const &ClearsignedTarget,
    /** \brief Create a new pkgAcqMetaClearSig. */
    pkgAcqMetaClearSig(pkgAcquire * const Owner,
                IndexTarget const &ClearsignedTarget,

diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc
index c65aef329b3d3123de076b24bf94e8ffb8aabc13..7a44d859913d76ab2dcdfcef6e0142f7056182a0 100644 (file)
--- a/apt-pkg/acquire.cc
+++ b/apt-pkg/acquire.cc
@@ -1147,7 +1147,7 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner)
    // Compute the total number of bytes to fetch
    unsigned int Unknown = 0;
    unsigned int Count = 0;
    // Compute the total number of bytes to fetch
    unsigned int Unknown = 0;
    unsigned int Count = 0;

-   bool UnfetchedReleaseFiles = false;
+   bool ExpectAdditionalItems = false;

    for (pkgAcquire::ItemCIterator I = Owner->ItemsBegin(); 
         I != Owner->ItemsEnd();
        ++I, ++Count)
    for (pkgAcquire::ItemCIterator I = Owner->ItemsBegin(); 
         I != Owner->ItemsEnd();
        ++I, ++Count)


@@ -1156,12 +1156,9 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner)
       if ((*I)->Status == pkgAcquire::Item::StatDone)
         ++CurrentItems;
 
       if ((*I)->Status == pkgAcquire::Item::StatDone)
         ++CurrentItems;
 

-      // see if the method tells us to expect more
-      TotalItems += (*I)->ExpectedAdditionalItems;
-
-      // check if there are unfetched Release files
-      if ((*I)->Status != pkgAcquire::Item::StatDone && (*I)->ExpectedAdditionalItems > 0)
-         UnfetchedReleaseFiles = true;
+      // do we expect to acquire more files than we know of yet?
+      if ((*I)->ExpectedAdditionalItems > 0)
+         ExpectAdditionalItems = true;

 
       TotalBytes += (*I)->FileSize;
       if ((*I)->Complete == true)
 
       TotalBytes += (*I)->FileSize;
       if ((*I)->Complete == true)


@@ -1218,7 +1215,7 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner)
 
    double const OldPercent = Percent;
    // calculate the percentage, if we have too little data assume 1%
 
    double const OldPercent = Percent;
    // calculate the percentage, if we have too little data assume 1%

-   if (TotalBytes > 0 && UnfetchedReleaseFiles)
+   if (ExpectAdditionalItems)

       Percent = 0;
    else
       // use both files and bytes because bytes can be unreliable
       Percent = 0;
    else
       // use both files and bytes because bytes can be unreliable