]> git.saurik.com Git - apt.git/blobdiff - apt-pkg/acquire-item.cc
support compression and by-hash for .diff/Index files
[apt.git] / apt-pkg / acquire-item.cc
index 8628679322dc91a11f615da825212b1ca13328ef..7abb7b206a6156a5d347fa8f247e43cef261749e 100644 (file)
@@ -175,7 +175,7 @@ static void ReportMirrorFailureToCentral(pkgAcquire::Item const &I, std::string
 }
                                                                        /*}}}*/
 
-static bool MessageInsecureRepository(bool const isError, char const * const msg, std::string const &repo)/*{{{*/
+static APT_NONNULL(2) bool MessageInsecureRepository(bool const isError, char const * const msg, std::string const &repo)/*{{{*/
 {
    std::string m;
    strprintf(m, msg, repo.c_str());
@@ -195,7 +195,28 @@ static bool MessageInsecureRepository(bool const isError, char const * const msg
                                                                        /*}}}*/
 // AllowInsecureRepositories                                           /*{{{*/
 enum class InsecureType { UNSIGNED, WEAK, NORELEASE };
-static bool APT_NONNULL(3, 4, 5) AllowInsecureRepositories(InsecureType msg, std::string const &repo,
+static bool TargetIsAllowedToBe(IndexTarget const &Target, InsecureType const type)
+{
+   if (_config->FindB("Acquire::AllowInsecureRepositories"))
+      return true;
+
+   if (Target.OptionBool(IndexTarget::ALLOW_INSECURE))
+      return true;
+
+   switch (type)
+   {
+      case InsecureType::UNSIGNED: break;
+      case InsecureType::NORELEASE: break;
+      case InsecureType::WEAK:
+        if (_config->FindB("Acquire::AllowWeakRepositories"))
+           return true;
+        if (Target.OptionBool(IndexTarget::ALLOW_WEAK))
+           return true;
+        break;
+   }
+   return false;
+}
+static bool APT_NONNULL(3, 4, 5) AllowInsecureRepositories(InsecureType const msg, std::string const &repo,
       metaIndex const * const MetaIndexParser, pkgAcqMetaClearSig * const TransactionManager, pkgAcquire::Item * const I)
 {
    // we skip weak downgrades as its unlikely that a repository gets really weaker –
@@ -213,7 +234,8 @@ static bool APT_NONNULL(3, 4, 5) AllowInsecureRepositories(InsecureType msg, std
            case InsecureType::NORELEASE: msgstr = _("The repository '%s' does no longer have a Release file."); break;
            case InsecureType::WEAK: /* unreachable */ break;
         }
-        if (_config->FindB("Acquire::AllowDowngradeToInsecureRepositories"))
+        if (_config->FindB("Acquire::AllowDowngradeToInsecureRepositories") ||
+              TransactionManager->Target.OptionBool(IndexTarget::ALLOW_DOWNGRADE_TO_INSECURE))
         {
            // meh, the users wants to take risks (we still mark the packages
            // from this repository as unauthenticated)
@@ -241,7 +263,7 @@ static bool APT_NONNULL(3, 4, 5) AllowInsecureRepositories(InsecureType msg, std
       case InsecureType::WEAK: msgstr = _("The repository '%s' provides only weak security information."); break;
    }
 
-   if (_config->FindB("Acquire::AllowInsecureRepositories") == true)
+   if (TargetIsAllowedToBe(TransactionManager->Target, msg) == true)
    {
       MessageInsecureRepository(false, msgstr, repo);
       return true;
@@ -277,7 +299,20 @@ APT_CONST bool pkgAcqTransactionItem::HashesRequired() const
       we can at least trust them for integrity of the download itself.
       Only repositories without a Release file can (obviously) not have
       hashes – and they are very uncommon and strongly discouraged */
-   return TransactionManager->MetaIndexParser->GetLoadedSuccessfully() == metaIndex::TRI_YES;
+   if (TransactionManager->MetaIndexParser->GetLoadedSuccessfully() != metaIndex::TRI_YES)
+      return false;
+   if (TargetIsAllowedToBe(Target, InsecureType::WEAK))
+   {
+      /* If we allow weak hashes, we check that we have some (weak) and then
+         declare hashes not needed. That will tip us in the right direction
+        as if hashes exist, they will be used, even if not required */
+      auto const hsl = GetExpectedHashes();
+      if (hsl.usable())
+        return true;
+      if (hsl.empty() == false)
+        return false;
+   }
+   return true;
 }
 HashStringList pkgAcqTransactionItem::GetExpectedHashes() const
 {
@@ -375,7 +410,7 @@ bool pkgAcqTransactionItem::QueueURI(pkgAcquire::ItemDesc &Item)
       return false;
    }
    // If we got the InRelease file via a mirror, pick all indexes directly from this mirror, too
-   if (TransactionManager->BaseURI.empty() == false &&
+   if (TransactionManager->BaseURI.empty() == false && UsedMirror.empty() &&
         URI::SiteOnly(Item.URI) != URI::SiteOnly(TransactionManager->BaseURI))
    {
       // this ensures we rewrite only once and only the first step
@@ -416,7 +451,11 @@ std::string pkgAcquire::Item::GetFinalFilename() const
 }
 std::string pkgAcqDiffIndex::GetFinalFilename() const
 {
-   return GetFinalFileNameFromURI(GetDiffIndexURI(Target));
+   std::string const FinalFile = GetFinalFileNameFromURI(GetDiffIndexURI(Target));
+   // we don't want recompress, so lets keep whatever we got
+   if (CurrentCompressionExtension == "uncompressed")
+      return FinalFile;
+   return FinalFile + "." + CurrentCompressionExtension;
 }
 std::string pkgAcqIndex::GetFinalFilename() const
 {
@@ -453,7 +492,10 @@ std::string pkgAcqIndex::GetMetaKey() const
 }
 std::string pkgAcqDiffIndex::GetMetaKey() const
 {
-   return GetDiffIndexFileName(Target.MetaKey);
+   auto const metakey = GetDiffIndexFileName(Target.MetaKey);
+   if (CurrentCompressionExtension == "uncompressed")
+      return metakey;
+   return metakey + "." + CurrentCompressionExtension;
 }
                                                                        /*}}}*/
 //pkgAcqTransactionItem::TransactionState and specialisations for child classes        /*{{{*/
@@ -650,10 +692,15 @@ class APT_HIDDEN CleanupItem : public pkgAcqTransactionItem               /*{{{*/
                                                                        /*}}}*/
 
 // Acquire::Item::Item - Constructor                                   /*{{{*/
+class pkgAcquire::Item::Private
+{
+public:
+   std::vector<std::string> PastRedirections;
+};
 APT_IGNORE_DEPRECATED_PUSH
 pkgAcquire::Item::Item(pkgAcquire * const owner) :
    FileSize(0), PartialSize(0), Mode(0), ID(0), Complete(false), Local(false),
-    QueueCounter(0), ExpectedAdditionalItems(0), Owner(owner), d(NULL)
+    QueueCounter(0), ExpectedAdditionalItems(0), Owner(owner), d(new Private())
 {
    Owner->Add(this);
    Status = StatIdle;
@@ -664,6 +711,7 @@ APT_IGNORE_DEPRECATED_POP
 pkgAcquire::Item::~Item()
 {
    Owner->Remove(this);
+   delete d;
 }
                                                                        /*}}}*/
 std::string pkgAcquire::Item::Custom600Headers() const                 /*{{{*/
@@ -731,38 +779,51 @@ void pkgAcquire::Item::Failed(string const &Message,pkgAcquire::MethodConfig con
    }
 
    string const FailReason = LookupTag(Message, "FailReason");
-   enum { MAXIMUM_SIZE_EXCEEDED, HASHSUM_MISMATCH, WEAK_HASHSUMS, OTHER } failreason = OTHER;
+   enum { MAXIMUM_SIZE_EXCEEDED, HASHSUM_MISMATCH, WEAK_HASHSUMS, REDIRECTION_LOOP, OTHER } failreason = OTHER;
    if ( FailReason == "MaximumSizeExceeded")
       failreason = MAXIMUM_SIZE_EXCEEDED;
    else if ( FailReason == "WeakHashSums")
       failreason = WEAK_HASHSUMS;
+   else if (FailReason == "RedirectionLoop")
+      failreason = REDIRECTION_LOOP;
    else if (Status == StatAuthError)
       failreason = HASHSUM_MISMATCH;
 
    if(ErrorText.empty())
    {
+      std::ostringstream out;
+      switch (failreason)
+      {
+        case HASHSUM_MISMATCH:
+           out << _("Hash Sum mismatch") << std::endl;
+           break;
+        case WEAK_HASHSUMS:
+           out << _("Insufficient information available to perform this download securely") << std::endl;
+           break;
+        case REDIRECTION_LOOP:
+           out << "Redirection loop encountered" << std::endl;
+           break;
+        case MAXIMUM_SIZE_EXCEEDED:
+           out << LookupTag(Message, "Message") << std::endl;
+           break;
+        case OTHER:
+           out << LookupTag(Message, "Message");
+           break;
+      }
+
       if (Status == StatAuthError)
       {
-        std::ostringstream out;
-        switch (failreason)
-        {
-           case HASHSUM_MISMATCH:
-              out << _("Hash Sum mismatch") << std::endl;
-              break;
-           case WEAK_HASHSUMS:
-              out << _("Insufficient information available to perform this download securely") << std::endl;
-              break;
-           case MAXIMUM_SIZE_EXCEEDED:
-           case OTHER:
-              out << LookupTag(Message, "Message") << std::endl;
-              break;
-        }
         auto const ExpectedHashes = GetExpectedHashes();
         if (ExpectedHashes.empty() == false)
         {
            out << "Hashes of expected file:" << std::endl;
            for (auto const &hs: ExpectedHashes)
-              out << " - " << hs.toStr() << std::endl;
+           {
+              out << " - " << hs.toStr();
+              if (hs.usable() == false)
+                 out << " [weak]";
+              out << std::endl;
+           }
         }
         if (failreason == HASHSUM_MISMATCH)
         {
@@ -772,14 +833,18 @@ void pkgAcquire::Item::Failed(string const &Message,pkgAcquire::MethodConfig con
               std::string const tagname = std::string(*type) + "-Hash";
               std::string const hashsum = LookupTag(Message, tagname.c_str());
               if (hashsum.empty() == false)
-                 out << " - " << HashString(*type, hashsum).toStr() << std::endl;
+              {
+                 auto const hs = HashString(*type, hashsum);
+                 out << " - " << hs.toStr();
+                 if (hs.usable() == false)
+                    out << " [weak]";
+                 out << std::endl;
+              }
            }
            out << "Last modification reported: " << LookupTag(Message, "Last-Modified", "<none>") << std::endl;
         }
-        ErrorText = out.str();
       }
-      else
-        ErrorText = LookupTag(Message,"Message");
+      ErrorText = out.str();
    }
 
    switch (failreason)
@@ -787,6 +852,7 @@ void pkgAcquire::Item::Failed(string const &Message,pkgAcquire::MethodConfig con
       case MAXIMUM_SIZE_EXCEEDED: RenameOnError(MaximumSizeExceeded); break;
       case HASHSUM_MISMATCH: RenameOnError(HashSumMismatch); break;
       case WEAK_HASHSUMS: break;
+      case REDIRECTION_LOOP: break;
       case OTHER: break;
    }
 
@@ -930,6 +996,27 @@ std::string pkgAcquire::Item::HashSum() const                              /*{{{*/
    return hs != NULL ? hs->toStr() : "";
 }
                                                                        /*}}}*/
+bool pkgAcquire::Item::IsRedirectionLoop(std::string const &NewURI)    /*{{{*/
+{
+   // store can fail due to permission errors and the item will "loop" then
+   if (APT::String::Startswith(NewURI, "store:"))
+      return false;
+   if (d->PastRedirections.empty())
+   {
+      d->PastRedirections.push_back(NewURI);
+      return false;
+   }
+   auto const LastURI = std::prev(d->PastRedirections.end());
+   // redirections to the same file are a way of restarting/resheduling,
+   // individual methods will have to make sure that they aren't looping this way
+   if (*LastURI == NewURI)
+      return false;
+   if (std::find(d->PastRedirections.begin(), LastURI, NewURI) != LastURI)
+      return true;
+   d->PastRedirections.push_back(NewURI);
+   return false;
+}
+                                                                       /*}}}*/
 
 pkgAcqTransactionItem::pkgAcqTransactionItem(pkgAcquire * const Owner, /*{{{*/
       pkgAcqMetaClearSig * const transactionManager, IndexTarget const &target) :
@@ -1121,7 +1208,7 @@ string pkgAcqMetaBase::Custom600Headers() const
    string const FinalFile = GetFinalFilename();
    struct stat Buf;
    if (stat(FinalFile.c_str(),&Buf) == 0)
-      Header += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
+      Header += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime, false);
 
    return Header;
 }
@@ -1146,9 +1233,15 @@ bool pkgAcqMetaBase::CheckDownloadDone(pkgAcqTransactionItem * const I, const st
    if (I->UsedMirror.empty() == false && _config->FindB("Acquire::SameMirrorForAllIndexes", true))
    {
       if (APT::String::Endswith(I->Desc.URI, "InRelease"))
+      {
         TransactionManager->BaseURI = I->Desc.URI.substr(0, I->Desc.URI.length() - strlen("InRelease"));
+        TransactionManager->UsedMirror = I->UsedMirror;
+      }
       else if (APT::String::Endswith(I->Desc.URI, "Release"))
+      {
         TransactionManager->BaseURI = I->Desc.URI.substr(0, I->Desc.URI.length() - strlen("Release"));
+        TransactionManager->UsedMirror = I->UsedMirror;
+      }
    }
 
    std::string const FileName = LookupTag(Message,"Filename");
@@ -1265,8 +1358,7 @@ void pkgAcqMetaClearSig::QueueIndexes(bool const verify)                  /*{{{*/
       // than invent an entirely new flag we would need to carry for all of eternity.
       if (hasReleaseFile && Target.Option(IndexTarget::ARCHITECTURE) == "all")
       {
-        if (TransactionManager->MetaIndexParser->IsArchitectureSupported("all") == false ||
-              TransactionManager->MetaIndexParser->IsArchitectureAllSupportedFor(Target) == false)
+        if (TransactionManager->MetaIndexParser->IsArchitectureAllSupportedFor(Target) == false)
         {
            new CleanupItem(Owner, TransactionManager, Target);
            continue;
@@ -1322,7 +1414,7 @@ void pkgAcqMetaClearSig::QueueIndexes(bool const verify)                  /*{{{*/
            auto const hashes = GetExpectedHashesFor(Target.MetaKey);
            if (hashes.empty() == false)
            {
-              if (hashes.usable() == false)
+              if (hashes.usable() == false && TargetIsAllowedToBe(TransactionManager->Target, InsecureType::WEAK) == false)
               {
                  new CleanupItem(Owner, TransactionManager, Target);
                  _error->Warning(_("Skipping acquire of configured file '%s' as repository '%s' provides only weak security information for it"),
@@ -1514,8 +1606,7 @@ pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire * const Owner,  /*{{{*/
       IndexTarget const &DetachedDataTarget, IndexTarget const &DetachedSigTarget,
       metaIndex * const MetaIndexParser) :
    pkgAcqMetaIndex(Owner, this, ClearsignedTarget, DetachedSigTarget),
-   d(NULL), ClearsignedTarget(ClearsignedTarget),
-   DetachedDataTarget(DetachedDataTarget),
+   d(NULL), DetachedDataTarget(DetachedDataTarget),
    MetaIndexParser(MetaIndexParser), LastMetaIndexParser(NULL)
 {
    // index targets + (worst case:) Release/Release.gpg
@@ -1629,7 +1720,7 @@ void pkgAcqMetaClearSig::Failed(string const &Message,pkgAcquire::MethodConfig c
       if(CheckStopAuthentication(this, Message))
          return;
 
-      if(AllowInsecureRepositories(InsecureType::UNSIGNED, ClearsignedTarget.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true)
+      if(AllowInsecureRepositories(InsecureType::UNSIGNED, Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true)
       {
         Status = StatDone;
 
@@ -1871,7 +1962,7 @@ void pkgAcqBaseIndex::Failed(std::string const &Message,pkgAcquire::MethodConfig
    if (timespec == 0)
       ErrorText.append("<unknown>");
    else
-      ErrorText.append(TimeRFC1123(timespec));
+      ErrorText.append(TimeRFC1123(timespec, true));
    ErrorText.append("\n");
 }
                                                                        /*}}}*/
@@ -1887,44 +1978,33 @@ pkgAcqBaseIndex::~pkgAcqBaseIndex() {}
 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire * const Owner,
                                  pkgAcqMetaClearSig * const TransactionManager,
                                  IndexTarget const &Target)
-   : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL), diffs(NULL)
+   : pkgAcqIndex(Owner, TransactionManager, Target, true), d(NULL), diffs(NULL)
 {
    // FIXME: Magic number as an upper bound on pdiffs we will reasonably acquire
    ExpectedAdditionalItems = 40;
-
    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 
-   Desc.Owner = this;
-   Desc.Description = GetDiffIndexFileName(Target.Description);
-   Desc.ShortDesc = Target.ShortDesc;
-   Desc.URI = GetDiffIndexURI(Target);
-
-   DestFile = GetPartialFileNameFromURI(Desc.URI);
+   CompressionExtensions.clear();
+   {
+      std::vector<std::string> types = APT::Configuration::getCompressionTypes();
+      if (types.empty() == false)
+      {
+        std::ostringstream os;
+        std::copy_if(types.begin(), types.end()-1, std::ostream_iterator<std::string>(os, " "), [&](std::string const type) {
+              if (type == "uncompressed")
+                 return true;
+              return TransactionManager->MetaIndexParser->Exists(GetDiffIndexFileName(Target.MetaKey) + '.' + type);
+        });
+        os << *types.rbegin();
+        CompressionExtensions = os.str();
+      }
+   }
+   if (Target.Option(IndexTarget::COMPRESSIONTYPES).find("by-hash") != std::string::npos)
+      CompressionExtensions = "by-hash " + CompressionExtensions;
+   Init(GetDiffIndexURI(Target), GetDiffIndexFileName(Target.Description), Target.ShortDesc);
 
    if(Debug)
       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
-
-   QueueURI(Desc);
-}
-                                                                       /*}}}*/
-// AcqIndex::Custom600Headers - Insert custom request headers          /*{{{*/
-// ---------------------------------------------------------------------
-/* The only header we use is the last-modified header. */
-string pkgAcqDiffIndex::Custom600Headers() const
-{
-   if (TransactionManager->LastMetaIndexParser != NULL)
-      return "\nIndex-File: true";
-
-   string const Final = GetFinalFilename();
-
-   if(Debug)
-      std::clog << "Custom600Header-IMS: " << Final << std::endl;
-
-   struct stat Buf;
-   if (stat(Final.c_str(),&Buf) != 0)
-      return "\nIndex-File: true";
-   
-   return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 }
                                                                        /*}}}*/
 void pkgAcqDiffIndex::QueueOnIMSHit() const                            /*{{{*/
@@ -1934,6 +2014,18 @@ void pkgAcqDiffIndex::QueueOnIMSHit() const                              /*{{{*/
    new pkgAcqIndexDiffs(Owner, TransactionManager, Target);
 }
                                                                        /*}}}*/
+static bool RemoveFileForBootstrapLinking(bool const Debug, std::string const &For, std::string const &Boot)/*{{{*/
+{
+   if (FileExists(Boot) && RemoveFile("Bootstrap-linking", Boot) == false)
+   {
+      if (Debug)
+        std::clog << "Bootstrap-linking for patching " << For
+           << " by removing stale " << Boot << " failed!" << std::endl;
+      return false;
+   }
+   return true;
+}
+                                                                       /*}}}*/
 bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)      /*{{{*/
 {
    ExpectedAdditionalItems = 0;
@@ -1943,7 +2035,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
       std::clog << "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile
         << std::endl;
 
-   FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
+   FileFd Fd(IndexDiffFile, FileFd::ReadOnly, FileFd::Extension);
    pkgTagFile TF(&Fd);
    if (Fd.IsOpen() == false || Fd.Failed())
       return false;
@@ -1955,6 +2047,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
    HashStringList ServerHashes;
    unsigned long long ServerSize = 0;
 
+   auto const &posix = std::locale("C.UTF-8");
    for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
    {
       std::string tagname = *type;
@@ -1966,6 +2059,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
       string hash;
       unsigned long long size;
       std::stringstream ss(tmp);
+      ss.imbue(posix);
       ss >> hash >> size;
       if (unlikely(hash.empty() == true))
         continue;
@@ -2044,6 +2138,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
       string hash, filename;
       unsigned long long size;
       std::stringstream ss(tmp);
+      ss.imbue(posix);
 
       while (ss >> hash >> size >> filename)
       {
@@ -2102,6 +2197,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
       string hash, filename;
       unsigned long long size;
       std::stringstream ss(tmp);
+      ss.imbue(posix);
 
       while (ss >> hash >> size >> filename)
       {
@@ -2139,6 +2235,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
       string hash, filename;
       unsigned long long size;
       std::stringstream ss(tmp);
+      ss.imbue(posix);
 
       // FIXME: all of pdiff supports only .gz compressed patches
       while (ss >> hash >> size >> filename)
@@ -2269,23 +2366,15 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)       /*{{{*/
       if (unlikely(Final.empty())) // because we wouldn't be called in such a case
         return false;
       std::string const PartialFile = GetPartialFileNameFromURI(Target.URI);
-      if (FileExists(PartialFile) && RemoveFile("Bootstrap-linking", PartialFile) == false)
-      {
-        if (Debug)
-           std::clog << "Bootstrap-linking for patching " << CurrentPackagesFile
-              << " by removing stale " << PartialFile << " failed!" << std::endl;
+      std::string const PatchedFile = GetKeepCompressedFileName(PartialFile + "-patched", Target);
+      if (RemoveFileForBootstrapLinking(Debug, CurrentPackagesFile, PartialFile) == false ||
+           RemoveFileForBootstrapLinking(Debug, CurrentPackagesFile, PatchedFile) == false)
         return false;
-      }
       for (auto const &ext : APT::Configuration::getCompressorExtensions())
       {
-        std::string const Partial = PartialFile + ext;
-        if (FileExists(Partial) && RemoveFile("Bootstrap-linking", Partial) == false)
-        {
-           if (Debug)
-              std::clog << "Bootstrap-linking for patching " << CurrentPackagesFile
-                 << " by removing stale " << Partial << " failed!" << std::endl;
+        if (RemoveFileForBootstrapLinking(Debug, CurrentPackagesFile, PartialFile + ext) == false ||
+              RemoveFileForBootstrapLinking(Debug, CurrentPackagesFile, PatchedFile + ext) == false)
            return false;
-        }
       }
       std::string const Ext = Final.substr(CurrentPackagesFile.length());
       std::string const Partial = PartialFile + Ext;
@@ -2318,7 +2407,9 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
                                                                        /*}}}*/
 void pkgAcqDiffIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
 {
-   pkgAcqBaseIndex::Failed(Message,Cnf);
+   if (CommonFailed(GetDiffIndexURI(Target), GetDiffIndexFileName(Target.Description), Message, Cnf))
+      return;
+
    Status = StatDone;
    ExpectedAdditionalItems = 0;
 
@@ -2567,9 +2658,10 @@ std::string pkgAcqIndexDiffs::Custom600Headers() const                   /*{{{*/
    if(State != StateApplyDiff)
       return pkgAcqBaseIndex::Custom600Headers();
    std::ostringstream patchhashes;
-   HashStringList const ExpectedHashes = available_patches[0].patch_hashes;
-   for (HashStringList::const_iterator hs = ExpectedHashes.begin(); hs != ExpectedHashes.end(); ++hs)
-      patchhashes <<  "\nPatch-0-" << hs->HashType() << "-Hash: " << hs->HashValue();
+   for (auto && hs : available_patches[0].result_hashes)
+      patchhashes <<  "\nStart-" << hs.HashType() << "-Hash: " << hs.HashValue();
+   for (auto && hs : available_patches[0].patch_hashes)
+      patchhashes <<  "\nPatch-0-" << hs.HashType() << "-Hash: " << hs.HashValue();
    patchhashes << pkgAcqBaseIndex::Custom600Headers();
    return patchhashes.str();
 }
@@ -2716,12 +2808,14 @@ std::string pkgAcqIndexMergeDiffs::Custom600Headers() const             /*{{{*/
       return pkgAcqBaseIndex::Custom600Headers();
    std::ostringstream patchhashes;
    unsigned int seen_patches = 0;
+   for (auto && hs : (*allPatches)[0]->patch.result_hashes)
+      patchhashes <<  "\nStart-" << hs.HashType() << "-Hash: " << hs.HashValue();
    for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
         I != allPatches->end(); ++I)
    {
       HashStringList const ExpectedHashes = (*I)->patch.patch_hashes;
       for (HashStringList::const_iterator hs = ExpectedHashes.begin(); hs != ExpectedHashes.end(); ++hs)
-        patchhashes <<  "\nPatch-" << seen_patches << "-" << hs->HashType() << "-Hash: " << hs->HashValue();
+        patchhashes <<  "\nPatch-" << std::to_string(seen_patches) << "-" << hs->HashType() << "-Hash: " << hs->HashValue();
       ++seen_patches;
    }
    patchhashes << pkgAcqBaseIndex::Custom600Headers();
@@ -2733,10 +2827,12 @@ pkgAcqIndexMergeDiffs::~pkgAcqIndexMergeDiffs() {}
 // AcqIndex::AcqIndex - Constructor                                    /*{{{*/
 pkgAcqIndex::pkgAcqIndex(pkgAcquire * const Owner,
                          pkgAcqMetaClearSig * const TransactionManager,
-                         IndexTarget const &Target)
+                         IndexTarget const &Target, bool const Derived)
    : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL), Stage(STAGE_DOWNLOAD),
    CompressionExtensions(Target.Option(IndexTarget::COMPRESSIONTYPES))
 {
+   if (Derived)
+      return;
    Init(Target.URI, Target.Description, Target.ShortDesc);
 
    if(_config->FindB("Debug::Acquire::Transaction", false) == true)
@@ -2769,10 +2865,6 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc,
    DestFile = GetPartialFileNameFromURI(URI);
    NextCompressionExtension(CurrentCompressionExtension, CompressionExtensions, false);
 
-   // store file size of the download to ensure the fetcher gives
-   // accurate progress reporting
-   FileSize = GetExpectedHashes().FileSize();
-
    if (CurrentCompressionExtension == "uncompressed")
    {
       Desc.URI = URI;
@@ -2787,6 +2879,8 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc,
         Desc.URI = URI + '.' + CurrentCompressionExtension;
         DestFile = DestFile + '.' + CurrentCompressionExtension;
       }
+      else
+        Desc.URI = URI;
 
       HashStringList const Hashes = GetExpectedHashes();
       HashString const * const TargetHash = Hashes.find(NULL);
@@ -2809,6 +2903,9 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc,
       DestFile = DestFile + '.' + CurrentCompressionExtension;
    }
 
+   // store file size of the download to ensure the fetcher gives
+   // accurate progress reporting
+   FileSize = GetExpectedHashes().FileSize();
 
    Desc.Description = URIDesc;
    Desc.Owner = this;
@@ -2831,7 +2928,7 @@ string pkgAcqIndex::Custom600Headers() const
 
       struct stat Buf;
       if (stat(Final.c_str(),&Buf) == 0)
-        msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
+        msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime, false);
    }
 
    if(Target.IsOptional)
@@ -2841,20 +2938,40 @@ string pkgAcqIndex::Custom600Headers() const
 }
                                                                        /*}}}*/
 // AcqIndex::Failed - getting the indexfile failed                     /*{{{*/
-void pkgAcqIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)
+bool pkgAcqIndex::CommonFailed(std::string const &TargetURI, std::string const TargetDesc,
+      std::string const &Message, pkgAcquire::MethodConfig const * const Cnf)
 {
    pkgAcqBaseIndex::Failed(Message,Cnf);
 
+   if (UsedMirror.empty() == false && UsedMirror != "DIRECT" &&
+        LookupTag(Message, "FailReason") == "HttpError404")
+   {
+      UsedMirror = "DIRECT";
+      if (Desc.URI.find("/by-hash/") != std::string::npos)
+        CompressionExtensions = "by-hash " + CompressionExtensions;
+      else
+        CompressionExtensions = CurrentCompressionExtension + ' ' + CompressionExtensions;
+      Init(TargetURI, TargetDesc, Desc.ShortDesc);
+      Status = StatIdle;
+      return true;
+   }
+
    // authorisation matches will not be fixed by other compression types
    if (Status != StatAuthError)
    {
       if (CompressionExtensions.empty() == false)
       {
-        Init(Target.URI, Desc.Description, Desc.ShortDesc);
+        Init(TargetURI, Desc.Description, Desc.ShortDesc);
         Status = StatIdle;
-        return;
+        return true;
       }
    }
+   return false;
+}
+void pkgAcqIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)
+{
+   if (CommonFailed(Target.URI, Target.Description, Message, Cnf))
+      return;
 
    if(Target.IsOptional && GetExpectedHashes().empty() && Stage == STAGE_DOWNLOAD)
       Status = StatDone;