ensure all removes are reported to hook scripts

[apt.git] / apt-pkg / acquire-item.cc

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index 8628679322dc91a11f615da825212b1ca13328ef..ad8cb7f245795298c4c5a54d361b86012881e534 100644 (file)
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -175,7 +175,7 @@ static void ReportMirrorFailureToCentral(pkgAcquire::Item const &I, std::string
 }
                                                                        /*}}}*/
 
 }
                                                                        /*}}}*/
 

-static bool MessageInsecureRepository(bool const isError, char const * const msg, std::string const &repo)/*{{{*/
+static APT_NONNULL(2) bool MessageInsecureRepository(bool const isError, char const * const msg, std::string const &repo)/*{{{*/

 {
    std::string m;
    strprintf(m, msg, repo.c_str());
 {
    std::string m;
    strprintf(m, msg, repo.c_str());


@@ -195,7 +195,28 @@ static bool MessageInsecureRepository(bool const isError, char const * const msg
                                                                        /*}}}*/
 // AllowInsecureRepositories                                           /*{{{*/
 enum class InsecureType { UNSIGNED, WEAK, NORELEASE };
                                                                        /*}}}*/
 // AllowInsecureRepositories                                           /*{{{*/
 enum class InsecureType { UNSIGNED, WEAK, NORELEASE };

-static bool APT_NONNULL(3, 4, 5) AllowInsecureRepositories(InsecureType msg, std::string const &repo,
+static bool TargetIsAllowedToBe(IndexTarget const &Target, InsecureType const type)
+{
+   if (_config->FindB("Acquire::AllowInsecureRepositories"))
+      return true;
+
+   if (Target.OptionBool(IndexTarget::ALLOW_INSECURE))
+      return true;
+
+   switch (type)
+   {
+      case InsecureType::UNSIGNED: break;
+      case InsecureType::NORELEASE: break;
+      case InsecureType::WEAK:
+        if (_config->FindB("Acquire::AllowWeakRepositories"))
+           return true;
+        if (Target.OptionBool(IndexTarget::ALLOW_WEAK))
+           return true;
+        break;
+   }
+   return false;
+}
+static bool APT_NONNULL(3, 4, 5) AllowInsecureRepositories(InsecureType const msg, std::string const &repo,

       metaIndex const * const MetaIndexParser, pkgAcqMetaClearSig * const TransactionManager, pkgAcquire::Item * const I)
 {
    // we skip weak downgrades as its unlikely that a repository gets really weaker –
       metaIndex const * const MetaIndexParser, pkgAcqMetaClearSig * const TransactionManager, pkgAcquire::Item * const I)
 {
    // we skip weak downgrades as its unlikely that a repository gets really weaker –


@@ -213,7 +234,8 @@ static bool APT_NONNULL(3, 4, 5) AllowInsecureRepositories(InsecureType msg, std
            case InsecureType::NORELEASE: msgstr = _("The repository '%s' does no longer have a Release file."); break;
            case InsecureType::WEAK: /* unreachable */ break;
         }
            case InsecureType::NORELEASE: msgstr = _("The repository '%s' does no longer have a Release file."); break;
            case InsecureType::WEAK: /* unreachable */ break;
         }

-        if (_config->FindB("Acquire::AllowDowngradeToInsecureRepositories"))
+        if (_config->FindB("Acquire::AllowDowngradeToInsecureRepositories") ||
+              TransactionManager->Target.OptionBool(IndexTarget::ALLOW_DOWNGRADE_TO_INSECURE))

         {
            // meh, the users wants to take risks (we still mark the packages
            // from this repository as unauthenticated)
         {
            // meh, the users wants to take risks (we still mark the packages
            // from this repository as unauthenticated)


@@ -241,7 +263,7 @@ static bool APT_NONNULL(3, 4, 5) AllowInsecureRepositories(InsecureType msg, std
       case InsecureType::WEAK: msgstr = _("The repository '%s' provides only weak security information."); break;
    }
 
       case InsecureType::WEAK: msgstr = _("The repository '%s' provides only weak security information."); break;
    }
 

-   if (_config->FindB("Acquire::AllowInsecureRepositories") == true)
+   if (TargetIsAllowedToBe(TransactionManager->Target, msg) == true)

    {
       MessageInsecureRepository(false, msgstr, repo);
       return true;
    {
       MessageInsecureRepository(false, msgstr, repo);
       return true;


@@ -277,7 +299,20 @@ APT_CONST bool pkgAcqTransactionItem::HashesRequired() const
       we can at least trust them for integrity of the download itself.
       Only repositories without a Release file can (obviously) not have
       hashes – and they are very uncommon and strongly discouraged */
       we can at least trust them for integrity of the download itself.
       Only repositories without a Release file can (obviously) not have
       hashes – and they are very uncommon and strongly discouraged */

-   return TransactionManager->MetaIndexParser->GetLoadedSuccessfully() == metaIndex::TRI_YES;
+   if (TransactionManager->MetaIndexParser->GetLoadedSuccessfully() != metaIndex::TRI_YES)
+      return false;
+   if (TargetIsAllowedToBe(Target, InsecureType::WEAK))
+   {
+      /* If we allow weak hashes, we check that we have some (weak) and then
+         declare hashes not needed. That will tip us in the right direction
+        as if hashes exist, they will be used, even if not required */
+      auto const hsl = GetExpectedHashes();
+      if (hsl.usable())
+        return true;
+      if (hsl.empty() == false)
+        return false;
+   }
+   return true;

 }
 HashStringList pkgAcqTransactionItem::GetExpectedHashes() const
 {
 }
 HashStringList pkgAcqTransactionItem::GetExpectedHashes() const
 {


@@ -762,7 +797,12 @@ void pkgAcquire::Item::Failed(string const &Message,pkgAcquire::MethodConfig con
         {
            out << "Hashes of expected file:" << std::endl;
            for (auto const &hs: ExpectedHashes)
         {
            out << "Hashes of expected file:" << std::endl;
            for (auto const &hs: ExpectedHashes)

-              out << " - " << hs.toStr() << std::endl;
+           {
+              out << " - " << hs.toStr();
+              if (hs.usable() == false)
+                 out << " [weak]";
+              out << std::endl;
+           }

         }
         if (failreason == HASHSUM_MISMATCH)
         {
         }
         if (failreason == HASHSUM_MISMATCH)
         {


@@ -772,7 +812,13 @@ void pkgAcquire::Item::Failed(string const &Message,pkgAcquire::MethodConfig con
               std::string const tagname = std::string(*type) + "-Hash";
               std::string const hashsum = LookupTag(Message, tagname.c_str());
               if (hashsum.empty() == false)
               std::string const tagname = std::string(*type) + "-Hash";
               std::string const hashsum = LookupTag(Message, tagname.c_str());
               if (hashsum.empty() == false)

-                 out << " - " << HashString(*type, hashsum).toStr() << std::endl;
+              {
+                 auto const hs = HashString(*type, hashsum);
+                 out << " - " << hs.toStr();
+                 if (hs.usable() == false)
+                    out << " [weak]";
+                 out << std::endl;
+              }

            }
            out << "Last modification reported: " << LookupTag(Message, "Last-Modified", "<none>") << std::endl;
         }
            }
            out << "Last modification reported: " << LookupTag(Message, "Last-Modified", "<none>") << std::endl;
         }


@@ -1121,7 +1167,7 @@ string pkgAcqMetaBase::Custom600Headers() const
    string const FinalFile = GetFinalFilename();
    struct stat Buf;
    if (stat(FinalFile.c_str(),&Buf) == 0)
    string const FinalFile = GetFinalFilename();
    struct stat Buf;
    if (stat(FinalFile.c_str(),&Buf) == 0)

-      Header += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
+      Header += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime, false);

 
    return Header;
 }
 
    return Header;
 }


@@ -1265,8 +1311,7 @@ void pkgAcqMetaClearSig::QueueIndexes(bool const verify)                  /*{{{*/
       // than invent an entirely new flag we would need to carry for all of eternity.
       if (hasReleaseFile && Target.Option(IndexTarget::ARCHITECTURE) == "all")
       {
       // than invent an entirely new flag we would need to carry for all of eternity.
       if (hasReleaseFile && Target.Option(IndexTarget::ARCHITECTURE) == "all")
       {

-        if (TransactionManager->MetaIndexParser->IsArchitectureSupported("all") == false ||
-              TransactionManager->MetaIndexParser->IsArchitectureAllSupportedFor(Target) == false)
+        if (TransactionManager->MetaIndexParser->IsArchitectureAllSupportedFor(Target) == false)

         {
            new CleanupItem(Owner, TransactionManager, Target);
            continue;
         {
            new CleanupItem(Owner, TransactionManager, Target);
            continue;


@@ -1322,7 +1367,7 @@ void pkgAcqMetaClearSig::QueueIndexes(bool const verify)                  /*{{{*/
            auto const hashes = GetExpectedHashesFor(Target.MetaKey);
            if (hashes.empty() == false)
            {
            auto const hashes = GetExpectedHashesFor(Target.MetaKey);
            if (hashes.empty() == false)
            {

-              if (hashes.usable() == false)
+              if (hashes.usable() == false && TargetIsAllowedToBe(TransactionManager->Target, InsecureType::WEAK) == false)

               {
                  new CleanupItem(Owner, TransactionManager, Target);
                  _error->Warning(_("Skipping acquire of configured file '%s' as repository '%s' provides only weak security information for it"),
               {
                  new CleanupItem(Owner, TransactionManager, Target);
                  _error->Warning(_("Skipping acquire of configured file '%s' as repository '%s' provides only weak security information for it"),


@@ -1514,8 +1559,7 @@ pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire * const Owner,  /*{{{*/
       IndexTarget const &DetachedDataTarget, IndexTarget const &DetachedSigTarget,
       metaIndex * const MetaIndexParser) :
    pkgAcqMetaIndex(Owner, this, ClearsignedTarget, DetachedSigTarget),
       IndexTarget const &DetachedDataTarget, IndexTarget const &DetachedSigTarget,
       metaIndex * const MetaIndexParser) :
    pkgAcqMetaIndex(Owner, this, ClearsignedTarget, DetachedSigTarget),

-   d(NULL), ClearsignedTarget(ClearsignedTarget),
-   DetachedDataTarget(DetachedDataTarget),
+   d(NULL), DetachedDataTarget(DetachedDataTarget),

    MetaIndexParser(MetaIndexParser), LastMetaIndexParser(NULL)
 {
    // index targets + (worst case:) Release/Release.gpg
    MetaIndexParser(MetaIndexParser), LastMetaIndexParser(NULL)
 {
    // index targets + (worst case:) Release/Release.gpg


@@ -1629,7 +1673,7 @@ void pkgAcqMetaClearSig::Failed(string const &Message,pkgAcquire::MethodConfig c
       if(CheckStopAuthentication(this, Message))
          return;
 
       if(CheckStopAuthentication(this, Message))
          return;
 

-      if(AllowInsecureRepositories(InsecureType::UNSIGNED, ClearsignedTarget.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true)
+      if(AllowInsecureRepositories(InsecureType::UNSIGNED, Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true)

       {
         Status = StatDone;
 
       {
         Status = StatDone;
 


@@ -1871,7 +1915,7 @@ void pkgAcqBaseIndex::Failed(std::string const &Message,pkgAcquire::MethodConfig
    if (timespec == 0)
       ErrorText.append("<unknown>");
    else
    if (timespec == 0)
       ErrorText.append("<unknown>");
    else

-      ErrorText.append(TimeRFC1123(timespec));
+      ErrorText.append(TimeRFC1123(timespec, true));

    ErrorText.append("\n");
 }
                                                                        /*}}}*/
    ErrorText.append("\n");
 }
                                                                        /*}}}*/


@@ -1924,7 +1968,7 @@ string pkgAcqDiffIndex::Custom600Headers() const
    if (stat(Final.c_str(),&Buf) != 0)
       return "\nIndex-File: true";
    
    if (stat(Final.c_str(),&Buf) != 0)
       return "\nIndex-File: true";
    

-   return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
+   return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime, false);

 }
                                                                        /*}}}*/
 void pkgAcqDiffIndex::QueueOnIMSHit() const                            /*{{{*/
 }
                                                                        /*}}}*/
 void pkgAcqDiffIndex::QueueOnIMSHit() const                            /*{{{*/


@@ -1934,6 +1978,18 @@ void pkgAcqDiffIndex::QueueOnIMSHit() const                              /*{{{*/
    new pkgAcqIndexDiffs(Owner, TransactionManager, Target);
 }
                                                                        /*}}}*/
    new pkgAcqIndexDiffs(Owner, TransactionManager, Target);
 }
                                                                        /*}}}*/

+static bool RemoveFileForBootstrapLinking(bool const Debug, std::string const &For, std::string const &Boot)/*{{{*/
+{
+   if (FileExists(Boot) && RemoveFile("Bootstrap-linking", Boot) == false)
+   {
+      if (Debug)
+        std::clog << "Bootstrap-linking for patching " << For
+           << " by removing stale " << Boot << " failed!" << std::endl;
+      return false;
+   }
+   return true;
+}
+                                                                       /*}}}*/

 bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)      /*{{{*/
 {
    ExpectedAdditionalItems = 0;
 bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)      /*{{{*/
 {
    ExpectedAdditionalItems = 0;


@@ -1955,6 +2011,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
    HashStringList ServerHashes;
    unsigned long long ServerSize = 0;
 
    HashStringList ServerHashes;
    unsigned long long ServerSize = 0;
 

+   auto const &posix = std::locale("C.UTF-8");

    for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
    {
       std::string tagname = *type;
    for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
    {
       std::string tagname = *type;


@@ -1966,6 +2023,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
       string hash;
       unsigned long long size;
       std::stringstream ss(tmp);
       string hash;
       unsigned long long size;
       std::stringstream ss(tmp);

+      ss.imbue(posix);

       ss >> hash >> size;
       if (unlikely(hash.empty() == true))
         continue;
       ss >> hash >> size;
       if (unlikely(hash.empty() == true))
         continue;


@@ -2044,6 +2102,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
       string hash, filename;
       unsigned long long size;
       std::stringstream ss(tmp);
       string hash, filename;
       unsigned long long size;
       std::stringstream ss(tmp);

+      ss.imbue(posix);

 
       while (ss >> hash >> size >> filename)
       {
 
       while (ss >> hash >> size >> filename)
       {


@@ -2102,6 +2161,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
       string hash, filename;
       unsigned long long size;
       std::stringstream ss(tmp);
       string hash, filename;
       unsigned long long size;
       std::stringstream ss(tmp);

+      ss.imbue(posix);

 
       while (ss >> hash >> size >> filename)
       {
 
       while (ss >> hash >> size >> filename)
       {


@@ -2139,6 +2199,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
       string hash, filename;
       unsigned long long size;
       std::stringstream ss(tmp);
       string hash, filename;
       unsigned long long size;
       std::stringstream ss(tmp);

+      ss.imbue(posix);

 
       // FIXME: all of pdiff supports only .gz compressed patches
       while (ss >> hash >> size >> filename)
 
       // FIXME: all of pdiff supports only .gz compressed patches
       while (ss >> hash >> size >> filename)


@@ -2269,23 +2330,15 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)       /*{{{*/
       if (unlikely(Final.empty())) // because we wouldn't be called in such a case
         return false;
       std::string const PartialFile = GetPartialFileNameFromURI(Target.URI);
       if (unlikely(Final.empty())) // because we wouldn't be called in such a case
         return false;
       std::string const PartialFile = GetPartialFileNameFromURI(Target.URI);

-      if (FileExists(PartialFile) && RemoveFile("Bootstrap-linking", PartialFile) == false)
-      {
-        if (Debug)
-           std::clog << "Bootstrap-linking for patching " << CurrentPackagesFile
-              << " by removing stale " << PartialFile << " failed!" << std::endl;
+      std::string const PatchedFile = GetKeepCompressedFileName(PartialFile + "-patched", Target);
+      if (RemoveFileForBootstrapLinking(Debug, CurrentPackagesFile, PartialFile) == false ||
+           RemoveFileForBootstrapLinking(Debug, CurrentPackagesFile, PatchedFile) == false)

         return false;
         return false;

-      }

       for (auto const &ext : APT::Configuration::getCompressorExtensions())
       {
       for (auto const &ext : APT::Configuration::getCompressorExtensions())
       {

-        std::string const Partial = PartialFile + ext;
-        if (FileExists(Partial) && RemoveFile("Bootstrap-linking", Partial) == false)
-        {
-           if (Debug)
-              std::clog << "Bootstrap-linking for patching " << CurrentPackagesFile
-                 << " by removing stale " << Partial << " failed!" << std::endl;
+        if (RemoveFileForBootstrapLinking(Debug, CurrentPackagesFile, PartialFile + ext) == false ||
+              RemoveFileForBootstrapLinking(Debug, CurrentPackagesFile, PatchedFile + ext) == false)

            return false;
            return false;

-        }

       }
       std::string const Ext = Final.substr(CurrentPackagesFile.length());
       std::string const Partial = PartialFile + Ext;
       }
       std::string const Ext = Final.substr(CurrentPackagesFile.length());
       std::string const Partial = PartialFile + Ext;


@@ -2567,9 +2620,10 @@ std::string pkgAcqIndexDiffs::Custom600Headers() const                   /*{{{*/
    if(State != StateApplyDiff)
       return pkgAcqBaseIndex::Custom600Headers();
    std::ostringstream patchhashes;
    if(State != StateApplyDiff)
       return pkgAcqBaseIndex::Custom600Headers();
    std::ostringstream patchhashes;

-   HashStringList const ExpectedHashes = available_patches[0].patch_hashes;
-   for (HashStringList::const_iterator hs = ExpectedHashes.begin(); hs != ExpectedHashes.end(); ++hs)
-      patchhashes <<  "\nPatch-0-" << hs->HashType() << "-Hash: " << hs->HashValue();
+   for (auto && hs : available_patches[0].result_hashes)
+      patchhashes <<  "\nStart-" << hs.HashType() << "-Hash: " << hs.HashValue();
+   for (auto && hs : available_patches[0].patch_hashes)
+      patchhashes <<  "\nPatch-0-" << hs.HashType() << "-Hash: " << hs.HashValue();

    patchhashes << pkgAcqBaseIndex::Custom600Headers();
    return patchhashes.str();
 }
    patchhashes << pkgAcqBaseIndex::Custom600Headers();
    return patchhashes.str();
 }


@@ -2716,12 +2770,14 @@ std::string pkgAcqIndexMergeDiffs::Custom600Headers() const             /*{{{*/
       return pkgAcqBaseIndex::Custom600Headers();
    std::ostringstream patchhashes;
    unsigned int seen_patches = 0;
       return pkgAcqBaseIndex::Custom600Headers();
    std::ostringstream patchhashes;
    unsigned int seen_patches = 0;

+   for (auto && hs : (*allPatches)[0]->patch.result_hashes)
+      patchhashes <<  "\nStart-" << hs.HashType() << "-Hash: " << hs.HashValue();

    for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
         I != allPatches->end(); ++I)
    {
       HashStringList const ExpectedHashes = (*I)->patch.patch_hashes;
       for (HashStringList::const_iterator hs = ExpectedHashes.begin(); hs != ExpectedHashes.end(); ++hs)
    for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
         I != allPatches->end(); ++I)
    {
       HashStringList const ExpectedHashes = (*I)->patch.patch_hashes;
       for (HashStringList::const_iterator hs = ExpectedHashes.begin(); hs != ExpectedHashes.end(); ++hs)

-        patchhashes <<  "\nPatch-" << seen_patches << "-" << hs->HashType() << "-Hash: " << hs->HashValue();
+        patchhashes <<  "\nPatch-" << std::to_string(seen_patches) << "-" << hs->HashType() << "-Hash: " << hs->HashValue();

       ++seen_patches;
    }
    patchhashes << pkgAcqBaseIndex::Custom600Headers();
       ++seen_patches;
    }
    patchhashes << pkgAcqBaseIndex::Custom600Headers();


@@ -2831,7 +2887,7 @@ string pkgAcqIndex::Custom600Headers() const
 
       struct stat Buf;
       if (stat(Final.c_str(),&Buf) == 0)
 
       struct stat Buf;
       if (stat(Final.c_str(),&Buf) == 0)

-        msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
+        msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime, false);

    }
 
    if(Target.IsOptional)
    }
 
    if(Target.IsOptional)