apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <config.h>
  17
  18 #include <apt-pkg/acquire-item.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/aptconfiguration.h>
  21 #include <apt-pkg/sourcelist.h>
  22 #include <apt-pkg/error.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/sha1.h>
  26 #include <apt-pkg/tagfile.h>
  27 #include <apt-pkg/indexrecords.h>
  28 #include <apt-pkg/acquire.h>
  29 #include <apt-pkg/hashes.h>
  30 #include <apt-pkg/indexfile.h>
  31 #include <apt-pkg/pkgcache.h>
  32 #include <apt-pkg/cacheiterators.h>
  33 #include <apt-pkg/pkgrecords.h>
  34
  35 #include <stddef.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38 #include <iostream>
  39 #include <vector>
  40 #include <sys/stat.h>
  41 #include <unistd.h>
  42 #include <errno.h>
  43 #include <string>
  44 #include <sstream>
  45 #include <stdio.h>
  46 #include <ctime>
  47 #include <sys/types.h>
  48 #include <pwd.h>
  49 #include <grp.h>
  50
  51 #include <apti18n.h>
  52                                                                         /*}}}*/
  53
  54 using namespace std;
  55
  56 static void printHashSumComparision(std::string const &URI, HashStringList const &Expected, HashStringList const &Actual) /*{{{*/
  57 {
  58    if (_config->FindB("Debug::Acquire::HashSumMismatch", false) == false)
  59       return;
  60    std::cerr << std::endl << URI << ":" << std::endl << " Expected Hash: " << std::endl;
  61    for (HashStringList::const_iterator hs = Expected.begin(); hs != Expected.end(); ++hs)
  62       std::cerr <<  "\t- " << hs->toStr() << std::endl;
  63    std::cerr << " Actual Hash: " << std::endl;
  64    for (HashStringList::const_iterator hs = Actual.begin(); hs != Actual.end(); ++hs)
  65       std::cerr <<  "\t- " << hs->toStr() << std::endl;
  66 }
  67                                                                         /*}}}*/
  68 static void ChangeOwnerAndPermissionOfFile(char const * const requester, char const * const file, char const * const user, char const * const group, mode_t const mode) /*{{{*/
  69 {
  70    if (getuid() == 0 && strlen(user) != 0 && strlen(group) != 0) // if we aren't root, we can't chown, so don't try it
  71    {
  72       // ensure the file is owned by root and has good permissions
  73       struct passwd const * const pw = getpwnam(user);
  74       struct group const * const gr = getgrnam(group);
  75       if (pw != NULL && gr != NULL && chown(file, pw->pw_uid, gr->gr_gid) != 0)
  76          _error->WarningE(requester, "chown to %s:%s of file %s failed", user, group, file);
  77    }
  78    if (chmod(file, mode) != 0)
  79       _error->WarningE(requester, "chmod 0%o of file %s failed", mode, file);
  80 }
  81                                                                         /*}}}*/
  82 static std::string GetPartialFileName(std::string const &file)          /*{{{*/
  83 {
  84    std::string DestFile = _config->FindDir("Dir::State::lists") + "partial/";
  85    DestFile += file;
  86    return DestFile;
  87 }
  88                                                                         /*}}}*/
  89 static std::string GetPartialFileNameFromURI(std::string const &uri)    /*{{{*/
  90 {
  91    return GetPartialFileName(URItoFileName(uri));
  92 }
  93                                                                         /*}}}*/
  94 static std::string GetCompressedFileName(std::string const &URI, std::string const &Name, std::string const &Ext) /*{{{*/
  95 {
  96    if (Ext.empty() || Ext == "uncompressed")
  97       return Name;
  98
  99    // do not reverify cdrom sources as apt-cdrom may rewrite the Packages
 100    // file when its doing the indexcopy
 101    if (URI.substr(0,6) == "cdrom:")
 102       return Name;
 103
 104    // adjust DestFile if its compressed on disk
 105    if (_config->FindB("Acquire::GzipIndexes",false) == true)
 106       return Name + '.' + Ext;
 107    return Name;
 108 }
 109                                                                         /*}}}*/
 110 static bool AllowInsecureRepositories(indexRecords const * const MetaIndexParser, pkgAcqMetaBase * const TransactionManager, pkgAcquire::Item * const I) /*{{{*/
 111 {
 112    if(MetaIndexParser->IsAlwaysTrusted() || _config->FindB("Acquire::AllowInsecureRepositories") == true)
 113       return true;
 114
 115    _error->Error(_("Use --allow-insecure-repositories to force the update"));
 116    TransactionManager->AbortTransaction();
 117    I->Status = pkgAcquire::Item::StatError;
 118    return false;
 119 }
 120                                                                         /*}}}*/
 121
 122
 123 // Acquire::Item::Item - Constructor                                    /*{{{*/
 124 #if __GNUC__ >= 4
 125         #pragma GCC diagnostic push
 126         #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 127 #endif
 128 pkgAcquire::Item::Item(pkgAcquire *Owner,
 129                        HashStringList const &ExpectedHashes,
 130                        pkgAcqMetaBase *TransactionManager)
 131    : Owner(Owner), FileSize(0), PartialSize(0), Mode(0), ID(0), Complete(false),
 132      Local(false), QueueCounter(0), TransactionManager(TransactionManager),
 133      ExpectedAdditionalItems(0), ExpectedHashes(ExpectedHashes)
 134 {
 135    Owner->Add(this);
 136    Status = StatIdle;
 137    if(TransactionManager != NULL)
 138       TransactionManager->Add(this);
 139 }
 140 #if __GNUC__ >= 4
 141         #pragma GCC diagnostic pop
 142 #endif
 143                                                                         /*}}}*/
 144 // Acquire::Item::~Item - Destructor                                    /*{{{*/
 145 // ---------------------------------------------------------------------
 146 /* */
 147 pkgAcquire::Item::~Item()
 148 {
 149    Owner->Remove(this);
 150 }
 151                                                                         /*}}}*/
 152 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
 153 // ---------------------------------------------------------------------
 154 /* We return to an idle state if there are still other queues that could
 155    fetch this object */
 156 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 157 {
 158    if (RealFileExists(DestFile))
 159       ChangeOwnerAndPermissionOfFile("Item::Failed", DestFile.c_str(), "root", "root", 0644);
 160
 161    if(ErrorText.empty())
 162       ErrorText = LookupTag(Message,"Message");
 163    UsedMirror =  LookupTag(Message,"UsedMirror");
 164    if (QueueCounter <= 1)
 165    {
 166       /* This indicates that the file is not available right now but might
 167          be sometime later. If we do a retry cycle then this should be
 168          retried [CDROMs] */
 169       if (Cnf != NULL && Cnf->LocalOnly == true &&
 170           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
 171       {
 172          Status = StatIdle;
 173          Dequeue();
 174          return;
 175       }
 176
 177       Status = StatError;
 178       Complete = false;
 179       Dequeue();
 180    }
 181    else
 182       Status = StatIdle;
 183
 184    // check fail reason
 185    string const FailReason = LookupTag(Message, "FailReason");
 186    if(FailReason == "MaximumSizeExceeded")
 187       RenameOnError(MaximumSizeExceeded);
 188
 189    // report mirror failure back to LP if we actually use a mirror
 190    if(FailReason.size() != 0)
 191       ReportMirrorFailure(FailReason);
 192    else
 193       ReportMirrorFailure(ErrorText);
 194 }
 195                                                                         /*}}}*/
 196 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
 197 // ---------------------------------------------------------------------
 198 /* Stash status and the file size. Note that setting Complete means
 199    sub-phases of the acquire process such as decompresion are operating */
 200 void pkgAcquire::Item::Start(string /*Message*/,unsigned long long Size)
 201 {
 202    Status = StatFetching;
 203    ErrorText.clear();
 204    if (FileSize == 0 && Complete == false)
 205       FileSize = Size;
 206 }
 207                                                                         /*}}}*/
 208 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
 209 // ---------------------------------------------------------------------
 210 /* */
 211 void pkgAcquire::Item::Done(string Message,unsigned long long Size,HashStringList const &/*Hash*/,
 212                             pkgAcquire::MethodConfig * /*Cnf*/)
 213 {
 214    // We just downloaded something..
 215    string FileName = LookupTag(Message,"Filename");
 216    UsedMirror = LookupTag(Message,"UsedMirror");
 217    if (Complete == false && !Local && FileName == DestFile)
 218    {
 219       if (Owner->Log != 0)
 220          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 221    }
 222    if (RealFileExists(DestFile))
 223       ChangeOwnerAndPermissionOfFile("Item::Done", DestFile.c_str(), "root", "root", 0644);
 224
 225    if (FileSize == 0)
 226       FileSize= Size;
 227    Status = StatDone;
 228    ErrorText = string();
 229    Owner->Dequeue(this);
 230 }
 231                                                                         /*}}}*/
 232 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 233 // ---------------------------------------------------------------------
 234 /* This helper function is used by a lot of item methods as their final
 235    step */
 236 bool pkgAcquire::Item::Rename(string From,string To)
 237 {
 238    if (rename(From.c_str(),To.c_str()) == 0)
 239       return true;
 240    ChangeOwnerAndPermissionOfFile("Item::Failed", To.c_str(), "root", "root", 0644);
 241
 242    std::string S;
 243    strprintf(S, _("rename failed, %s (%s -> %s)."), strerror(errno),
 244          From.c_str(),To.c_str());
 245    Status = StatError;
 246    ErrorText += S;
 247    return false;
 248 }
 249                                                                         /*}}}*/
 250 void pkgAcquire::Item::QueueURI(ItemDesc &Item)                         /*{{{*/
 251 {
 252    if (RealFileExists(DestFile))
 253    {
 254       std::string SandboxUser = _config->Find("APT::Sandbox::User");
 255       ChangeOwnerAndPermissionOfFile("Item::QueueURI", DestFile.c_str(),
 256                                      SandboxUser.c_str(), "root", 0600);
 257    }
 258    Owner->Enqueue(Item);
 259 }
 260                                                                         /*}}}*/
 261 void pkgAcquire::Item::Dequeue()                                        /*{{{*/
 262 {
 263    Owner->Dequeue(this);
 264 }
 265                                                                         /*}}}*/
 266 bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const error)/*{{{*/
 267 {
 268    if (RealFileExists(DestFile))
 269       Rename(DestFile, DestFile + ".FAILED");
 270
 271    switch (error)
 272    {
 273       case HashSumMismatch:
 274          ErrorText = _("Hash Sum mismatch");
 275          Status = StatAuthError;
 276          ReportMirrorFailure("HashChecksumFailure");
 277          break;
 278       case SizeMismatch:
 279          ErrorText = _("Size mismatch");
 280          Status = StatAuthError;
 281          ReportMirrorFailure("SizeFailure");
 282          break;
 283       case InvalidFormat:
 284          ErrorText = _("Invalid file format");
 285          Status = StatError;
 286          // do not report as usually its not the mirrors fault, but Portal/Proxy
 287          break;
 288       case SignatureError:
 289          ErrorText = _("Signature error");
 290          Status = StatError;
 291          break;
 292       case NotClearsigned:
 293          ErrorText = _("Does not start with a cleartext signature");
 294          Status = StatError;
 295          break;
 296       case MaximumSizeExceeded:
 297          // the method is expected to report a good error for this
 298          Status = StatError;
 299          break;
 300    }
 301    return false;
 302 }
 303                                                                         /*}}}*/
 304 void pkgAcquire::Item::SetActiveSubprocess(const std::string &subprocess)/*{{{*/
 305 {
 306       ActiveSubprocess = subprocess;
 307 #if __GNUC__ >= 4
 308         #pragma GCC diagnostic push
 309         #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 310 #endif
 311         Mode = ActiveSubprocess.c_str();
 312 #if __GNUC__ >= 4
 313         #pragma GCC diagnostic pop
 314 #endif
 315 }
 316                                                                         /*}}}*/
 317 // Acquire::Item::ReportMirrorFailure                                   /*{{{*/
 318 // ---------------------------------------------------------------------
 319 void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
 320 {
 321    // we only act if a mirror was used at all
 322    if(UsedMirror.empty())
 323       return;
 324 #if 0
 325    std::cerr << "\nReportMirrorFailure: "
 326              << UsedMirror
 327              << " Uri: " << DescURI()
 328              << " FailCode: "
 329              << FailCode << std::endl;
 330 #endif
 331    string report = _config->Find("Methods::Mirror::ProblemReporting",
 332                                  "/usr/lib/apt/apt-report-mirror-failure");
 333    if(!FileExists(report))
 334       return;
 335
 336    std::vector<char const*> Args;
 337    Args.push_back(report.c_str());
 338    Args.push_back(UsedMirror.c_str());
 339    Args.push_back(DescURI().c_str());
 340    Args.push_back(FailCode.c_str());
 341    Args.push_back(NULL);
 342
 343    pid_t pid = ExecFork();
 344    if(pid < 0)
 345    {
 346       _error->Error("ReportMirrorFailure Fork failed");
 347       return;
 348    }
 349    else if(pid == 0)
 350    {
 351       execvp(Args[0], (char**)Args.data());
 352       std::cerr << "Could not exec " << Args[0] << std::endl;
 353       _exit(100);
 354    }
 355    if(!ExecWait(pid, "report-mirror-failure"))
 356    {
 357       _error->Warning("Couldn't report problem to '%s'",
 358                       _config->Find("Methods::Mirror::ProblemReporting").c_str());
 359    }
 360 }
 361                                                                         /*}}}*/
 362 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 363 // ---------------------------------------------------------------------
 364 /* Get the DiffIndex file first and see if there are patches available
 365  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 366  * patches. If anything goes wrong in that process, it will fall back to
 367  * the original packages file
 368  */
 369 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 370                                  pkgAcqMetaBase *TransactionManager,
 371                                  IndexTarget const * const Target,
 372                                  HashStringList const &ExpectedHashes,
 373                                  indexRecords *MetaIndexParser)
 374    : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHashes,
 375                      MetaIndexParser), PackagesFileReadyInPartial(false)
 376 {
 377
 378    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 379
 380    RealURI = Target->URI;
 381    Desc.Owner = this;
 382    Desc.Description = Target->Description + ".diff/Index";
 383    Desc.ShortDesc = Target->ShortDesc;
 384    Desc.URI = Target->URI + ".diff/Index";
 385
 386    DestFile = GetPartialFileNameFromURI(Desc.URI);
 387
 388    if(Debug)
 389       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 390
 391    // look for the current package file
 392    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 393    CurrentPackagesFile += URItoFileName(RealURI);
 394
 395    // FIXME: this file:/ check is a hack to prevent fetching
 396    //        from local sources. this is really silly, and
 397    //        should be fixed cleanly as soon as possible
 398    if(!FileExists(CurrentPackagesFile) ||
 399       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 400    {
 401       // we don't have a pkg file or we don't want to queue
 402       Failed("No index file, local or canceld by user", NULL);
 403       return;
 404    }
 405
 406    if(Debug)
 407       std::clog << "pkgAcqDiffIndex::pkgAcqDiffIndex(): "
 408          << CurrentPackagesFile << std::endl;
 409
 410    QueueURI(Desc);
 411
 412 }
 413                                                                         /*}}}*/
 414 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 415 // ---------------------------------------------------------------------
 416 /* The only header we use is the last-modified header. */
 417 string pkgAcqDiffIndex::Custom600Headers() const
 418 {
 419    string Final = _config->FindDir("Dir::State::lists");
 420    Final += URItoFileName(Desc.URI);
 421
 422    if(Debug)
 423       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 424
 425    struct stat Buf;
 426    if (stat(Final.c_str(),&Buf) != 0)
 427       return "\nIndex-File: true";
 428
 429    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 430 }
 431                                                                         /*}}}*/
 432 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 433 {
 434    // failing here is fine: our caller will take care of trying to
 435    // get the complete file if patching fails
 436    if(Debug)
 437       std::clog << "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile
 438          << std::endl;
 439
 440    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 441    pkgTagFile TF(&Fd);
 442    if (_error->PendingError() == true)
 443       return false;
 444
 445    pkgTagSection Tags;
 446    if(unlikely(TF.Step(Tags) == false))
 447       return false;
 448
 449    HashStringList ServerHashes;
 450    unsigned long long ServerSize = 0;
 451
 452    for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
 453    {
 454       std::string tagname = *type;
 455       tagname.append("-Current");
 456       std::string const tmp = Tags.FindS(tagname.c_str());
 457       if (tmp.empty() == true)
 458          continue;
 459
 460       string hash;
 461       unsigned long long size;
 462       std::stringstream ss(tmp);
 463       ss >> hash >> size;
 464       if (unlikely(hash.empty() == true))
 465          continue;
 466       if (unlikely(ServerSize != 0 && ServerSize != size))
 467          continue;
 468       ServerHashes.push_back(HashString(*type, hash));
 469       ServerSize = size;
 470    }
 471
 472    if (ServerHashes.usable() == false)
 473    {
 474       if (Debug == true)
 475          std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": Did not find a good hashsum in the index" << std::endl;
 476       return false;
 477    }
 478
 479    if (ServerHashes != HashSums())
 480    {
 481       if (Debug == true)
 482       {
 483          std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": Index has different hashes than parser, probably older, so fail pdiffing" << std::endl;
 484          printHashSumComparision(CurrentPackagesFile, ServerHashes, HashSums());
 485       }
 486       return false;
 487    }
 488
 489    if (ServerHashes.VerifyFile(CurrentPackagesFile) == true)
 490    {
 491       // we have the same sha1 as the server so we are done here
 492       if(Debug)
 493          std::clog << "pkgAcqDiffIndex: Package file " << CurrentPackagesFile << " is up-to-date" << std::endl;
 494
 495       // list cleanup needs to know that this file as well as the already
 496       // present index is ours, so we create an empty diff to save it for us
 497       new pkgAcqIndexDiffs(Owner, TransactionManager, Target,
 498                            ExpectedHashes, MetaIndexParser);
 499       return true;
 500    }
 501
 502    FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 503    Hashes LocalHashesCalc;
 504    LocalHashesCalc.AddFD(fd);
 505    HashStringList const LocalHashes = LocalHashesCalc.GetHashStringList();
 506
 507    if(Debug)
 508       std::clog << "Server-Current: " << ServerHashes.find(NULL)->toStr() << " and we start at "
 509          << fd.Name() << " " << fd.FileSize() << " " << LocalHashes.find(NULL)->toStr() << std::endl;
 510
 511    // parse all of (provided) history
 512    vector<DiffInfo> available_patches;
 513    bool firstAcceptedHashes = true;
 514    for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
 515    {
 516       if (LocalHashes.find(*type) == NULL)
 517          continue;
 518
 519       std::string tagname = *type;
 520       tagname.append("-History");
 521       std::string const tmp = Tags.FindS(tagname.c_str());
 522       if (tmp.empty() == true)
 523          continue;
 524
 525       string hash, filename;
 526       unsigned long long size;
 527       std::stringstream ss(tmp);
 528
 529       while (ss >> hash >> size >> filename)
 530       {
 531          if (unlikely(hash.empty() == true || filename.empty() == true))
 532             continue;
 533
 534          // see if we have a record for this file already
 535          std::vector<DiffInfo>::iterator cur = available_patches.begin();
 536          for (; cur != available_patches.end(); ++cur)
 537          {
 538             if (cur->file != filename || unlikely(cur->result_size != size))
 539                continue;
 540             cur->result_hashes.push_back(HashString(*type, hash));
 541             break;
 542          }
 543          if (cur != available_patches.end())
 544             continue;
 545          if (firstAcceptedHashes == true)
 546          {
 547             DiffInfo next;
 548             next.file = filename;
 549             next.result_hashes.push_back(HashString(*type, hash));
 550             next.result_size = size;
 551             next.patch_size = 0;
 552             available_patches.push_back(next);
 553          }
 554          else
 555          {
 556             if (Debug == true)
 557                std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename
 558                   << " wasn't in the list for the first parsed hash! (history)" << std::endl;
 559             break;
 560          }
 561       }
 562       firstAcceptedHashes = false;
 563    }
 564
 565    if (unlikely(available_patches.empty() == true))
 566    {
 567       if (Debug)
 568          std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": "
 569             << "Couldn't find any patches for the patch series." << std::endl;
 570       return false;
 571    }
 572
 573    for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
 574    {
 575       if (LocalHashes.find(*type) == NULL)
 576          continue;
 577
 578       std::string tagname = *type;
 579       tagname.append("-Patches");
 580       std::string const tmp = Tags.FindS(tagname.c_str());
 581       if (tmp.empty() == true)
 582          continue;
 583
 584       string hash, filename;
 585       unsigned long long size;
 586       std::stringstream ss(tmp);
 587
 588       while (ss >> hash >> size >> filename)
 589       {
 590          if (unlikely(hash.empty() == true || filename.empty() == true))
 591             continue;
 592
 593          // see if we have a record for this file already
 594          std::vector<DiffInfo>::iterator cur = available_patches.begin();
 595          for (; cur != available_patches.end(); ++cur)
 596          {
 597             if (cur->file != filename)
 598                continue;
 599             if (unlikely(cur->patch_size != 0 && cur->patch_size != size))
 600                continue;
 601             cur->patch_hashes.push_back(HashString(*type, hash));
 602             cur->patch_size = size;
 603             break;
 604          }
 605          if (cur != available_patches.end())
 606                continue;
 607          if (Debug == true)
 608             std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename
 609                << " wasn't in the list for the first parsed hash! (patches)" << std::endl;
 610          break;
 611       }
 612    }
 613
 614    bool foundStart = false;
 615    for (std::vector<DiffInfo>::iterator cur = available_patches.begin();
 616          cur != available_patches.end(); ++cur)
 617    {
 618       if (LocalHashes != cur->result_hashes)
 619          continue;
 620
 621       available_patches.erase(available_patches.begin(), cur);
 622       foundStart = true;
 623       break;
 624    }
 625
 626    if (foundStart == false || unlikely(available_patches.empty() == true))
 627    {
 628       if (Debug)
 629          std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": "
 630             << "Couldn't find the start of the patch series." << std::endl;
 631       return false;
 632    }
 633
 634    // patching with too many files is rather slow compared to a fast download
 635    unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 636    if (fileLimit != 0 && fileLimit < available_patches.size())
 637    {
 638       if (Debug)
 639          std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 640             << ") so fallback to complete download" << std::endl;
 641       return false;
 642    }
 643
 644    // calculate the size of all patches we have to get
 645    // note that all sizes are uncompressed, while we download compressed files
 646    unsigned long long patchesSize = 0;
 647    for (std::vector<DiffInfo>::const_iterator cur = available_patches.begin();
 648          cur != available_patches.end(); ++cur)
 649       patchesSize += cur->patch_size;
 650    unsigned long long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 651    if (false && sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 652    {
 653       if (Debug)
 654          std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 655             << ") so fallback to complete download" << std::endl;
 656       return false;
 657    }
 658
 659    // FIXME: make this use the method
 660    PackagesFileReadyInPartial = true;
 661    std::string const Partial = GetPartialFileNameFromURI(RealURI);
 662
 663    FileFd From(CurrentPackagesFile, FileFd::ReadOnly);
 664    FileFd To(Partial, FileFd::WriteEmpty);
 665    if(CopyFile(From, To) == false)
 666       return _error->Errno("CopyFile", "failed to copy");
 667
 668    if(Debug)
 669       std::cerr << "Done copying " << CurrentPackagesFile
 670                 << " -> " << Partial
 671                 << std::endl;
 672
 673    // we have something, queue the diffs
 674    string::size_type const last_space = Description.rfind(" ");
 675    if(last_space != string::npos)
 676       Description.erase(last_space, Description.size()-last_space);
 677
 678    /* decide if we should download patches one by one or in one go:
 679       The first is good if the server merges patches, but many don't so client
 680       based merging can be attempt in which case the second is better.
 681       "bad things" will happen if patches are merged on the server,
 682       but client side merging is attempt as well */
 683    bool pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true);
 684    if (pdiff_merge == true)
 685    {
 686       // reprepro adds this flag if it has merged patches on the server
 687       std::string const precedence = Tags.FindS("X-Patch-Precedence");
 688       pdiff_merge = (precedence != "merged");
 689    }
 690
 691    if (pdiff_merge == false)
 692    {
 693       new pkgAcqIndexDiffs(Owner, TransactionManager, Target, ExpectedHashes,
 694                            MetaIndexParser, available_patches);
 695    }
 696    else
 697    {
 698       std::vector<pkgAcqIndexMergeDiffs*> *diffs = new std::vector<pkgAcqIndexMergeDiffs*>(available_patches.size());
 699       for(size_t i = 0; i < available_patches.size(); ++i)
 700          (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, TransactionManager,
 701                Target,
 702                ExpectedHashes,
 703                MetaIndexParser,
 704                available_patches[i],
 705                diffs);
 706    }
 707
 708    Complete = false;
 709    Status = StatDone;
 710    Dequeue();
 711    return true;
 712 }
 713                                                                         /*}}}*/
 714 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * Cnf)/*{{{*/
 715 {
 716    Item::Failed(Message,Cnf);
 717    Status = StatDone;
 718
 719    if(Debug)
 720       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl
 721                 << "Falling back to normal index file acquire" << std::endl;
 722
 723    new pkgAcqIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser);
 724 }
 725                                                                         /*}}}*/
 726 void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/
 727                            pkgAcquire::MethodConfig *Cnf)
 728 {
 729    if(Debug)
 730       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 731
 732    Item::Done(Message, Size, Hashes, Cnf);
 733
 734    // verify the index target
 735    if(Target && Target->MetaKey != "" && MetaIndexParser && Hashes.usable())
 736    {
 737       std::string IndexMetaKey  = Target->MetaKey + ".diff/Index";
 738       indexRecords::checkSum *Record = MetaIndexParser->Lookup(IndexMetaKey);
 739       if(Record && Record->Hashes.usable() && Hashes != Record->Hashes)
 740       {
 741          RenameOnError(HashSumMismatch);
 742          printHashSumComparision(RealURI, Record->Hashes, Hashes);
 743          Failed(Message, Cnf);
 744          return;
 745       }
 746
 747    }
 748
 749    string FinalFile;
 750    FinalFile = _config->FindDir("Dir::State::lists");
 751    FinalFile += URItoFileName(Desc.URI);
 752
 753    if(StringToBool(LookupTag(Message,"IMS-Hit"),false))
 754       DestFile = FinalFile;
 755
 756    if(!ParseDiffIndex(DestFile))
 757       return Failed("Message: Couldn't parse pdiff index", Cnf);
 758
 759    // queue for final move
 760    TransactionManager->TransactionStageCopy(this, DestFile, FinalFile);
 761
 762    Complete = true;
 763    Status = StatDone;
 764    Dequeue();
 765    return;
 766 }
 767                                                                         /*}}}*/
 768 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 769 // ---------------------------------------------------------------------
 770 /* The package diff is added to the queue. one object is constructed
 771  * for each diff and the index
 772  */
 773 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 774                                    pkgAcqMetaBase *TransactionManager,
 775                                    struct IndexTarget const * const Target,
 776                                    HashStringList const &ExpectedHashes,
 777                                    indexRecords *MetaIndexParser,
 778                                    vector<DiffInfo> diffs)
 779    : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser),
 780      available_patches(diffs)
 781 {
 782    DestFile = GetPartialFileNameFromURI(Target->URI);
 783
 784    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 785
 786    RealURI = Target->URI;
 787    Desc.Owner = this;
 788    Description = Target->Description;
 789    Desc.ShortDesc = Target->ShortDesc;
 790
 791    if(available_patches.empty() == true)
 792    {
 793       // we are done (yeah!), check hashes against the final file
 794       DestFile = _config->FindDir("Dir::State::lists");
 795       DestFile += URItoFileName(Target->URI);
 796       Finish(true);
 797    }
 798    else
 799    {
 800       // get the next diff
 801       State = StateFetchDiff;
 802       QueueNextDiff();
 803    }
 804 }
 805                                                                         /*}}}*/
 806 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig * Cnf)/*{{{*/
 807 {
 808    Item::Failed(Message,Cnf);
 809    Status = StatDone;
 810
 811    if(Debug)
 812       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl
 813                 << "Falling back to normal index file acquire" << std::endl;
 814    new pkgAcqIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser);
 815    Finish();
 816 }
 817                                                                         /*}}}*/
 818 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 819 void pkgAcqIndexDiffs::Finish(bool allDone)
 820 {
 821    if(Debug)
 822       std::clog << "pkgAcqIndexDiffs::Finish(): "
 823                 << allDone << " "
 824                 << Desc.URI << std::endl;
 825
 826    // we restore the original name, this is required, otherwise
 827    // the file will be cleaned
 828    if(allDone)
 829    {
 830       if(HashSums().usable() && !HashSums().VerifyFile(DestFile))
 831       {
 832          RenameOnError(HashSumMismatch);
 833          Dequeue();
 834          return;
 835       }
 836
 837       // queue for copy
 838       std::string FinalFile = _config->FindDir("Dir::State::lists");
 839       FinalFile += URItoFileName(RealURI);
 840       TransactionManager->TransactionStageCopy(this, DestFile, FinalFile);
 841
 842       // this is for the "real" finish
 843       Complete = true;
 844       Status = StatDone;
 845       Dequeue();
 846       if(Debug)
 847          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 848       return;
 849    }
 850
 851    if(Debug)
 852       std::clog << "Finishing: " << Desc.URI << std::endl;
 853    Complete = false;
 854    Status = StatDone;
 855    Dequeue();
 856    return;
 857 }
 858                                                                         /*}}}*/
 859 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 860 {
 861    // calc sha1 of the just patched file
 862    std::string const FinalFile = GetPartialFileNameFromURI(RealURI);
 863
 864    if(!FileExists(FinalFile))
 865    {
 866       Failed("Message: No FinalFile " + FinalFile + " available", NULL);
 867       return false;
 868    }
 869
 870    FileFd fd(FinalFile, FileFd::ReadOnly);
 871    Hashes LocalHashesCalc;
 872    LocalHashesCalc.AddFD(fd);
 873    HashStringList const LocalHashes = LocalHashesCalc.GetHashStringList();
 874
 875    if(Debug)
 876       std::clog << "QueueNextDiff: " << FinalFile << " (" << LocalHashes.find(NULL)->toStr() << ")" << std::endl;
 877
 878    if (unlikely(LocalHashes.usable() == false || ExpectedHashes.usable() == false))
 879    {
 880       Failed("Local/Expected hashes are not usable", NULL);
 881       return false;
 882    }
 883
 884
 885    // final file reached before all patches are applied
 886    if(LocalHashes == ExpectedHashes)
 887    {
 888       Finish(true);
 889       return true;
 890    }
 891
 892    // remove all patches until the next matching patch is found
 893    // this requires the Index file to be ordered
 894    for(vector<DiffInfo>::iterator I = available_patches.begin();
 895        available_patches.empty() == false &&
 896           I != available_patches.end() &&
 897           I->result_hashes != LocalHashes;
 898        ++I)
 899    {
 900       available_patches.erase(I);
 901    }
 902
 903    // error checking and falling back if no patch was found
 904    if(available_patches.empty() == true)
 905    {
 906       Failed("No patches left to reach target", NULL);
 907       return false;
 908    }
 909
 910    // queue the right diff
 911    Desc.URI = RealURI + ".diff/" + available_patches[0].file + ".gz";
 912    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 913    DestFile = GetPartialFileNameFromURI(RealURI + ".diff/" + available_patches[0].file);
 914
 915    if(Debug)
 916       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 917
 918    QueueURI(Desc);
 919
 920    return true;
 921 }
 922                                                                         /*}}}*/
 923 void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size, HashStringList const &Hashes,       /*{{{*/
 924                             pkgAcquire::MethodConfig *Cnf)
 925 {
 926    if(Debug)
 927       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 928
 929    Item::Done(Message, Size, Hashes, Cnf);
 930
 931    // FIXME: verify this download too before feeding it to rred
 932    std::string const FinalFile = GetPartialFileNameFromURI(RealURI);
 933
 934    // success in downloading a diff, enter ApplyDiff state
 935    if(State == StateFetchDiff)
 936    {
 937       FileFd fd(DestFile, FileFd::ReadOnly, FileFd::Gzip);
 938       class Hashes LocalHashesCalc;
 939       LocalHashesCalc.AddFD(fd);
 940       HashStringList const LocalHashes = LocalHashesCalc.GetHashStringList();
 941
 942       if (fd.Size() != available_patches[0].patch_size ||
 943             available_patches[0].patch_hashes != LocalHashes)
 944       {
 945          Failed("Patch has Size/Hashsum mismatch", NULL);
 946          return;
 947       }
 948
 949       // rred excepts the patch as $FinalFile.ed
 950       Rename(DestFile,FinalFile+".ed");
 951
 952       if(Debug)
 953          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 954
 955       State = StateApplyDiff;
 956       Local = true;
 957       Desc.URI = "rred:" + FinalFile;
 958       QueueURI(Desc);
 959       SetActiveSubprocess("rred");
 960       return;
 961    }
 962
 963
 964    // success in download/apply a diff, queue next (if needed)
 965    if(State == StateApplyDiff)
 966    {
 967       // remove the just applied patch
 968       available_patches.erase(available_patches.begin());
 969       unlink((FinalFile + ".ed").c_str());
 970
 971       // move into place
 972       if(Debug)
 973       {
 974          std::clog << "Moving patched file in place: " << std::endl
 975                    << DestFile << " -> " << FinalFile << std::endl;
 976       }
 977       Rename(DestFile,FinalFile);
 978       chmod(FinalFile.c_str(),0644);
 979
 980       // see if there is more to download
 981       if(available_patches.empty() == false) {
 982          new pkgAcqIndexDiffs(Owner, TransactionManager, Target,
 983                               ExpectedHashes, MetaIndexParser,
 984                               available_patches);
 985          return Finish();
 986       } else
 987          // update
 988          DestFile = FinalFile;
 989          return Finish(true);
 990    }
 991 }
 992                                                                         /*}}}*/
 993 // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor                 /*{{{*/
 994 pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner,
 995                                              pkgAcqMetaBase *TransactionManager,
 996                                              struct IndexTarget const * const Target,
 997                                              HashStringList const &ExpectedHashes,
 998                                              indexRecords *MetaIndexParser,
 999                                              DiffInfo const &patch,
1000                                              std::vector<pkgAcqIndexMergeDiffs*> const * const allPatches)
1001   : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser),
1002      patch(patch), allPatches(allPatches), State(StateFetchDiff)
1003 {
1004    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
1005
1006    RealURI = Target->URI;
1007    Desc.Owner = this;
1008    Description = Target->Description;
1009    Desc.ShortDesc = Target->ShortDesc;
1010
1011    Desc.URI = RealURI + ".diff/" + patch.file + ".gz";
1012    Desc.Description = Description + " " + patch.file + string(".pdiff");
1013
1014    DestFile = GetPartialFileNameFromURI(RealURI + ".diff/" + patch.file);
1015
1016    if(Debug)
1017       std::clog << "pkgAcqIndexMergeDiffs: " << Desc.URI << std::endl;
1018
1019    QueueURI(Desc);
1020 }
1021                                                                         /*}}}*/
1022 void pkgAcqIndexMergeDiffs::Failed(string Message,pkgAcquire::MethodConfig * Cnf)/*{{{*/
1023 {
1024    if(Debug)
1025       std::clog << "pkgAcqIndexMergeDiffs failed: " << Desc.URI << " with " << Message << std::endl;
1026
1027    Item::Failed(Message,Cnf);
1028    Status = StatDone;
1029
1030    // check if we are the first to fail, otherwise we are done here
1031    State = StateDoneDiff;
1032    for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
1033          I != allPatches->end(); ++I)
1034       if ((*I)->State == StateErrorDiff)
1035          return;
1036
1037    // first failure means we should fallback
1038    State = StateErrorDiff;
1039    std::clog << "Falling back to normal index file acquire" << std::endl;
1040    new pkgAcqIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser);
1041 }
1042                                                                         /*}}}*/
1043 void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStringList const &Hashes,   /*{{{*/
1044                             pkgAcquire::MethodConfig *Cnf)
1045 {
1046    if(Debug)
1047       std::clog << "pkgAcqIndexMergeDiffs::Done(): " << Desc.URI << std::endl;
1048
1049    Item::Done(Message,Size,Hashes,Cnf);
1050
1051    // FIXME: verify download before feeding it to rred
1052    string const FinalFile = GetPartialFileNameFromURI(RealURI);
1053
1054    if (State == StateFetchDiff)
1055    {
1056       FileFd fd(DestFile, FileFd::ReadOnly, FileFd::Gzip);
1057       class Hashes LocalHashesCalc;
1058       LocalHashesCalc.AddFD(fd);
1059       HashStringList const LocalHashes = LocalHashesCalc.GetHashStringList();
1060
1061       if (fd.Size() != patch.patch_size || patch.patch_hashes != LocalHashes)
1062       {
1063          Failed("Patch has Size/Hashsum mismatch", NULL);
1064          return;
1065       }
1066
1067       // rred expects the patch as $FinalFile.ed.$patchname.gz
1068       Rename(DestFile, FinalFile + ".ed." + patch.file + ".gz");
1069
1070       // check if this is the last completed diff
1071       State = StateDoneDiff;
1072       for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
1073             I != allPatches->end(); ++I)
1074          if ((*I)->State != StateDoneDiff)
1075          {
1076             if(Debug)
1077                std::clog << "Not the last done diff in the batch: " << Desc.URI << std::endl;
1078             return;
1079          }
1080
1081       // this is the last completed diff, so we are ready to apply now
1082       State = StateApplyDiff;
1083
1084       if(Debug)
1085          std::clog << "Sending to rred method: " << FinalFile << std::endl;
1086
1087       Local = true;
1088       Desc.URI = "rred:" + FinalFile;
1089       QueueURI(Desc);
1090       SetActiveSubprocess("rred");
1091       return;
1092    }
1093    // success in download/apply all diffs, clean up
1094    else if (State == StateApplyDiff)
1095    {
1096       // see if we really got the expected file
1097       if(ExpectedHashes.usable() && !ExpectedHashes.VerifyFile(DestFile))
1098       {
1099          RenameOnError(HashSumMismatch);
1100          return;
1101       }
1102
1103
1104       std::string FinalFile = _config->FindDir("Dir::State::lists");
1105       FinalFile += URItoFileName(RealURI);
1106
1107       // move the result into place
1108       if(Debug)
1109          std::clog << "Queue patched file in place: " << std::endl
1110                    << DestFile << " -> " << FinalFile << std::endl;
1111
1112       // queue for copy by the transaction manager
1113       TransactionManager->TransactionStageCopy(this, DestFile, FinalFile);
1114
1115       // ensure the ed's are gone regardless of list-cleanup
1116       for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
1117             I != allPatches->end(); ++I)
1118       {
1119          std::string const PartialFile = GetPartialFileNameFromURI(RealURI);
1120          std::string patch = PartialFile + ".ed." + (*I)->patch.file + ".gz";
1121          unlink(patch.c_str());
1122       }
1123
1124       // all set and done
1125       Complete = true;
1126       if(Debug)
1127          std::clog << "allDone: " << DestFile << "\n" << std::endl;
1128    }
1129 }
1130                                                                         /*}}}*/
1131 // AcqBaseIndex::VerifyHashByMetaKey - verify hash for the given metakey /*{{{*/
1132 bool pkgAcqBaseIndex::VerifyHashByMetaKey(HashStringList const &Hashes)
1133 {
1134    if(MetaKey != "" && Hashes.usable())
1135    {
1136       indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey);
1137       if(Record && Record->Hashes.usable() && Hashes != Record->Hashes)
1138       {
1139          printHashSumComparision(RealURI, Record->Hashes, Hashes);
1140          return false;
1141       }
1142    }
1143    return true;
1144 }
1145                                                                         /*}}}*/
1146 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
1147 // ---------------------------------------------------------------------
1148 /* The package file is added to the queue and a second class is
1149    instantiated to fetch the revision file */
1150 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
1151                          string URI,string URIDesc,string ShortDesc,
1152                          HashStringList const  &ExpectedHash)
1153    : pkgAcqBaseIndex(Owner, 0, NULL, ExpectedHash, NULL)
1154 {
1155    RealURI = URI;
1156
1157    AutoSelectCompression();
1158    Init(URI, URIDesc, ShortDesc);
1159
1160    if(_config->FindB("Debug::Acquire::Transaction", false) == true)
1161       std::clog << "New pkgIndex with TransactionManager "
1162                 << TransactionManager << std::endl;
1163 }
1164                                                                         /*}}}*/
1165 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
1166 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
1167                          pkgAcqMetaBase *TransactionManager,
1168                          IndexTarget const *Target,
1169                          HashStringList const &ExpectedHash,
1170                          indexRecords *MetaIndexParser)
1171    : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHash,
1172                      MetaIndexParser)
1173 {
1174    RealURI = Target->URI;
1175
1176    // autoselect the compression method
1177    AutoSelectCompression();
1178    Init(Target->URI, Target->Description, Target->ShortDesc);
1179
1180    if(_config->FindB("Debug::Acquire::Transaction", false) == true)
1181       std::clog << "New pkgIndex with TransactionManager "
1182                 << TransactionManager << std::endl;
1183 }
1184                                                                         /*}}}*/
1185 // AcqIndex::AutoSelectCompression - Select compression                 /*{{{*/
1186 void pkgAcqIndex::AutoSelectCompression()
1187 {
1188    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
1189    CompressionExtensions = "";
1190    if (ExpectedHashes.usable())
1191    {
1192       for (std::vector<std::string>::const_iterator t = types.begin();
1193            t != types.end(); ++t)
1194       {
1195          std::string CompressedMetaKey = string(Target->MetaKey).append(".").append(*t);
1196          if (*t == "uncompressed" ||
1197              MetaIndexParser->Exists(CompressedMetaKey) == true)
1198             CompressionExtensions.append(*t).append(" ");
1199       }
1200    }
1201    else
1202    {
1203       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
1204          CompressionExtensions.append(*t).append(" ");
1205    }
1206    if (CompressionExtensions.empty() == false)
1207       CompressionExtensions.erase(CompressionExtensions.end()-1);
1208 }
1209                                                                         /*}}}*/
1210 // AcqIndex::Init - defered Constructor                                 /*{{{*/
1211 void pkgAcqIndex::Init(string const &URI, string const &URIDesc,
1212                        string const &ShortDesc)
1213 {
1214    Stage = STAGE_DOWNLOAD;
1215
1216    DestFile = GetPartialFileNameFromURI(URI);
1217
1218    CurrentCompressionExtension = CompressionExtensions.substr(0, CompressionExtensions.find(' '));
1219    if (CurrentCompressionExtension == "uncompressed")
1220    {
1221       Desc.URI = URI;
1222       if(Target)
1223          MetaKey = string(Target->MetaKey);
1224    }
1225    else
1226    {
1227       Desc.URI = URI + '.' + CurrentCompressionExtension;
1228       DestFile = DestFile + '.' + CurrentCompressionExtension;
1229       if(Target)
1230          MetaKey = string(Target->MetaKey) + '.' + CurrentCompressionExtension;
1231    }
1232
1233    // load the filesize
1234    if(MetaIndexParser)
1235    {
1236       indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey);
1237       if(Record)
1238          FileSize = Record->Size;
1239
1240       InitByHashIfNeeded(MetaKey);
1241    }
1242
1243    Desc.Description = URIDesc;
1244    Desc.Owner = this;
1245    Desc.ShortDesc = ShortDesc;
1246
1247    QueueURI(Desc);
1248 }
1249                                                                         /*}}}*/
1250 // AcqIndex::AdjustForByHash - modify URI for by-hash support           /*{{{*/
1251 void pkgAcqIndex::InitByHashIfNeeded(const std::string MetaKey)
1252 {
1253    // TODO:
1254    //  - (maybe?) add support for by-hash into the sources.list as flag
1255    //  - make apt-ftparchive generate the hashes (and expire?)
1256    std::string HostKnob = "APT::Acquire::" + ::URI(Desc.URI).Host + "::By-Hash";
1257    if(_config->FindB("APT::Acquire::By-Hash", false) == true ||
1258       _config->FindB(HostKnob, false) == true ||
1259       MetaIndexParser->GetSupportsAcquireByHash())
1260    {
1261       indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey);
1262       if(Record)
1263       {
1264          // FIXME: should we really use the best hash here? or a fixed one?
1265          const HashString *TargetHash = Record->Hashes.find("");
1266          std::string ByHash = "/by-hash/" + TargetHash->HashType() + "/" + TargetHash->HashValue();
1267          size_t trailing_slash = Desc.URI.find_last_of("/");
1268          Desc.URI = Desc.URI.replace(
1269             trailing_slash,
1270             Desc.URI.substr(trailing_slash+1).size()+1,
1271             ByHash);
1272       } else {
1273          _error->Warning(
1274             "Fetching ByHash requested but can not find record for %s",
1275             MetaKey.c_str());
1276       }
1277    }
1278 }
1279                                                                         /*}}}*/
1280 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
1281 // ---------------------------------------------------------------------
1282 /* The only header we use is the last-modified header. */
1283 string pkgAcqIndex::Custom600Headers() const
1284 {
1285    string Final = GetFinalFilename();
1286
1287    string msg = "\nIndex-File: true";
1288    struct stat Buf;
1289    if (stat(Final.c_str(),&Buf) == 0)
1290       msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1291
1292    return msg;
1293 }
1294                                                                         /*}}}*/
1295 // pkgAcqIndex::Failed - getting the indexfile failed                   /*{{{*/
1296 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1297 {
1298    Item::Failed(Message,Cnf);
1299
1300    size_t const nextExt = CompressionExtensions.find(' ');
1301    if (nextExt != std::string::npos)
1302    {
1303       CompressionExtensions = CompressionExtensions.substr(nextExt+1);
1304       Init(RealURI, Desc.Description, Desc.ShortDesc);
1305       Status = StatIdle;
1306       return;
1307    }
1308
1309    // on decompression failure, remove bad versions in partial/
1310    if (Stage == STAGE_DECOMPRESS_AND_VERIFY)
1311    {
1312       unlink(EraseFileName.c_str());
1313    }
1314
1315    /// cancel the entire transaction
1316    TransactionManager->AbortTransaction();
1317 }
1318                                                                         /*}}}*/
1319 // pkgAcqIndex::GetFinalFilename - Return the full final file path      /*{{{*/
1320 std::string pkgAcqIndex::GetFinalFilename() const
1321 {
1322    std::string FinalFile = _config->FindDir("Dir::State::lists");
1323    FinalFile += URItoFileName(RealURI);
1324    return GetCompressedFileName(RealURI, FinalFile, CurrentCompressionExtension);
1325 }
1326                                                                         /*}}}*/
1327 // AcqIndex::ReverifyAfterIMS - Reverify index after an ims-hit         /*{{{*/
1328 void pkgAcqIndex::ReverifyAfterIMS()
1329 {
1330    // update destfile to *not* include the compression extension when doing
1331    // a reverify (as its uncompressed on disk already)
1332    DestFile = GetCompressedFileName(RealURI, GetPartialFileNameFromURI(RealURI), CurrentCompressionExtension);
1333
1334    // copy FinalFile into partial/ so that we check the hash again
1335    string FinalFile = GetFinalFilename();
1336    Stage = STAGE_DECOMPRESS_AND_VERIFY;
1337    Desc.URI = "copy:" + FinalFile;
1338    QueueURI(Desc);
1339 }
1340                                                                         /*}}}*/
1341 // AcqIndex::ValidateFile - Validate the content of the downloaded file /*{{{*/
1342 bool pkgAcqIndex::ValidateFile(const std::string &FileName)
1343 {
1344    // FIXME: this can go away once we only ever download stuff that
1345    //        has a valid hash and we never do GET based probing
1346    // FIXME2: this also leaks debian-isms into the code and should go therefore
1347
1348    /* Always validate the index file for correctness (all indexes must
1349     * have a Package field) (LP: #346386) (Closes: #627642)
1350     */
1351    FileFd fd(FileName, FileFd::ReadOnly, FileFd::Extension);
1352    // Only test for correctness if the content of the file is not empty
1353    // (empty is ok)
1354    if (fd.Size() > 0)
1355    {
1356       pkgTagSection sec;
1357       pkgTagFile tag(&fd);
1358
1359       // all our current indexes have a field 'Package' in each section
1360       if (_error->PendingError() == true ||
1361           tag.Step(sec) == false ||
1362           sec.Exists("Package") == false)
1363          return false;
1364    }
1365    return true;
1366 }
1367                                                                         /*}}}*/
1368 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
1369 // ---------------------------------------------------------------------
1370 /* This goes through a number of states.. On the initial fetch the
1371    method could possibly return an alternate filename which points
1372    to the uncompressed version of the file. If this is so the file
1373    is copied into the partial directory. In all other cases the file
1374    is decompressed with a compressed uri. */
1375 void pkgAcqIndex::Done(string Message,
1376                        unsigned long long Size,
1377                        HashStringList const &Hashes,
1378                        pkgAcquire::MethodConfig *Cfg)
1379 {
1380    Item::Done(Message,Size,Hashes,Cfg);
1381
1382    switch(Stage)
1383    {
1384       case STAGE_DOWNLOAD:
1385          StageDownloadDone(Message, Hashes, Cfg);
1386          break;
1387       case STAGE_DECOMPRESS_AND_VERIFY:
1388          StageDecompressDone(Message, Hashes, Cfg);
1389          break;
1390    }
1391 }
1392                                                                         /*}}}*/
1393 // AcqIndex::StageDownloadDone - Queue for decompress and verify        /*{{{*/
1394 void pkgAcqIndex::StageDownloadDone(string Message,
1395                                     HashStringList const &Hashes,
1396                                     pkgAcquire::MethodConfig *Cfg)
1397 {
1398    // First check if the calculcated Hash of the (compressed) downloaded
1399    // file matches the hash we have in the MetaIndexRecords for this file
1400    if(VerifyHashByMetaKey(Hashes) == false)
1401    {
1402       RenameOnError(HashSumMismatch);
1403       Failed(Message, Cfg);
1404       return;
1405    }
1406
1407    Complete = true;
1408
1409    // Handle the unzipd case
1410    string FileName = LookupTag(Message,"Alt-Filename");
1411    if (FileName.empty() == false)
1412    {
1413       Stage = STAGE_DECOMPRESS_AND_VERIFY;
1414       Local = true;
1415       DestFile += ".decomp";
1416       Desc.URI = "copy:" + FileName;
1417       QueueURI(Desc);
1418       SetActiveSubprocess("copy");
1419       return;
1420    }
1421
1422    FileName = LookupTag(Message,"Filename");
1423    if (FileName.empty() == true)
1424    {
1425       Status = StatError;
1426       ErrorText = "Method gave a blank filename";
1427    }
1428
1429    // Methods like e.g. "file:" will give us a (compressed) FileName that is
1430    // not the "DestFile" we set, in this case we uncompress from the local file
1431    if (FileName != DestFile)
1432       Local = true;
1433    else
1434       EraseFileName = FileName;
1435
1436    // we need to verify the file against the current Release file again
1437    // on if-modfied-since hit to avoid a stale attack against us
1438    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1439    {
1440       // The files timestamp matches, reverify by copy into partial/
1441       EraseFileName = "";
1442       ReverifyAfterIMS();
1443       return;
1444    }
1445
1446    // If we have compressed indexes enabled, queue for hash verification
1447    if (_config->FindB("Acquire::GzipIndexes",false))
1448    {
1449       DestFile = GetPartialFileNameFromURI(RealURI + '.' + CurrentCompressionExtension);
1450       EraseFileName = "";
1451       Stage = STAGE_DECOMPRESS_AND_VERIFY;
1452       Desc.URI = "copy:" + FileName;
1453       QueueURI(Desc);
1454       SetActiveSubprocess("copy");
1455       return;
1456     }
1457
1458    // get the binary name for your used compression type
1459    string decompProg;
1460    if(CurrentCompressionExtension == "uncompressed")
1461       decompProg = "copy";
1462    else
1463       decompProg = _config->Find(string("Acquire::CompressionTypes::").append(CurrentCompressionExtension),"");
1464    if(decompProg.empty() == true)
1465    {
1466       _error->Error("Unsupported extension: %s", CurrentCompressionExtension.c_str());
1467       return;
1468    }
1469
1470    // queue uri for the next stage
1471    Stage = STAGE_DECOMPRESS_AND_VERIFY;
1472    DestFile += ".decomp";
1473    Desc.URI = decompProg + ":" + FileName;
1474    QueueURI(Desc);
1475    SetActiveSubprocess(decompProg);
1476 }
1477                                                                         /*}}}*/
1478 // pkgAcqIndex::StageDecompressDone - Final verification                /*{{{*/
1479 void pkgAcqIndex::StageDecompressDone(string Message,
1480                                       HashStringList const &Hashes,
1481                                       pkgAcquire::MethodConfig *Cfg)
1482 {
1483    if (ExpectedHashes.usable() && ExpectedHashes != Hashes)
1484    {
1485       Desc.URI = RealURI;
1486       RenameOnError(HashSumMismatch);
1487       printHashSumComparision(RealURI, ExpectedHashes, Hashes);
1488       Failed(Message, Cfg);
1489       return;
1490    }
1491
1492    if(!ValidateFile(DestFile))
1493    {
1494       RenameOnError(InvalidFormat);
1495       Failed(Message, Cfg);
1496       return;
1497    }
1498
1499    // remove the compressed version of the file
1500    unlink(EraseFileName.c_str());
1501
1502    // Done, queue for rename on transaction finished
1503    TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename());
1504
1505    return;
1506 }
1507                                                                         /*}}}*/
1508 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
1509 // ---------------------------------------------------------------------
1510 /* The Translation file is added to the queue */
1511 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
1512                             string URI,string URIDesc,string ShortDesc)
1513   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashStringList())
1514 {
1515 }
1516 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
1517                                    pkgAcqMetaBase *TransactionManager,
1518                                    IndexTarget const * const Target,
1519                                    HashStringList const &ExpectedHashes,
1520                                    indexRecords *MetaIndexParser)
1521    : pkgAcqIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser)
1522 {
1523 }
1524                                                                         /*}}}*/
1525 // AcqIndexTrans::Custom600Headers - Insert custom request headers      /*{{{*/
1526 string pkgAcqIndexTrans::Custom600Headers() const
1527 {
1528    string Final = GetFinalFilename();
1529
1530    struct stat Buf;
1531    if (stat(Final.c_str(),&Buf) != 0)
1532       return "\nFail-Ignore: true\nIndex-File: true";
1533    return "\nFail-Ignore: true\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1534 }
1535                                                                         /*}}}*/
1536 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
1537 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1538 {
1539    Item::Failed(Message,Cnf);
1540
1541    size_t const nextExt = CompressionExtensions.find(' ');
1542    if (nextExt != std::string::npos)
1543    {
1544       CompressionExtensions = CompressionExtensions.substr(nextExt+1);
1545       Init(RealURI, Desc.Description, Desc.ShortDesc);
1546       Status = StatIdle;
1547       return;
1548    }
1549
1550    // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor
1551    if (Cnf->LocalOnly == true ||
1552        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1553    {
1554       // Ignore this
1555       Status = StatDone;
1556    }
1557 }
1558                                                                         /*}}}*/
1559 // AcqMetaBase::Add - Add a item to the current Transaction             /*{{{*/
1560 void pkgAcqMetaBase::Add(Item *I)
1561 {
1562    Transaction.push_back(I);
1563 }
1564                                                                         /*}}}*/
1565 // AcqMetaBase::AbortTransaction - Abort the current Transaction        /*{{{*/
1566 void pkgAcqMetaBase::AbortTransaction()
1567 {
1568    if(_config->FindB("Debug::Acquire::Transaction", false) == true)
1569       std::clog << "AbortTransaction: " << TransactionManager << std::endl;
1570
1571    // ensure the toplevel is in error state too
1572    for (std::vector<Item*>::iterator I = Transaction.begin();
1573         I != Transaction.end(); ++I)
1574    {
1575       if(_config->FindB("Debug::Acquire::Transaction", false) == true)
1576          std::clog << "  Cancel: " << (*I)->DestFile << std::endl;
1577       // the transaction will abort, so stop anything that is idle
1578       if ((*I)->Status == pkgAcquire::Item::StatIdle)
1579          (*I)->Status = pkgAcquire::Item::StatDone;
1580
1581       // reverify might act on a file outside of partial
1582       // (as it itself is good, but needed to verify others, like Release)
1583       if ((*I)->DestFile == (*I)->PartialFile && RealFileExists((*I)->DestFile))
1584          ChangeOwnerAndPermissionOfFile("AbortTransaction", (*I)->DestFile.c_str(), "root", "root", 0644);
1585    }
1586    Transaction.clear();
1587 }
1588                                                                         /*}}}*/
1589 // AcqMetaBase::TransactionHasError - Check for errors in Transaction   /*{{{*/
1590 bool pkgAcqMetaBase::TransactionHasError()
1591 {
1592    for (pkgAcquire::ItemIterator I = Transaction.begin();
1593         I != Transaction.end(); ++I)
1594       if((*I)->Status != pkgAcquire::Item::StatDone &&
1595          (*I)->Status != pkgAcquire::Item::StatIdle)
1596          return true;
1597
1598    return false;
1599 }
1600                                                                         /*}}}*/
1601 // AcqMetaBase::CommitTransaction - Commit a transaction                /*{{{*/
1602 void pkgAcqMetaBase::CommitTransaction()
1603 {
1604    if(_config->FindB("Debug::Acquire::Transaction", false) == true)
1605       std::clog << "CommitTransaction: " << this << std::endl;
1606
1607    // move new files into place *and* remove files that are not
1608    // part of the transaction but are still on disk
1609    for (std::vector<Item*>::iterator I = Transaction.begin();
1610         I != Transaction.end(); ++I)
1611    {
1612       if((*I)->PartialFile != "")
1613       {
1614          if(_config->FindB("Debug::Acquire::Transaction", false) == true)
1615             std::clog << "mv " << (*I)->PartialFile << " -> "<< (*I)->DestFile << " "
1616                << (*I)->DescURI() << std::endl;
1617
1618          Rename((*I)->PartialFile, (*I)->DestFile);
1619       } else {
1620          if(_config->FindB("Debug::Acquire::Transaction", false) == true)
1621             std::clog << "rm "
1622                       <<  (*I)->DestFile
1623                       << " "
1624                       << (*I)->DescURI()
1625                       << std::endl;
1626          unlink((*I)->DestFile.c_str());
1627       }
1628       // mark that this transaction is finished
1629       (*I)->TransactionManager = 0;
1630    }
1631    Transaction.clear();
1632 }
1633                                                                         /*}}}*/
1634 // AcqMetaBase::TransactionStageCopy - Stage a file for copying         /*{{{*/
1635 void pkgAcqMetaBase::TransactionStageCopy(Item *I,
1636                                           const std::string &From,
1637                                           const std::string &To)
1638 {
1639    I->PartialFile = From;
1640    I->DestFile = To;
1641 }
1642                                                                         /*}}}*/
1643 // AcqMetaBase::TransactionStageRemoval - Sage a file for removal       /*{{{*/
1644 void pkgAcqMetaBase::TransactionStageRemoval(Item *I,
1645                                              const std::string &FinalFile)
1646 {
1647    I->PartialFile = "";
1648    I->DestFile = FinalFile;
1649 }
1650                                                                         /*}}}*/
1651 // AcqMetaBase::GenerateAuthWarning - Check gpg authentication error    /*{{{*/
1652 bool pkgAcqMetaBase::CheckStopAuthentication(const std::string &RealURI,
1653                                              const std::string &Message)
1654 {
1655    // FIXME: this entire function can do now that we disallow going to
1656    //        a unauthenticated state and can cleanly rollback
1657
1658    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1659
1660    if(FileExists(Final))
1661    {
1662       Status = StatTransientNetworkError;
1663       _error->Warning(_("An error occurred during the signature "
1664                         "verification. The repository is not updated "
1665                         "and the previous index files will be used. "
1666                         "GPG error: %s: %s\n"),
1667                       Desc.Description.c_str(),
1668                       LookupTag(Message,"Message").c_str());
1669       RunScripts("APT::Update::Auth-Failure");
1670       return true;
1671    } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) {
1672       /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */
1673       _error->Error(_("GPG error: %s: %s"),
1674                     Desc.Description.c_str(),
1675                     LookupTag(Message,"Message").c_str());
1676       Status = StatError;
1677       return true;
1678    } else {
1679       _error->Warning(_("GPG error: %s: %s"),
1680                       Desc.Description.c_str(),
1681                       LookupTag(Message,"Message").c_str());
1682    }
1683    // gpgv method failed
1684    ReportMirrorFailure("GPGFailure");
1685    return false;
1686 }
1687                                                                         /*}}}*/
1688 // AcqMetaSig::AcqMetaSig - Constructor                                 /*{{{*/
1689 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,
1690                              pkgAcqMetaBase *TransactionManager,
1691                              string URI,string URIDesc,string ShortDesc,
1692                              string MetaIndexFile,
1693                              const vector<IndexTarget*>* IndexTargets,
1694                              indexRecords* MetaIndexParser) :
1695    pkgAcqMetaBase(Owner, IndexTargets, MetaIndexParser,
1696                   HashStringList(), TransactionManager),
1697    RealURI(URI), MetaIndexFile(MetaIndexFile), URIDesc(URIDesc),
1698    ShortDesc(ShortDesc)
1699 {
1700    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1701    DestFile += URItoFileName(RealURI);
1702
1703    // remove any partial downloaded sig-file in partial/.
1704    // it may confuse proxies and is too small to warrant a
1705    // partial download anyway
1706    unlink(DestFile.c_str());
1707
1708    // set the TransactionManager
1709    if(_config->FindB("Debug::Acquire::Transaction", false) == true)
1710       std::clog << "New pkgAcqMetaSig with TransactionManager "
1711                 << TransactionManager << std::endl;
1712
1713    // Create the item
1714    Desc.Description = URIDesc;
1715    Desc.Owner = this;
1716    Desc.ShortDesc = ShortDesc;
1717    Desc.URI = URI;
1718
1719    QueueURI(Desc);
1720 }
1721                                                                         /*}}}*/
1722 pkgAcqMetaSig::~pkgAcqMetaSig()                                         /*{{{*/
1723 {
1724 }
1725                                                                         /*}}}*/
1726 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
1727 // ---------------------------------------------------------------------
1728 string pkgAcqMetaSig::Custom600Headers() const
1729 {
1730    std::string Header = GetCustom600Headers(RealURI);
1731    return Header;
1732 }
1733                                                                         /*}}}*/
1734 // pkgAcqMetaSig::Done - The signature was downloaded/verified          /*{{{*/
1735 // ---------------------------------------------------------------------
1736 /* The only header we use is the last-modified header. */
1737 void pkgAcqMetaSig::Done(string Message,unsigned long long Size,
1738                          HashStringList const &Hashes,
1739                          pkgAcquire::MethodConfig *Cfg)
1740 {
1741    Item::Done(Message, Size, Hashes, Cfg);
1742
1743    if(AuthPass == false)
1744    {
1745       if(CheckDownloadDone(Message, RealURI) == true)
1746       {
1747          // destfile will be modified to point to MetaIndexFile for the
1748          // gpgv method, so we need to save it here
1749          MetaIndexFileSignature = DestFile;
1750          QueueForSignatureVerify(MetaIndexFile, MetaIndexFileSignature);
1751       }
1752       return;
1753    }
1754    else
1755    {
1756       if(CheckAuthDone(Message, RealURI) == true)
1757       {
1758          std::string FinalFile = _config->FindDir("Dir::State::lists");
1759          FinalFile += URItoFileName(RealURI);
1760          TransactionManager->TransactionStageCopy(this, MetaIndexFileSignature, FinalFile);
1761       }
1762    }
1763 }
1764                                                                         /*}}}*/
1765 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
1766 {
1767    Item::Failed(Message,Cnf);
1768
1769    // check if we need to fail at this point
1770    if (AuthPass == true && CheckStopAuthentication(RealURI, Message))
1771          return;
1772
1773    // FIXME: meh, this is not really elegant
1774    string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1775    string const InReleaseURI = RealURI.replace(RealURI.rfind("Release.gpg"), 12,
1776                                          "InRelease");
1777    string const FinalInRelease = _config->FindDir("Dir::State::lists") + URItoFileName(InReleaseURI);
1778
1779    if (RealFileExists(Final) || RealFileExists(FinalInRelease))
1780    {
1781       std::string downgrade_msg;
1782       strprintf(downgrade_msg, _("The repository '%s' is no longer signed."),
1783                 URIDesc.c_str());
1784       if(_config->FindB("Acquire::AllowDowngradeToInsecureRepositories"))
1785       {
1786          // meh, the users wants to take risks (we still mark the packages
1787          // from this repository as unauthenticated)
1788          _error->Warning("%s", downgrade_msg.c_str());
1789          _error->Warning(_("This is normally not allowed, but the option "
1790                            "Acquire::AllowDowngradeToInsecureRepositories was "
1791                            "given to override it."));
1792          Status = StatDone;
1793       } else {
1794          _error->Error("%s", downgrade_msg.c_str());
1795          Rename(MetaIndexFile, MetaIndexFile+".FAILED");
1796          Item::Failed("Message: " + downgrade_msg, Cnf);
1797          TransactionManager->AbortTransaction();
1798          return;
1799       }
1800    }
1801    else
1802       _error->Warning(_("The data from '%s' is not signed. Packages "
1803                "from that repository can not be authenticated."),
1804             URIDesc.c_str());
1805
1806    // this ensures that any file in the lists/ dir is removed by the
1807    // transaction
1808    DestFile = GetPartialFileNameFromURI(RealURI);
1809    TransactionManager->TransactionStageRemoval(this, DestFile);
1810
1811    // only allow going further if the users explicitely wants it
1812    if(AllowInsecureRepositories(MetaIndexParser, TransactionManager, this) == true)
1813    {
1814       // we parse the indexes here because at this point the user wanted
1815       // a repository that may potentially harm him
1816       MetaIndexParser->Load(MetaIndexFile);
1817       QueueIndexes(true);
1818    }
1819
1820    // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor
1821    if (Cnf->LocalOnly == true ||
1822        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1823    {
1824       // Ignore this
1825       Status = StatDone;
1826    }
1827 }
1828                                                                         /*}}}*/
1829 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
1830                                  pkgAcqMetaBase *TransactionManager,
1831                                  string URI,string URIDesc,string ShortDesc,
1832                                  string MetaIndexSigURI,string MetaIndexSigURIDesc, string MetaIndexSigShortDesc,
1833                                  const vector<IndexTarget*>* IndexTargets,
1834                                  indexRecords* MetaIndexParser) :
1835    pkgAcqMetaBase(Owner, IndexTargets, MetaIndexParser, HashStringList(),
1836                   TransactionManager),
1837    RealURI(URI), URIDesc(URIDesc), ShortDesc(ShortDesc),
1838    MetaIndexSigURI(MetaIndexSigURI), MetaIndexSigURIDesc(MetaIndexSigURIDesc),
1839    MetaIndexSigShortDesc(MetaIndexSigShortDesc)
1840 {
1841    if(TransactionManager == NULL)
1842    {
1843       this->TransactionManager = this;
1844       this->TransactionManager->Add(this);
1845    }
1846
1847    if(_config->FindB("Debug::Acquire::Transaction", false) == true)
1848       std::clog << "New pkgAcqMetaIndex with TransactionManager "
1849                 << this->TransactionManager << std::endl;
1850
1851
1852    Init(URIDesc, ShortDesc);
1853 }
1854                                                                         /*}}}*/
1855 // pkgAcqMetaIndex::Init - Delayed constructor                          /*{{{*/
1856 void pkgAcqMetaIndex::Init(std::string URIDesc, std::string ShortDesc)
1857 {
1858    DestFile = GetPartialFileNameFromURI(RealURI);
1859
1860    // Create the item
1861    Desc.Description = URIDesc;
1862    Desc.Owner = this;
1863    Desc.ShortDesc = ShortDesc;
1864    Desc.URI = RealURI;
1865
1866    // we expect more item
1867    ExpectedAdditionalItems = IndexTargets->size();
1868    QueueURI(Desc);
1869 }
1870                                                                         /*}}}*/
1871 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
1872 // ---------------------------------------------------------------------
1873 string pkgAcqMetaIndex::Custom600Headers() const
1874 {
1875    return GetCustom600Headers(RealURI);
1876 }
1877                                                                         /*}}}*/
1878 void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,      /*{{{*/
1879                            HashStringList const &Hashes,
1880                            pkgAcquire::MethodConfig *Cfg)
1881 {
1882    Item::Done(Message,Size,Hashes,Cfg);
1883
1884    if(CheckDownloadDone(Message, RealURI))
1885    {
1886       // we have a Release file, now download the Signature, all further
1887       // verify/queue for additional downloads will be done in the
1888       // pkgAcqMetaSig::Done() code
1889       std::string MetaIndexFile = DestFile;
1890       new pkgAcqMetaSig(Owner, TransactionManager,
1891                         MetaIndexSigURI, MetaIndexSigURIDesc,
1892                         MetaIndexSigShortDesc, MetaIndexFile, IndexTargets,
1893                         MetaIndexParser);
1894
1895       string FinalFile = _config->FindDir("Dir::State::lists");
1896       FinalFile += URItoFileName(RealURI);
1897       TransactionManager->TransactionStageCopy(this, DestFile, FinalFile);
1898    }
1899 }
1900                                                                         /*}}}*/
1901 bool pkgAcqMetaBase::CheckAuthDone(string Message, const string &RealURI)       /*{{{*/
1902 {
1903    // At this point, the gpgv method has succeeded, so there is a
1904    // valid signature from a key in the trusted keyring.  We
1905    // perform additional verification of its contents, and use them
1906    // to verify the indexes we are about to download
1907
1908    if (!MetaIndexParser->Load(DestFile))
1909    {
1910       Status = StatAuthError;
1911       ErrorText = MetaIndexParser->ErrorText;
1912       return false;
1913    }
1914
1915    if (!VerifyVendor(Message, RealURI))
1916    {
1917       return false;
1918    }
1919
1920    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1921       std::cerr << "Signature verification succeeded: "
1922                 << DestFile << std::endl;
1923
1924    // Download further indexes with verification
1925    //
1926    // it would be really nice if we could simply do
1927    //    if (IMSHit == false) QueueIndexes(true)
1928    // and skip the download if the Release file has not changed
1929    // - but right now the list cleaner will needs to be tricked
1930    //   to not delete all our packages/source indexes in this case
1931    QueueIndexes(true);
1932
1933    return true;
1934 }
1935                                                                         /*}}}*/
1936 // pkgAcqMetaBase::GetCustom600Headers - Get header for AcqMetaBase     /*{{{*/
1937 // ---------------------------------------------------------------------
1938 string pkgAcqMetaBase::GetCustom600Headers(const string &RealURI) const
1939 {
1940    std::string Header = "\nIndex-File: true";
1941    std::string MaximumSize;
1942    strprintf(MaximumSize, "\nMaximum-Size: %i",
1943              _config->FindI("Acquire::MaxReleaseFileSize", 10*1000*1000));
1944    Header += MaximumSize;
1945
1946    string FinalFile = _config->FindDir("Dir::State::lists");
1947    FinalFile += URItoFileName(RealURI);
1948
1949    struct stat Buf;
1950    if (stat(FinalFile.c_str(),&Buf) == 0)
1951       Header += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1952
1953    return Header;
1954 }
1955                                                                         /*}}}*/
1956 // pkgAcqMetaBase::QueueForSignatureVerify                              /*{{{*/
1957 void pkgAcqMetaBase::QueueForSignatureVerify(const std::string &MetaIndexFile,
1958                                     const std::string &MetaIndexFileSignature)
1959 {
1960    AuthPass = true;
1961    Desc.URI = "gpgv:" + MetaIndexFileSignature;
1962    DestFile = MetaIndexFile;
1963    QueueURI(Desc);
1964    SetActiveSubprocess("gpgv");
1965 }
1966                                                                         /*}}}*/
1967 // pkgAcqMetaBase::CheckDownloadDone                                    /*{{{*/
1968 bool pkgAcqMetaBase::CheckDownloadDone(const std::string &Message,
1969                                        const std::string &RealURI)
1970 {
1971    // We have just finished downloading a Release file (it is not
1972    // verified yet)
1973
1974    string FileName = LookupTag(Message,"Filename");
1975    if (FileName.empty() == true)
1976    {
1977       Status = StatError;
1978       ErrorText = "Method gave a blank filename";
1979       return false;
1980    }
1981
1982    if (FileName != DestFile)
1983    {
1984       Local = true;
1985       Desc.URI = "copy:" + FileName;
1986       QueueURI(Desc);
1987       return false;
1988    }
1989
1990    // make sure to verify against the right file on I-M-S hit
1991    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1992    if(IMSHit)
1993    {
1994       string FinalFile = _config->FindDir("Dir::State::lists");
1995       FinalFile += URItoFileName(RealURI);
1996       DestFile = FinalFile;
1997    }
1998
1999    // set Item to complete as the remaining work is all local (verify etc)
2000    Complete = true;
2001
2002    return true;
2003 }
2004                                                                         /*}}}*/
2005 void pkgAcqMetaBase::QueueIndexes(bool verify)                          /*{{{*/
2006 {
2007    bool transInRelease = false;
2008    {
2009       std::vector<std::string> const keys = MetaIndexParser->MetaKeys();
2010       for (std::vector<std::string>::const_iterator k = keys.begin(); k != keys.end(); ++k)
2011          // FIXME: Feels wrong to check for hardcoded string here, but what should we do else…
2012          if (k->find("Translation-") != std::string::npos)
2013          {
2014             transInRelease = true;
2015             break;
2016          }
2017    }
2018
2019    // at this point the real Items are loaded in the fetcher
2020    ExpectedAdditionalItems = 0;
2021    for (vector <IndexTarget*>::const_iterator Target = IndexTargets->begin();
2022         Target != IndexTargets->end();
2023         ++Target)
2024    {
2025       HashStringList ExpectedIndexHashes;
2026       const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
2027       bool compressedAvailable = false;
2028       if (Record == NULL)
2029       {
2030          if ((*Target)->IsOptional() == true)
2031          {
2032             std::vector<std::string> types = APT::Configuration::getCompressionTypes();
2033             for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
2034                if (MetaIndexParser->Exists((*Target)->MetaKey + "." + *t) == true)
2035                {
2036                   compressedAvailable = true;
2037                   break;
2038                }
2039          }
2040          else if (verify == true)
2041          {
2042             Status = StatAuthError;
2043             strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), (*Target)->MetaKey.c_str());
2044             return;
2045          }
2046       }
2047       else
2048       {
2049          ExpectedIndexHashes = Record->Hashes;
2050          if (_config->FindB("Debug::pkgAcquire::Auth", false))
2051          {
2052             std::cerr << "Queueing: " << (*Target)->URI << std::endl
2053                << "Expected Hash:" << std::endl;
2054             for (HashStringList::const_iterator hs = ExpectedIndexHashes.begin(); hs != ExpectedIndexHashes.end(); ++hs)
2055                std::cerr <<  "\t- " << hs->toStr() << std::endl;
2056             std::cerr << "For: " << Record->MetaKeyFilename << std::endl;
2057          }
2058          if (verify == true && ExpectedIndexHashes.empty() == true && (*Target)->IsOptional() == false)
2059          {
2060             Status = StatAuthError;
2061             strprintf(ErrorText, _("Unable to find hash sum for '%s' in Release file"), (*Target)->MetaKey.c_str());
2062             return;
2063          }
2064       }
2065
2066       if ((*Target)->IsOptional() == true)
2067       {
2068          if (transInRelease == false || Record != NULL || compressedAvailable == true)
2069          {
2070             if (_config->FindB("Acquire::PDiffs",true) == true && transInRelease == true &&
2071                 MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true)
2072                new pkgAcqDiffIndex(Owner, TransactionManager, *Target, ExpectedIndexHashes, MetaIndexParser);
2073             else
2074                new pkgAcqIndexTrans(Owner, TransactionManager, *Target, ExpectedIndexHashes, MetaIndexParser);
2075          }
2076          continue;
2077       }
2078
2079       /* Queue Packages file (either diff or full packages files, depending
2080          on the users option) - we also check if the PDiff Index file is listed
2081          in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
2082          instead, but passing the required info to it is to much hassle */
2083       if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
2084           MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true))
2085          new pkgAcqDiffIndex(Owner, TransactionManager, *Target, ExpectedIndexHashes, MetaIndexParser);
2086       else
2087          new pkgAcqIndex(Owner, TransactionManager, *Target, ExpectedIndexHashes, MetaIndexParser);
2088    }
2089 }
2090                                                                         /*}}}*/
2091 bool pkgAcqMetaBase::VerifyVendor(string Message, const string &RealURI)/*{{{*/
2092 {
2093    string::size_type pos;
2094
2095    // check for missing sigs (that where not fatal because otherwise we had
2096    // bombed earlier)
2097    string missingkeys;
2098    string msg = _("There is no public key available for the "
2099                   "following key IDs:\n");
2100    pos = Message.find("NO_PUBKEY ");
2101    if (pos != std::string::npos)
2102    {
2103       string::size_type start = pos+strlen("NO_PUBKEY ");
2104       string Fingerprint = Message.substr(start, Message.find("\n")-start);
2105       missingkeys += (Fingerprint);
2106    }
2107    if(!missingkeys.empty())
2108       _error->Warning("%s", (msg + missingkeys).c_str());
2109
2110    string Transformed = MetaIndexParser->GetExpectedDist();
2111
2112    if (Transformed == "../project/experimental")
2113    {
2114       Transformed = "experimental";
2115    }
2116
2117    pos = Transformed.rfind('/');
2118    if (pos != string::npos)
2119    {
2120       Transformed = Transformed.substr(0, pos);
2121    }
2122
2123    if (Transformed == ".")
2124    {
2125       Transformed = "";
2126    }
2127
2128    if (_config->FindB("Acquire::Check-Valid-Until", true) == true &&
2129        MetaIndexParser->GetValidUntil() > 0) {
2130       time_t const invalid_since = time(NULL) - MetaIndexParser->GetValidUntil();
2131       if (invalid_since > 0)
2132          // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
2133          // the time since then the file is invalid - formated in the same way as in
2134          // the download progress display (e.g. 7d 3h 42min 1s)
2135          return _error->Error(
2136             _("Release file for %s is expired (invalid since %s). "
2137               "Updates for this repository will not be applied."),
2138             RealURI.c_str(), TimeToStr(invalid_since).c_str());
2139    }
2140
2141    if (_config->FindB("Debug::pkgAcquire::Auth", false))
2142    {
2143       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
2144       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
2145       std::cerr << "Transformed Dist: " << Transformed << std::endl;
2146    }
2147
2148    if (MetaIndexParser->CheckDist(Transformed) == false)
2149    {
2150       // This might become fatal one day
2151 //       Status = StatAuthError;
2152 //       ErrorText = "Conflicting distribution; expected "
2153 //          + MetaIndexParser->GetExpectedDist() + " but got "
2154 //          + MetaIndexParser->GetDist();
2155 //       return false;
2156       if (!Transformed.empty())
2157       {
2158          _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
2159                          Desc.Description.c_str(),
2160                          Transformed.c_str(),
2161                          MetaIndexParser->GetDist().c_str());
2162       }
2163    }
2164
2165    return true;
2166 }
2167                                                                         /*}}}*/
2168 // pkgAcqMetaIndex::Failed - no Release file present                    /*{{{*/
2169 void pkgAcqMetaIndex::Failed(string Message,
2170                              pkgAcquire::MethodConfig * Cnf)
2171 {
2172    pkgAcquire::Item::Failed(Message, Cnf);
2173    Status = StatDone;
2174
2175    string FinalFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
2176
2177    _error->Warning(_("The repository '%s' does not have a Release file. "
2178                      "This is deprecated, please contact the owner of the "
2179                      "repository."), URIDesc.c_str());
2180
2181    // No Release file was present so fall
2182    // back to queueing Packages files without verification
2183    // only allow going further if the users explicitely wants it
2184    if(AllowInsecureRepositories(MetaIndexParser, TransactionManager, this) == true)
2185    {
2186       // Done, queue for rename on transaction finished
2187       if (FileExists(DestFile))
2188          TransactionManager->TransactionStageCopy(this, DestFile, FinalFile);
2189
2190       // queue without any kind of hashsum support
2191       QueueIndexes(false);
2192    }
2193 }
2194                                                                         /*}}}*/
2195 void pkgAcqMetaIndex::Finished()                                        /*{{{*/
2196 {
2197    if(_config->FindB("Debug::Acquire::Transaction", false) == true)
2198       std::clog << "Finished: " << DestFile <<std::endl;
2199    if(TransactionManager != NULL &&
2200       TransactionManager->TransactionHasError() == false)
2201       TransactionManager->CommitTransaction();
2202 }
2203                                                                         /*}}}*/
2204 pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner,               /*{{{*/
2205                 string const &URI, string const &URIDesc, string const &ShortDesc,
2206                 string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc,
2207                 string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc,
2208                 const vector<IndexTarget*>* IndexTargets,
2209                 indexRecords* MetaIndexParser) :
2210    pkgAcqMetaIndex(Owner, NULL, URI, URIDesc, ShortDesc, MetaSigURI, MetaSigURIDesc,MetaSigShortDesc, IndexTargets, MetaIndexParser),
2211        MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
2212        MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc)
2213 {
2214    // index targets + (worst case:) Release/Release.gpg
2215    ExpectedAdditionalItems = IndexTargets->size() + 2;
2216
2217 }
2218                                                                         /*}}}*/
2219 pkgAcqMetaClearSig::~pkgAcqMetaClearSig()                               /*{{{*/
2220 {
2221 }
2222                                                                         /*}}}*/
2223 // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/
2224 // ---------------------------------------------------------------------
2225 string pkgAcqMetaClearSig::Custom600Headers() const
2226 {
2227    string Header = GetCustom600Headers(RealURI);
2228    Header += "\nFail-Ignore: true";
2229    return Header;
2230 }
2231                                                                         /*}}}*/
2232 // pkgAcqMetaClearSig::Done - We got a file                             /*{{{*/
2233 // ---------------------------------------------------------------------
2234 void pkgAcqMetaClearSig::Done(std::string Message,unsigned long long Size,
2235                               HashStringList const &Hashes,
2236                               pkgAcquire::MethodConfig *Cnf)
2237 {
2238    Item::Done(Message, Size, Hashes, Cnf);
2239
2240    // if we expect a ClearTextSignature (InRelase), ensure that
2241    // this is what we get and if not fail to queue a
2242    // Release/Release.gpg, see #346386
2243    if (FileExists(DestFile) && !StartsWithGPGClearTextSignature(DestFile))
2244    {
2245       pkgAcquire::Item::Failed(Message, Cnf);
2246       RenameOnError(NotClearsigned);
2247       TransactionManager->AbortTransaction();
2248       return;
2249    }
2250
2251    if(AuthPass == false)
2252    {
2253       if(CheckDownloadDone(Message, RealURI) == true)
2254          QueueForSignatureVerify(DestFile, DestFile);
2255       return;
2256    }
2257    else
2258    {
2259       if(CheckAuthDone(Message, RealURI) == true)
2260       {
2261          string FinalFile = _config->FindDir("Dir::State::lists");
2262          FinalFile += URItoFileName(RealURI);
2263
2264          // queue for copy in place
2265          TransactionManager->TransactionStageCopy(this, DestFile, FinalFile);
2266       }
2267    }
2268 }
2269                                                                         /*}}}*/
2270 void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/
2271 {
2272    Item::Failed(Message, Cnf);
2273
2274    // we failed, we will not get additional items from this method
2275    ExpectedAdditionalItems = 0;
2276
2277    if (AuthPass == false)
2278    {
2279       // Queue the 'old' InRelease file for removal if we try Release.gpg
2280       // as otherwise the file will stay around and gives a false-auth
2281       // impression (CVE-2012-0214)
2282       string FinalFile = _config->FindDir("Dir::State::lists");
2283       FinalFile.append(URItoFileName(RealURI));
2284       TransactionManager->TransactionStageRemoval(this, FinalFile);
2285       Status = StatDone;
2286
2287       new pkgAcqMetaIndex(Owner, TransactionManager,
2288                         MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
2289                         MetaSigURI, MetaSigURIDesc, MetaSigShortDesc,
2290                         IndexTargets, MetaIndexParser);
2291    }
2292    else
2293    {
2294       if(CheckStopAuthentication(RealURI, Message))
2295          return;
2296
2297       _error->Warning(_("The data from '%s' is not signed. Packages "
2298                         "from that repository can not be authenticated."),
2299                       URIDesc.c_str());
2300
2301       // No Release file was present, or verification failed, so fall
2302       // back to queueing Packages files without verification
2303       // only allow going further if the users explicitely wants it
2304       if(AllowInsecureRepositories(MetaIndexParser, TransactionManager, this) == true)
2305       {
2306          Status = StatDone;
2307
2308          /* Always move the meta index, even if gpgv failed. This ensures
2309           * that PackageFile objects are correctly filled in */
2310          if (FileExists(DestFile))
2311          {
2312             string FinalFile = _config->FindDir("Dir::State::lists");
2313             FinalFile += URItoFileName(RealURI);
2314             /* InRelease files become Release files, otherwise
2315              * they would be considered as trusted later on */
2316             RealURI = RealURI.replace(RealURI.rfind("InRelease"), 9,
2317                                       "Release");
2318             FinalFile = FinalFile.replace(FinalFile.rfind("InRelease"), 9,
2319                                           "Release");
2320
2321             // Done, queue for rename on transaction finished
2322             TransactionManager->TransactionStageCopy(this, DestFile, FinalFile);
2323          }
2324          QueueIndexes(false);
2325       }
2326    }
2327 }
2328                                                                         /*}}}*/
2329 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
2330 // ---------------------------------------------------------------------
2331 /* This just sets up the initial fetch environment and queues the first
2332    possibilitiy */
2333 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
2334                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
2335                              string &StoreFilename) :
2336                Item(Owner, HashStringList()), Version(Version), Sources(Sources), Recs(Recs),
2337                StoreFilename(StoreFilename), Vf(Version.FileList()),
2338                Trusted(false)
2339 {
2340    Retries = _config->FindI("Acquire::Retries",0);
2341
2342    if (Version.Arch() == 0)
2343    {
2344       _error->Error(_("I wasn't able to locate a file for the %s package. "
2345                       "This might mean you need to manually fix this package. "
2346                       "(due to missing arch)"),
2347                     Version.ParentPkg().FullName().c_str());
2348       return;
2349    }
2350
2351    /* We need to find a filename to determine the extension. We make the
2352       assumption here that all the available sources for this version share
2353       the same extension.. */
2354    // Skip not source sources, they do not have file fields.
2355    for (; Vf.end() == false; ++Vf)
2356    {
2357       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
2358          continue;
2359       break;
2360    }
2361
2362    // Does not really matter here.. we are going to fail out below
2363    if (Vf.end() != true)
2364    {
2365       // If this fails to get a file name we will bomb out below.
2366       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
2367       if (_error->PendingError() == true)
2368          return;
2369
2370       // Generate the final file name as: package_version_arch.foo
2371       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
2372                       QuoteString(Version.VerStr(),"_:") + '_' +
2373                       QuoteString(Version.Arch(),"_:.") +
2374                       "." + flExtension(Parse.FileName());
2375    }
2376
2377    // check if we have one trusted source for the package. if so, switch
2378    // to "TrustedOnly" mode - but only if not in AllowUnauthenticated mode
2379    bool const allowUnauth = _config->FindB("APT::Get::AllowUnauthenticated", false);
2380    bool const debugAuth = _config->FindB("Debug::pkgAcquire::Auth", false);
2381    bool seenUntrusted = false;
2382    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; ++i)
2383    {
2384       pkgIndexFile *Index;
2385       if (Sources->FindIndex(i.File(),Index) == false)
2386          continue;
2387
2388       if (debugAuth == true)
2389          std::cerr << "Checking index: " << Index->Describe()
2390                    << "(Trusted=" << Index->IsTrusted() << ")" << std::endl;
2391
2392       if (Index->IsTrusted() == true)
2393       {
2394          Trusted = true;
2395          if (allowUnauth == false)
2396             break;
2397       }
2398       else
2399          seenUntrusted = true;
2400    }
2401
2402    // "allow-unauthenticated" restores apts old fetching behaviour
2403    // that means that e.g. unauthenticated file:// uris are higher
2404    // priority than authenticated http:// uris
2405    if (allowUnauth == true && seenUntrusted == true)
2406       Trusted = false;
2407
2408    // Select a source
2409    if (QueueNext() == false && _error->PendingError() == false)
2410       _error->Error(_("Can't find a source to download version '%s' of '%s'"),
2411                     Version.VerStr(), Version.ParentPkg().FullName(false).c_str());
2412 }
2413                                                                         /*}}}*/
2414 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
2415 // ---------------------------------------------------------------------
2416 /* This queues the next available file version for download. It checks if
2417    the archive is already available in the cache and stashs the MD5 for
2418    checking later. */
2419 bool pkgAcqArchive::QueueNext()
2420 {
2421    for (; Vf.end() == false; ++Vf)
2422    {
2423       // Ignore not source sources
2424       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
2425          continue;
2426
2427       // Try to cross match against the source list
2428       pkgIndexFile *Index;
2429       if (Sources->FindIndex(Vf.File(),Index) == false)
2430             continue;
2431
2432       // only try to get a trusted package from another source if that source
2433       // is also trusted
2434       if(Trusted && !Index->IsTrusted())
2435          continue;
2436
2437       // Grab the text package record
2438       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
2439       if (_error->PendingError() == true)
2440          return false;
2441
2442       string PkgFile = Parse.FileName();
2443       ExpectedHashes = Parse.Hashes();
2444
2445       if (PkgFile.empty() == true)
2446          return _error->Error(_("The package index files are corrupted. No Filename: "
2447                               "field for package %s."),
2448                               Version.ParentPkg().Name());
2449
2450       Desc.URI = Index->ArchiveURI(PkgFile);
2451       Desc.Description = Index->ArchiveInfo(Version);
2452       Desc.Owner = this;
2453       Desc.ShortDesc = Version.ParentPkg().FullName(true);
2454
2455       // See if we already have the file. (Legacy filenames)
2456       FileSize = Version->Size;
2457       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
2458       struct stat Buf;
2459       if (stat(FinalFile.c_str(),&Buf) == 0)
2460       {
2461          // Make sure the size matches
2462          if ((unsigned long long)Buf.st_size == Version->Size)
2463          {
2464             Complete = true;
2465             Local = true;
2466             Status = StatDone;
2467             StoreFilename = DestFile = FinalFile;
2468             return true;
2469          }
2470
2471          /* Hmm, we have a file and its size does not match, this means it is
2472             an old style mismatched arch */
2473          unlink(FinalFile.c_str());
2474       }
2475
2476       // Check it again using the new style output filenames
2477       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
2478       if (stat(FinalFile.c_str(),&Buf) == 0)
2479       {
2480          // Make sure the size matches
2481          if ((unsigned long long)Buf.st_size == Version->Size)
2482          {
2483             Complete = true;
2484             Local = true;
2485             Status = StatDone;
2486             StoreFilename = DestFile = FinalFile;
2487             return true;
2488          }
2489
2490          /* Hmm, we have a file and its size does not match, this shouldn't
2491             happen.. */
2492          unlink(FinalFile.c_str());
2493       }
2494
2495       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
2496
2497       // Check the destination file
2498       if (stat(DestFile.c_str(),&Buf) == 0)
2499       {
2500          // Hmm, the partial file is too big, erase it
2501          if ((unsigned long long)Buf.st_size > Version->Size)
2502             unlink(DestFile.c_str());
2503          else
2504          {
2505             PartialSize = Buf.st_size;
2506             std::string SandboxUser = _config->Find("APT::Sandbox::User");
2507             ChangeOwnerAndPermissionOfFile("pkgAcqArchive::QueueNext",DestFile.c_str(), SandboxUser.c_str(), "root", 0600);
2508          }
2509       }
2510
2511       // Disables download of archives - useful if no real installation follows,
2512       // e.g. if we are just interested in proposed installation order
2513       if (_config->FindB("Debug::pkgAcqArchive::NoQueue", false) == true)
2514       {
2515          Complete = true;
2516          Local = true;
2517          Status = StatDone;
2518          StoreFilename = DestFile = FinalFile;
2519          return true;
2520       }
2521
2522       // Create the item
2523       Local = false;
2524       QueueURI(Desc);
2525
2526       ++Vf;
2527       return true;
2528    }
2529    return false;
2530 }
2531                                                                         /*}}}*/
2532 // AcqArchive::Done - Finished fetching                                 /*{{{*/
2533 // ---------------------------------------------------------------------
2534 /* */
2535 void pkgAcqArchive::Done(string Message,unsigned long long Size, HashStringList const &CalcHashes,
2536                          pkgAcquire::MethodConfig *Cfg)
2537 {
2538    Item::Done(Message, Size, CalcHashes, Cfg);
2539
2540    // Check the size
2541    if (Size != Version->Size)
2542    {
2543       RenameOnError(SizeMismatch);
2544       return;
2545    }
2546
2547    // FIXME: could this empty() check impose *any* sort of security issue?
2548    if(ExpectedHashes.usable() && ExpectedHashes != CalcHashes)
2549    {
2550       RenameOnError(HashSumMismatch);
2551       printHashSumComparision(DestFile, ExpectedHashes, CalcHashes);
2552       return;
2553    }
2554
2555    // Grab the output filename
2556    string FileName = LookupTag(Message,"Filename");
2557    if (FileName.empty() == true)
2558    {
2559       Status = StatError;
2560       ErrorText = "Method gave a blank filename";
2561       return;
2562    }
2563
2564    // Reference filename
2565    if (FileName != DestFile)
2566    {
2567       StoreFilename = DestFile = FileName;
2568       Local = true;
2569       Complete = true;
2570       return;
2571    }
2572
2573    // Done, move it into position
2574    string FinalFile = _config->FindDir("Dir::Cache::Archives");
2575    FinalFile += flNotDir(StoreFilename);
2576    Rename(DestFile,FinalFile);
2577    StoreFilename = DestFile = FinalFile;
2578    Complete = true;
2579 }
2580                                                                         /*}}}*/
2581 // AcqArchive::Failed - Failure handler                                 /*{{{*/
2582 // ---------------------------------------------------------------------
2583 /* Here we try other sources */
2584 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2585 {
2586    Item::Failed(Message,Cnf);
2587
2588    /* We don't really want to retry on failed media swaps, this prevents
2589       that. An interesting observation is that permanent failures are not
2590       recorded. */
2591    if (Cnf->Removable == true &&
2592        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2593    {
2594       // Vf = Version.FileList();
2595       while (Vf.end() == false) ++Vf;
2596       StoreFilename = string();
2597       return;
2598    }
2599
2600    Status = StatIdle;
2601    if (QueueNext() == false)
2602    {
2603       // This is the retry counter
2604       if (Retries != 0 &&
2605           Cnf->LocalOnly == false &&
2606           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2607       {
2608          Retries--;
2609          Vf = Version.FileList();
2610          if (QueueNext() == true)
2611             return;
2612       }
2613
2614       StoreFilename = string();
2615       Status = StatError;
2616    }
2617 }
2618                                                                         /*}}}*/
2619 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
2620 // ---------------------------------------------------------------------
2621 APT_PURE bool pkgAcqArchive::IsTrusted() const
2622 {
2623    return Trusted;
2624 }
2625                                                                         /*}}}*/
2626 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
2627 // ---------------------------------------------------------------------
2628 /* */
2629 void pkgAcqArchive::Finished()
2630 {
2631    if (Status == pkgAcquire::Item::StatDone &&
2632        Complete == true)
2633       return;
2634    StoreFilename = string();
2635 }
2636                                                                         /*}}}*/
2637 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
2638 // ---------------------------------------------------------------------
2639 /* The file is added to the queue */
2640 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI, HashStringList const &Hashes,
2641                        unsigned long long Size,string Dsc,string ShortDesc,
2642                        const string &DestDir, const string &DestFilename,
2643                        bool IsIndexFile) :
2644                        Item(Owner, Hashes), IsIndexFile(IsIndexFile)
2645 {
2646    Retries = _config->FindI("Acquire::Retries",0);
2647
2648    if(!DestFilename.empty())
2649       DestFile = DestFilename;
2650    else if(!DestDir.empty())
2651       DestFile = DestDir + "/" + flNotDir(URI);
2652    else
2653       DestFile = flNotDir(URI);
2654
2655    // Create the item
2656    Desc.URI = URI;
2657    Desc.Description = Dsc;
2658    Desc.Owner = this;
2659
2660    // Set the short description to the archive component
2661    Desc.ShortDesc = ShortDesc;
2662
2663    // Get the transfer sizes
2664    FileSize = Size;
2665    struct stat Buf;
2666    if (stat(DestFile.c_str(),&Buf) == 0)
2667    {
2668       // Hmm, the partial file is too big, erase it
2669       if ((Size > 0) && (unsigned long long)Buf.st_size > Size)
2670          unlink(DestFile.c_str());
2671       else
2672       {
2673          PartialSize = Buf.st_size;
2674          std::string SandboxUser = _config->Find("APT::Sandbox::User");
2675          ChangeOwnerAndPermissionOfFile("pkgAcqFile", DestFile.c_str(), SandboxUser.c_str(), "root", 0600);
2676       }
2677    }
2678
2679    QueueURI(Desc);
2680 }
2681                                                                         /*}}}*/
2682 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
2683 // ---------------------------------------------------------------------
2684 /* */
2685 void pkgAcqFile::Done(string Message,unsigned long long Size,HashStringList const &CalcHashes,
2686                       pkgAcquire::MethodConfig *Cnf)
2687 {
2688    Item::Done(Message,Size,CalcHashes,Cnf);
2689
2690    // Check the hash
2691    if(ExpectedHashes.usable() && ExpectedHashes != CalcHashes)
2692    {
2693       RenameOnError(HashSumMismatch);
2694       printHashSumComparision(DestFile, ExpectedHashes, CalcHashes);
2695       return;
2696    }
2697
2698    string FileName = LookupTag(Message,"Filename");
2699    if (FileName.empty() == true)
2700    {
2701       Status = StatError;
2702       ErrorText = "Method gave a blank filename";
2703       return;
2704    }
2705
2706    Complete = true;
2707
2708    // The files timestamp matches
2709    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
2710       return;
2711
2712    // We have to copy it into place
2713    if (FileName != DestFile)
2714    {
2715       Local = true;
2716       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
2717           Cnf->Removable == true)
2718       {
2719          Desc.URI = "copy:" + FileName;
2720          QueueURI(Desc);
2721          return;
2722       }
2723
2724       // Erase the file if it is a symlink so we can overwrite it
2725       struct stat St;
2726       if (lstat(DestFile.c_str(),&St) == 0)
2727       {
2728          if (S_ISLNK(St.st_mode) != 0)
2729             unlink(DestFile.c_str());
2730       }
2731
2732       // Symlink the file
2733       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
2734       {
2735          _error->PushToStack();
2736          _error->Errno("pkgAcqFile::Done", "Symlinking file %s failed", DestFile.c_str());
2737          std::stringstream msg;
2738          _error->DumpErrors(msg);
2739          _error->RevertToStack();
2740          ErrorText = msg.str();
2741          Status = StatError;
2742          Complete = false;
2743       }
2744    }
2745 }
2746                                                                         /*}}}*/
2747 // AcqFile::Failed - Failure handler                                    /*{{{*/
2748 // ---------------------------------------------------------------------
2749 /* Here we try other sources */
2750 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2751 {
2752    Item::Failed(Message,Cnf);
2753
2754    // This is the retry counter
2755    if (Retries != 0 &&
2756        Cnf->LocalOnly == false &&
2757        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2758    {
2759       --Retries;
2760       QueueURI(Desc);
2761       Status = StatIdle;
2762       return;
2763    }
2764
2765 }
2766                                                                         /*}}}*/
2767 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
2768 // ---------------------------------------------------------------------
2769 /* The only header we use is the last-modified header. */
2770 string pkgAcqFile::Custom600Headers() const
2771 {
2772    if (IsIndexFile)
2773       return "\nIndex-File: true";
2774    return "";
2775 }
2776                                                                         /*}}}*/