apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <config.h>
  17
  18 #include <apt-pkg/acquire-item.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/aptconfiguration.h>
  21 #include <apt-pkg/sourcelist.h>
  22 #include <apt-pkg/error.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/md5.h>
  26 #include <apt-pkg/sha1.h>
  27 #include <apt-pkg/tagfile.h>
  28 #include <apt-pkg/indexrecords.h>
  29 #include <apt-pkg/metaindex.h>
  30
  31 #include <sys/stat.h>
  32 #include <unistd.h>
  33 #include <errno.h>
  34 #include <string>
  35 #include <sstream>
  36 #include <stdio.h>
  37 #include <ctime>
  38
  39 #include <apti18n.h>
  40                                                                         /*}}}*/
  41
  42 using namespace std;
  43
  44 // Acquire::Item::Item - Constructor                                    /*{{{*/
  45 // ---------------------------------------------------------------------
  46 /* */
  47 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  48                        PartialSize(0), Mode(0), ID(0), Complete(false),
  49                        Local(false), QueueCounter(0)
  50 {
  51    Owner->Add(this);
  52    Status = StatIdle;
  53 }
  54                                                                         /*}}}*/
  55 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  56 // ---------------------------------------------------------------------
  57 /* */
  58 pkgAcquire::Item::~Item()
  59 {
  60    Owner->Remove(this);
  61 }
  62                                                                         /*}}}*/
  63 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  64 // ---------------------------------------------------------------------
  65 /* We return to an idle state if there are still other queues that could
  66    fetch this object */
  67 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  68 {
  69    Status = StatIdle;
  70    ErrorText = LookupTag(Message,"Message");
  71    UsedMirror =  LookupTag(Message,"UsedMirror");
  72    if (QueueCounter <= 1)
  73    {
  74       /* This indicates that the file is not available right now but might
  75          be sometime later. If we do a retry cycle then this should be
  76          retried [CDROMs] */
  77       if (Cnf->LocalOnly == true &&
  78           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  79       {
  80          Status = StatIdle;
  81          Dequeue();
  82          return;
  83       }
  84
  85       Status = StatError;
  86       Dequeue();
  87    }
  88
  89    // report mirror failure back to LP if we actually use a mirror
  90    string FailReason = LookupTag(Message, "FailReason");
  91    if(FailReason.size() != 0)
  92       ReportMirrorFailure(FailReason);
  93    else
  94       ReportMirrorFailure(ErrorText);
  95 }
  96                                                                         /*}}}*/
  97 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
  98 // ---------------------------------------------------------------------
  99 /* Stash status and the file size. Note that setting Complete means
 100    sub-phases of the acquire process such as decompresion are operating */
 101 void pkgAcquire::Item::Start(string /*Message*/,unsigned long long Size)
 102 {
 103    Status = StatFetching;
 104    if (FileSize == 0 && Complete == false)
 105       FileSize = Size;
 106 }
 107                                                                         /*}}}*/
 108 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
 109 // ---------------------------------------------------------------------
 110 /* */
 111 void pkgAcquire::Item::Done(string Message,unsigned long long Size,string Hash,
 112                             pkgAcquire::MethodConfig *Cnf)
 113 {
 114    // We just downloaded something..
 115    string FileName = LookupTag(Message,"Filename");
 116    UsedMirror =  LookupTag(Message,"UsedMirror");
 117    if (Complete == false && !Local && FileName == DestFile)
 118    {
 119       if (Owner->Log != 0)
 120          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 121    }
 122
 123    if (FileSize == 0)
 124       FileSize= Size;
 125    Status = StatDone;
 126    ErrorText = string();
 127    Owner->Dequeue(this);
 128 }
 129                                                                         /*}}}*/
 130 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 131 // ---------------------------------------------------------------------
 132 /* This helper function is used by alot of item methods as thier final
 133    step */
 134 void pkgAcquire::Item::Rename(string From,string To)
 135 {
 136    if (rename(From.c_str(),To.c_str()) != 0)
 137    {
 138       char S[300];
 139       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 140               From.c_str(),To.c_str());
 141       Status = StatError;
 142       ErrorText = S;
 143    }
 144 }
 145                                                                         /*}}}*/
 146 bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const error)/*{{{*/
 147 {
 148    if(FileExists(DestFile))
 149       Rename(DestFile, DestFile + ".FAILED");
 150
 151    switch (error)
 152    {
 153       case HashSumMismatch:
 154          ErrorText = _("Hash Sum mismatch");
 155          Status = StatAuthError;
 156          ReportMirrorFailure("HashChecksumFailure");
 157          break;
 158       case SizeMismatch:
 159          ErrorText = _("Size mismatch");
 160          Status = StatAuthError;
 161          ReportMirrorFailure("SizeFailure");
 162          break;
 163       case InvalidFormat:
 164          ErrorText = _("Invalid file format");
 165          Status = StatError;
 166          // do not report as usually its not the mirrors fault, but Portal/Proxy
 167          break;
 168    }
 169    return false;
 170 }
 171                                                                         /*}}}*/
 172 // Acquire::Item::ReportMirrorFailure                                   /*{{{*/
 173 // ---------------------------------------------------------------------
 174 void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
 175 {
 176    // we only act if a mirror was used at all
 177    if(UsedMirror.empty())
 178       return;
 179 #if 0
 180    std::cerr << "\nReportMirrorFailure: "
 181              << UsedMirror
 182              << " Uri: " << DescURI()
 183              << " FailCode: "
 184              << FailCode << std::endl;
 185 #endif
 186    const char *Args[40];
 187    unsigned int i = 0;
 188    string report = _config->Find("Methods::Mirror::ProblemReporting",
 189                                  "/usr/lib/apt/apt-report-mirror-failure");
 190    if(!FileExists(report))
 191       return;
 192    Args[i++] = report.c_str();
 193    Args[i++] = UsedMirror.c_str();
 194    Args[i++] = DescURI().c_str();
 195    Args[i++] = FailCode.c_str();
 196    Args[i++] = NULL;
 197    pid_t pid = ExecFork();
 198    if(pid < 0)
 199    {
 200       _error->Error("ReportMirrorFailure Fork failed");
 201       return;
 202    }
 203    else if(pid == 0)
 204    {
 205       execvp(Args[0], (char**)Args);
 206       std::cerr << "Could not exec " << Args[0] << std::endl;
 207       _exit(100);
 208    }
 209    if(!ExecWait(pid, "report-mirror-failure"))
 210    {
 211       _error->Warning("Couldn't report problem to '%s'",
 212                       _config->Find("Methods::Mirror::ProblemReporting").c_str());
 213    }
 214 }
 215                                                                         /*}}}*/
 216 // AcqSubIndex::AcqSubIndex - Constructor                               /*{{{*/
 217 // ---------------------------------------------------------------------
 218 /* Get a sub-index file based on checksums from a 'master' file and
 219    possibly query additional files */
 220 pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, string const &URI,
 221                                  string const &URIDesc, string const &ShortDesc,
 222                                  HashString const &ExpectedHash)
 223    : Item(Owner), ExpectedHash(ExpectedHash)
 224 {
 225    /* XXX: Beware: Currently this class does nothing (of value) anymore ! */
 226    Debug = _config->FindB("Debug::pkgAcquire::SubIndex",false);
 227
 228    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 229    DestFile += URItoFileName(URI);
 230
 231    Desc.URI = URI;
 232    Desc.Description = URIDesc;
 233    Desc.Owner = this;
 234    Desc.ShortDesc = ShortDesc;
 235
 236    QueueURI(Desc);
 237
 238    if(Debug)
 239       std::clog << "pkgAcqSubIndex: " << Desc.URI << std::endl;
 240 }
 241                                                                         /*}}}*/
 242 // AcqSubIndex::Custom600Headers - Insert custom request headers        /*{{{*/
 243 // ---------------------------------------------------------------------
 244 /* The only header we use is the last-modified header. */
 245 string pkgAcqSubIndex::Custom600Headers()
 246 {
 247    string Final = _config->FindDir("Dir::State::lists");
 248    Final += URItoFileName(Desc.URI);
 249
 250    struct stat Buf;
 251    if (stat(Final.c_str(),&Buf) != 0)
 252       return "\nIndex-File: true\nFail-Ignore: true\n";
 253    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 254 }
 255                                                                         /*}}}*/
 256 void pkgAcqSubIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)       /*{{{*/
 257 {
 258    if(Debug)
 259       std::clog << "pkgAcqSubIndex failed: " << Desc.URI << std::endl;
 260
 261    Complete = false;
 262    Status = StatDone;
 263    Dequeue();
 264
 265    // No good Index is provided
 266 }
 267                                                                         /*}}}*/
 268 void pkgAcqSubIndex::Done(string Message,unsigned long long Size,string Md5Hash,        /*{{{*/
 269                            pkgAcquire::MethodConfig *Cnf)
 270 {
 271    if(Debug)
 272       std::clog << "pkgAcqSubIndex::Done(): " << Desc.URI << std::endl;
 273
 274    string FileName = LookupTag(Message,"Filename");
 275    if (FileName.empty() == true)
 276    {
 277       Status = StatError;
 278       ErrorText = "Method gave a blank filename";
 279       return;
 280    }
 281
 282    if (FileName != DestFile)
 283    {
 284       Local = true;
 285       Desc.URI = "copy:" + FileName;
 286       QueueURI(Desc);
 287       return;
 288    }
 289
 290    Item::Done(Message,Size,Md5Hash,Cnf);
 291
 292    string FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(Desc.URI);
 293
 294    /* Downloaded invalid transindex => Error (LP: #346386) (Closes: #627642) */
 295    indexRecords SubIndexParser;
 296    if (FileExists(DestFile) == true && !SubIndexParser.Load(DestFile)) {
 297       Status = StatError;
 298       ErrorText = SubIndexParser.ErrorText;
 299       return;
 300    }
 301
 302    // sucess in downloading the index
 303    // rename the index
 304    if(Debug)
 305       std::clog << "Renaming: " << DestFile << " -> " << FinalFile << std::endl;
 306    Rename(DestFile,FinalFile);
 307    chmod(FinalFile.c_str(),0644);
 308    DestFile = FinalFile;
 309
 310    if(ParseIndex(DestFile) == false)
 311       return Failed("", NULL);
 312
 313    Complete = true;
 314    Status = StatDone;
 315    Dequeue();
 316    return;
 317 }
 318                                                                         /*}}}*/
 319 bool pkgAcqSubIndex::ParseIndex(string const &IndexFile)                /*{{{*/
 320 {
 321    indexRecords SubIndexParser;
 322    if (FileExists(IndexFile) == false || SubIndexParser.Load(IndexFile) == false)
 323       return false;
 324    // so something with the downloaded index
 325    return true;
 326 }
 327                                                                         /*}}}*/
 328 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 329 // ---------------------------------------------------------------------
 330 /* Get the DiffIndex file first and see if there are patches availabe
 331  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 332  * patches. If anything goes wrong in that process, it will fall back to
 333  * the original packages file
 334  */
 335 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 336                                  string URI,string URIDesc,string ShortDesc,
 337                                  HashString ExpectedHash)
 338    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 339      Description(URIDesc)
 340 {
 341
 342    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 343
 344    Desc.Description = URIDesc + "/DiffIndex";
 345    Desc.Owner = this;
 346    Desc.ShortDesc = ShortDesc;
 347    Desc.URI = URI + ".diff/Index";
 348
 349    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 350    DestFile += URItoFileName(URI) + string(".DiffIndex");
 351
 352    if(Debug)
 353       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 354
 355    // look for the current package file
 356    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 357    CurrentPackagesFile += URItoFileName(RealURI);
 358
 359    // FIXME: this file:/ check is a hack to prevent fetching
 360    //        from local sources. this is really silly, and
 361    //        should be fixed cleanly as soon as possible
 362    if(!FileExists(CurrentPackagesFile) ||
 363       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 364    {
 365       // we don't have a pkg file or we don't want to queue
 366       if(Debug)
 367          std::clog << "No index file, local or canceld by user" << std::endl;
 368       Failed("", NULL);
 369       return;
 370    }
 371
 372    if(Debug)
 373       std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): "
 374                 << CurrentPackagesFile << std::endl;
 375
 376    QueueURI(Desc);
 377
 378 }
 379                                                                         /*}}}*/
 380 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 381 // ---------------------------------------------------------------------
 382 /* The only header we use is the last-modified header. */
 383 string pkgAcqDiffIndex::Custom600Headers()
 384 {
 385    string Final = _config->FindDir("Dir::State::lists");
 386    Final += URItoFileName(RealURI) + string(".IndexDiff");
 387
 388    if(Debug)
 389       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 390
 391    struct stat Buf;
 392    if (stat(Final.c_str(),&Buf) != 0)
 393       return "\nIndex-File: true";
 394
 395    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 396 }
 397                                                                         /*}}}*/
 398 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 399 {
 400    if(Debug)
 401       std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile
 402                 << std::endl;
 403
 404    pkgTagSection Tags;
 405    string ServerSha1;
 406    vector<DiffInfo> available_patches;
 407
 408    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 409    pkgTagFile TF(&Fd);
 410    if (_error->PendingError() == true)
 411       return false;
 412
 413    if(TF.Step(Tags) == true)
 414    {
 415       bool found = false;
 416       DiffInfo d;
 417       string size;
 418
 419       string const tmp = Tags.FindS("SHA1-Current");
 420       std::stringstream ss(tmp);
 421       ss >> ServerSha1 >> size;
 422       unsigned long const ServerSize = atol(size.c_str());
 423
 424       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 425       SHA1Summation SHA1;
 426       SHA1.AddFD(fd);
 427       string const local_sha1 = SHA1.Result();
 428
 429       if(local_sha1 == ServerSha1)
 430       {
 431          // we have the same sha1 as the server
 432          if(Debug)
 433             std::clog << "Package file is up-to-date" << std::endl;
 434          // set found to true, this will queue a pkgAcqIndexDiffs with
 435          // a empty availabe_patches
 436          found = true;
 437       }
 438       else
 439       {
 440          if(Debug)
 441             std::clog << "SHA1-Current: " << ServerSha1 << " and we start at "<< fd.Name() << " " << fd.Size() << " " << local_sha1 << std::endl;
 442
 443          // check the historie and see what patches we need
 444          string const history = Tags.FindS("SHA1-History");
 445          std::stringstream hist(history);
 446          while(hist >> d.sha1 >> size >> d.file)
 447          {
 448             // read until the first match is found
 449             // from that point on, we probably need all diffs
 450             if(d.sha1 == local_sha1)
 451                found=true;
 452             else if (found == false)
 453                continue;
 454
 455             if(Debug)
 456                std::clog << "Need to get diff: " << d.file << std::endl;
 457             available_patches.push_back(d);
 458          }
 459
 460          if (available_patches.empty() == false)
 461          {
 462             // patching with too many files is rather slow compared to a fast download
 463             unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 464             if (fileLimit != 0 && fileLimit < available_patches.size())
 465             {
 466                if (Debug)
 467                   std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 468                         << ") so fallback to complete download" << std::endl;
 469                return false;
 470             }
 471
 472             // see if the patches are too big
 473             found = false; // it was true and it will be true again at the end
 474             d = *available_patches.begin();
 475             string const firstPatch = d.file;
 476             unsigned long patchesSize = 0;
 477             std::stringstream patches(Tags.FindS("SHA1-Patches"));
 478             while(patches >> d.sha1 >> size >> d.file)
 479             {
 480                if (firstPatch == d.file)
 481                   found = true;
 482                else if (found == false)
 483                   continue;
 484
 485                patchesSize += atol(size.c_str());
 486             }
 487             unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 488             if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 489             {
 490                if (Debug)
 491                   std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 492                         << ") so fallback to complete download" << std::endl;
 493                return false;
 494             }
 495          }
 496       }
 497
 498       // we have something, queue the next diff
 499       if(found)
 500       {
 501          // queue the diffs
 502          string::size_type const last_space = Description.rfind(" ");
 503          if(last_space != string::npos)
 504             Description.erase(last_space, Description.size()-last_space);
 505          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 506                               ExpectedHash, ServerSha1, available_patches);
 507          Complete = false;
 508          Status = StatDone;
 509          Dequeue();
 510          return true;
 511       }
 512    }
 513
 514    // Nothing found, report and return false
 515    // Failing here is ok, if we return false later, the full
 516    // IndexFile is queued
 517    if(Debug)
 518       std::clog << "Can't find a patch in the index file" << std::endl;
 519    return false;
 520 }
 521                                                                         /*}}}*/
 522 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)      /*{{{*/
 523 {
 524    if(Debug)
 525       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << std::endl
 526                 << "Falling back to normal index file aquire" << std::endl;
 527
 528    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 529                    ExpectedHash);
 530
 531    Complete = false;
 532    Status = StatDone;
 533    Dequeue();
 534 }
 535                                                                         /*}}}*/
 536 void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,string Md5Hash,       /*{{{*/
 537                            pkgAcquire::MethodConfig *Cnf)
 538 {
 539    if(Debug)
 540       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 541
 542    Item::Done(Message,Size,Md5Hash,Cnf);
 543
 544    string FinalFile;
 545    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 546
 547    // sucess in downloading the index
 548    // rename the index
 549    FinalFile += string(".IndexDiff");
 550    if(Debug)
 551       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 552                 << std::endl;
 553    Rename(DestFile,FinalFile);
 554    chmod(FinalFile.c_str(),0644);
 555    DestFile = FinalFile;
 556
 557    if(!ParseDiffIndex(DestFile))
 558       return Failed("", NULL);
 559
 560    Complete = true;
 561    Status = StatDone;
 562    Dequeue();
 563    return;
 564 }
 565                                                                         /*}}}*/
 566 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 567 // ---------------------------------------------------------------------
 568 /* The package diff is added to the queue. one object is constructed
 569  * for each diff and the index
 570  */
 571 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 572                                    string URI,string URIDesc,string ShortDesc,
 573                                    HashString ExpectedHash,
 574                                    string ServerSha1,
 575                                    vector<DiffInfo> diffs)
 576    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 577      available_patches(diffs), ServerSha1(ServerSha1)
 578 {
 579
 580    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 581    DestFile += URItoFileName(URI);
 582
 583    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 584
 585    Description = URIDesc;
 586    Desc.Owner = this;
 587    Desc.ShortDesc = ShortDesc;
 588
 589    if(available_patches.empty() == true)
 590    {
 591       // we are done (yeah!)
 592       Finish(true);
 593    }
 594    else
 595    {
 596       // get the next diff
 597       State = StateFetchDiff;
 598       QueueNextDiff();
 599    }
 600 }
 601                                                                         /*}}}*/
 602 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)     /*{{{*/
 603 {
 604    if(Debug)
 605       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl
 606                 << "Falling back to normal index file aquire" << std::endl;
 607    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 608                    ExpectedHash);
 609    Finish();
 610 }
 611                                                                         /*}}}*/
 612 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 613 void pkgAcqIndexDiffs::Finish(bool allDone)
 614 {
 615    // we restore the original name, this is required, otherwise
 616    // the file will be cleaned
 617    if(allDone)
 618    {
 619       DestFile = _config->FindDir("Dir::State::lists");
 620       DestFile += URItoFileName(RealURI);
 621
 622       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 623       {
 624          RenameOnError(HashSumMismatch);
 625          Dequeue();
 626          return;
 627       }
 628
 629       // this is for the "real" finish
 630       Complete = true;
 631       Status = StatDone;
 632       Dequeue();
 633       if(Debug)
 634          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 635       return;
 636    }
 637
 638    if(Debug)
 639       std::clog << "Finishing: " << Desc.URI << std::endl;
 640    Complete = false;
 641    Status = StatDone;
 642    Dequeue();
 643    return;
 644 }
 645                                                                         /*}}}*/
 646 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 647 {
 648
 649    // calc sha1 of the just patched file
 650    string FinalFile = _config->FindDir("Dir::State::lists");
 651    FinalFile += URItoFileName(RealURI);
 652
 653    FileFd fd(FinalFile, FileFd::ReadOnly);
 654    SHA1Summation SHA1;
 655    SHA1.AddFD(fd);
 656    string local_sha1 = string(SHA1.Result());
 657    if(Debug)
 658       std::clog << "QueueNextDiff: "
 659                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 660
 661    // final file reached before all patches are applied
 662    if(local_sha1 == ServerSha1)
 663    {
 664       Finish(true);
 665       return true;
 666    }
 667
 668    // remove all patches until the next matching patch is found
 669    // this requires the Index file to be ordered
 670    for(vector<DiffInfo>::iterator I=available_patches.begin();
 671        available_patches.empty() == false &&
 672           I != available_patches.end() &&
 673           I->sha1 != local_sha1;
 674        ++I)
 675    {
 676       available_patches.erase(I);
 677    }
 678
 679    // error checking and falling back if no patch was found
 680    if(available_patches.empty() == true)
 681    {
 682       Failed("", NULL);
 683       return false;
 684    }
 685
 686    // queue the right diff
 687    Desc.URI = string(RealURI) + ".diff/" + available_patches[0].file + ".gz";
 688    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 689    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 690    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 691
 692    if(Debug)
 693       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 694
 695    QueueURI(Desc);
 696
 697    return true;
 698 }
 699                                                                         /*}}}*/
 700 void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size,string Md5Hash,      /*{{{*/
 701                             pkgAcquire::MethodConfig *Cnf)
 702 {
 703    if(Debug)
 704       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 705
 706    Item::Done(Message,Size,Md5Hash,Cnf);
 707
 708    string FinalFile;
 709    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 710
 711    // sucess in downloading a diff, enter ApplyDiff state
 712    if(State == StateFetchDiff)
 713    {
 714
 715       // rred excepts the patch as $FinalFile.ed
 716       Rename(DestFile,FinalFile+".ed");
 717
 718       if(Debug)
 719          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 720
 721       State = StateApplyDiff;
 722       Local = true;
 723       Desc.URI = "rred:" + FinalFile;
 724       QueueURI(Desc);
 725       Mode = "rred";
 726       return;
 727    }
 728
 729
 730    // success in download/apply a diff, queue next (if needed)
 731    if(State == StateApplyDiff)
 732    {
 733       // remove the just applied patch
 734       available_patches.erase(available_patches.begin());
 735
 736       // move into place
 737       if(Debug)
 738       {
 739          std::clog << "Moving patched file in place: " << std::endl
 740                    << DestFile << " -> " << FinalFile << std::endl;
 741       }
 742       Rename(DestFile,FinalFile);
 743       chmod(FinalFile.c_str(),0644);
 744
 745       // see if there is more to download
 746       if(available_patches.empty() == false) {
 747          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 748                               ExpectedHash, ServerSha1, available_patches);
 749          return Finish();
 750       } else
 751          return Finish(true);
 752    }
 753 }
 754                                                                         /*}}}*/
 755 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 756 // ---------------------------------------------------------------------
 757 /* The package file is added to the queue and a second class is
 758    instantiated to fetch the revision file */
 759 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 760                          string URI,string URIDesc,string ShortDesc,
 761                          HashString ExpectedHash, string comprExt)
 762    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 763 {
 764    if(comprExt.empty() == true)
 765    {
 766       // autoselect the compression method
 767       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 768       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 769          comprExt.append(*t).append(" ");
 770       if (comprExt.empty() == false)
 771          comprExt.erase(comprExt.end()-1);
 772    }
 773    CompressionExtension = comprExt;
 774
 775    Init(URI, URIDesc, ShortDesc);
 776 }
 777 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target,
 778                          HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
 779    : Item(Owner), RealURI(Target->URI), ExpectedHash(ExpectedHash)
 780 {
 781    // autoselect the compression method
 782    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 783    CompressionExtension = "";
 784    if (ExpectedHash.empty() == false)
 785    {
 786       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 787          if (*t == "uncompressed" || MetaIndexParser->Exists(string(Target->MetaKey).append(".").append(*t)) == true)
 788             CompressionExtension.append(*t).append(" ");
 789    }
 790    else
 791    {
 792       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 793          CompressionExtension.append(*t).append(" ");
 794    }
 795    if (CompressionExtension.empty() == false)
 796       CompressionExtension.erase(CompressionExtension.end()-1);
 797
 798    // only verify non-optional targets, see acquire-item.h for a FIXME
 799    // to make this more flexible
 800    if (Target->IsOptional())
 801      Verify = false;
 802    else
 803      Verify = true;
 804
 805    Init(Target->URI, Target->Description, Target->ShortDesc);
 806 }
 807                                                                         /*}}}*/
 808 // AcqIndex::Init - defered Constructor                                 /*{{{*/
 809 void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &ShortDesc) {
 810    Decompression = false;
 811    Erase = false;
 812
 813    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 814    DestFile += URItoFileName(URI);
 815
 816    std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
 817    if (comprExt == "uncompressed")
 818       Desc.URI = URI;
 819    else
 820       Desc.URI = URI + '.' + comprExt;
 821
 822    Desc.Description = URIDesc;
 823    Desc.Owner = this;
 824    Desc.ShortDesc = ShortDesc;
 825
 826    QueueURI(Desc);
 827 }
 828                                                                         /*}}}*/
 829 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 830 // ---------------------------------------------------------------------
 831 /* The only header we use is the last-modified header. */
 832 string pkgAcqIndex::Custom600Headers()
 833 {
 834    string Final = _config->FindDir("Dir::State::lists");
 835    Final += URItoFileName(RealURI);
 836    if (_config->FindB("Acquire::GzipIndexes",false))
 837       Final += ".gz";
 838
 839    string msg = "\nIndex-File: true";
 840    // FIXME: this really should use "IndexTarget::IsOptional()" but that
 841    //        seems to be difficult without breaking ABI
 842    if (ShortDesc().find("Translation") != 0)
 843       msg += "\nFail-Ignore: true";
 844    struct stat Buf;
 845    if (stat(Final.c_str(),&Buf) == 0)
 846       msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 847
 848    return msg;
 849 }
 850                                                                         /*}}}*/
 851 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
 852 {
 853    size_t const nextExt = CompressionExtension.find(' ');
 854    if (nextExt != std::string::npos)
 855    {
 856       CompressionExtension = CompressionExtension.substr(nextExt+1);
 857       Init(RealURI, Desc.Description, Desc.ShortDesc);
 858       return;
 859    }
 860
 861    // on decompression failure, remove bad versions in partial/
 862    if (Decompression && Erase) {
 863       string s = _config->FindDir("Dir::State::lists") + "partial/";
 864       s.append(URItoFileName(RealURI));
 865       unlink(s.c_str());
 866    }
 867
 868    Item::Failed(Message,Cnf);
 869 }
 870                                                                         /*}}}*/
 871 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
 872 // ---------------------------------------------------------------------
 873 /* This goes through a number of states.. On the initial fetch the
 874    method could possibly return an alternate filename which points
 875    to the uncompressed version of the file. If this is so the file
 876    is copied into the partial directory. In all other cases the file
 877    is decompressed with a gzip uri. */
 878 void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash,
 879                        pkgAcquire::MethodConfig *Cfg)
 880 {
 881    Item::Done(Message,Size,Hash,Cfg);
 882
 883    if (Decompression == true)
 884    {
 885       if (_config->FindB("Debug::pkgAcquire::Auth", false))
 886       {
 887          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
 888          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
 889       }
 890
 891       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
 892       {
 893          RenameOnError(HashSumMismatch);
 894          return;
 895       }
 896
 897       /* Verify the index file for correctness (all indexes must
 898        * have a Package field) (LP: #346386) (Closes: #627642) */
 899       if (Verify == true)
 900       {
 901          FileFd fd(DestFile, FileFd::ReadOnly);
 902          // Only test for correctness if the file is not empty (empty is ok)
 903          if (fd.FileSize() > 0)
 904          {
 905             pkgTagSection sec;
 906             pkgTagFile tag(&fd);
 907
 908             // all our current indexes have a field 'Package' in each section
 909             if (_error->PendingError() == true || tag.Step(sec) == false || sec.Exists("Package") == false)
 910             {
 911                RenameOnError(InvalidFormat);
 912                return;
 913             }
 914          }
 915       }
 916
 917       // Done, move it into position
 918       string FinalFile = _config->FindDir("Dir::State::lists");
 919       FinalFile += URItoFileName(RealURI);
 920       Rename(DestFile,FinalFile);
 921       chmod(FinalFile.c_str(),0644);
 922
 923       /* We restore the original name to DestFile so that the clean operation
 924          will work OK */
 925       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 926       DestFile += URItoFileName(RealURI);
 927
 928       // Remove the compressed version.
 929       if (Erase == true)
 930          unlink(DestFile.c_str());
 931       return;
 932    }
 933
 934    Erase = false;
 935    Complete = true;
 936
 937    // Handle the unzipd case
 938    string FileName = LookupTag(Message,"Alt-Filename");
 939    if (FileName.empty() == false)
 940    {
 941       // The files timestamp matches
 942       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
 943          return;
 944       Decompression = true;
 945       Local = true;
 946       DestFile += ".decomp";
 947       Desc.URI = "copy:" + FileName;
 948       QueueURI(Desc);
 949       Mode = "copy";
 950       return;
 951    }
 952
 953    FileName = LookupTag(Message,"Filename");
 954    if (FileName.empty() == true)
 955    {
 956       Status = StatError;
 957       ErrorText = "Method gave a blank filename";
 958    }
 959
 960    std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
 961
 962    // The files timestamp matches
 963    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) {
 964        if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
 965           // Update DestFile for .gz suffix so that the clean operation keeps it
 966           DestFile += ".gz";
 967       return;
 968     }
 969
 970    if (FileName == DestFile)
 971       Erase = true;
 972    else
 973       Local = true;
 974
 975    string decompProg;
 976
 977    // If we enable compressed indexes and already have gzip, keep it
 978    if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) {
 979       string FinalFile = _config->FindDir("Dir::State::lists");
 980       FinalFile += URItoFileName(RealURI) + ".gz";
 981       Rename(DestFile,FinalFile);
 982       chmod(FinalFile.c_str(),0644);
 983
 984       // Update DestFile for .gz suffix so that the clean operation keeps it
 985       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 986       DestFile += URItoFileName(RealURI) + ".gz";
 987       return;
 988     }
 989
 990    // get the binary name for your used compression type
 991    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
 992    if(decompProg.empty() == false);
 993    else if(compExt == "uncompressed")
 994       decompProg = "copy";
 995    else {
 996       _error->Error("Unsupported extension: %s", compExt.c_str());
 997       return;
 998    }
 999
1000    Decompression = true;
1001    DestFile += ".decomp";
1002    Desc.URI = decompProg + ":" + FileName;
1003    QueueURI(Desc);
1004
1005    // FIXME: this points to a c++ string that goes out of scope
1006    Mode = decompProg.c_str();
1007 }
1008                                                                         /*}}}*/
1009 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
1010 // ---------------------------------------------------------------------
1011 /* The Translation file is added to the queue */
1012 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
1013                             string URI,string URIDesc,string ShortDesc)
1014   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
1015 {
1016 }
1017 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const *Target,
1018                          HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
1019   : pkgAcqIndex(Owner, Target, ExpectedHash, MetaIndexParser)
1020 {
1021 }
1022                                                                         /*}}}*/
1023 // AcqIndexTrans::Custom600Headers - Insert custom request headers      /*{{{*/
1024 // ---------------------------------------------------------------------
1025 string pkgAcqIndexTrans::Custom600Headers()
1026 {
1027    string Final = _config->FindDir("Dir::State::lists");
1028    Final += URItoFileName(RealURI);
1029
1030    struct stat Buf;
1031    if (stat(Final.c_str(),&Buf) != 0)
1032       return "\nFail-Ignore: true\nIndex-File: true";
1033    return "\nFail-Ignore: true\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1034 }
1035                                                                         /*}}}*/
1036 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
1037 // ---------------------------------------------------------------------
1038 /* */
1039 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1040 {
1041    size_t const nextExt = CompressionExtension.find(' ');
1042    if (nextExt != std::string::npos)
1043    {
1044       CompressionExtension = CompressionExtension.substr(nextExt+1);
1045       Init(RealURI, Desc.Description, Desc.ShortDesc);
1046       Status = StatIdle;
1047       return;
1048    }
1049
1050    if (Cnf->LocalOnly == true ||
1051        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1052    {
1053       // Ignore this
1054       Status = StatDone;
1055       Complete = false;
1056       Dequeue();
1057       return;
1058    }
1059
1060    Item::Failed(Message,Cnf);
1061 }
1062                                                                         /*}}}*/
1063 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
1064                              string URI,string URIDesc,string ShortDesc,
1065                              string MetaIndexURI, string MetaIndexURIDesc,
1066                              string MetaIndexShortDesc,
1067                              const vector<IndexTarget*>* IndexTargets,
1068                              indexRecords* MetaIndexParser) :
1069    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
1070    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1071    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
1072 {
1073    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1074    DestFile += URItoFileName(URI);
1075
1076    // remove any partial downloaded sig-file in partial/.
1077    // it may confuse proxies and is too small to warrant a
1078    // partial download anyway
1079    unlink(DestFile.c_str());
1080
1081    // Create the item
1082    Desc.Description = URIDesc;
1083    Desc.Owner = this;
1084    Desc.ShortDesc = ShortDesc;
1085    Desc.URI = URI;
1086
1087    string Final = _config->FindDir("Dir::State::lists");
1088    Final += URItoFileName(RealURI);
1089    if (RealFileExists(Final) == true)
1090    {
1091       // File was already in place.  It needs to be re-downloaded/verified
1092       // because Release might have changed, we do give it a differnt
1093       // name than DestFile because otherwise the http method will
1094       // send If-Range requests and there are too many broken servers
1095       // out there that do not understand them
1096       LastGoodSig = DestFile+".reverify";
1097       Rename(Final,LastGoodSig);
1098    }
1099
1100    QueueURI(Desc);
1101 }
1102                                                                         /*}}}*/
1103 pkgAcqMetaSig::~pkgAcqMetaSig()                                         /*{{{*/
1104 {
1105    // if the file was never queued undo file-changes done in the constructor
1106    if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false &&
1107          LastGoodSig.empty() == false)
1108    {
1109       string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1110       if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true)
1111          Rename(LastGoodSig, Final);
1112    }
1113
1114 }
1115                                                                         /*}}}*/
1116 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
1117 // ---------------------------------------------------------------------
1118 /* The only header we use is the last-modified header. */
1119 string pkgAcqMetaSig::Custom600Headers()
1120 {
1121    struct stat Buf;
1122    if (stat(LastGoodSig.c_str(),&Buf) != 0)
1123       return "\nIndex-File: true";
1124
1125    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1126 }
1127
1128 void pkgAcqMetaSig::Done(string Message,unsigned long long Size,string MD5,
1129                          pkgAcquire::MethodConfig *Cfg)
1130 {
1131    Item::Done(Message,Size,MD5,Cfg);
1132
1133    string FileName = LookupTag(Message,"Filename");
1134    if (FileName.empty() == true)
1135    {
1136       Status = StatError;
1137       ErrorText = "Method gave a blank filename";
1138       return;
1139    }
1140
1141    if (FileName != DestFile)
1142    {
1143       // We have to copy it into place
1144       Local = true;
1145       Desc.URI = "copy:" + FileName;
1146       QueueURI(Desc);
1147       return;
1148    }
1149
1150    Complete = true;
1151
1152    // put the last known good file back on i-m-s hit (it will
1153    // be re-verified again)
1154    // Else do nothing, we have the new file in DestFile then
1155    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1156       Rename(LastGoodSig, DestFile);
1157
1158    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
1159    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc,
1160                        MetaIndexShortDesc,  DestFile, IndexTargets,
1161                        MetaIndexParser);
1162
1163 }
1164                                                                         /*}}}*/
1165 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
1166 {
1167    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1168
1169    // if we get a network error we fail gracefully
1170    if(Status == StatTransientNetworkError)
1171    {
1172       Item::Failed(Message,Cnf);
1173       // move the sigfile back on transient network failures
1174       if(FileExists(LastGoodSig))
1175          Rename(LastGoodSig,Final);
1176
1177       // set the status back to , Item::Failed likes to reset it
1178       Status = pkgAcquire::Item::StatTransientNetworkError;
1179       return;
1180    }
1181
1182    // Delete any existing sigfile when the acquire failed
1183    unlink(Final.c_str());
1184
1185    // queue a pkgAcqMetaIndex with no sigfile
1186    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1187                        "", IndexTargets, MetaIndexParser);
1188
1189    if (Cnf->LocalOnly == true ||
1190        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1191    {
1192       // Ignore this
1193       Status = StatDone;
1194       Complete = false;
1195       Dequeue();
1196       return;
1197    }
1198
1199    Item::Failed(Message,Cnf);
1200 }
1201                                                                         /*}}}*/
1202 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
1203                                  string URI,string URIDesc,string ShortDesc,
1204                                  string SigFile,
1205                                  const vector<struct IndexTarget*>* IndexTargets,
1206                                  indexRecords* MetaIndexParser) :
1207    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
1208    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
1209 {
1210    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1211    DestFile += URItoFileName(URI);
1212
1213    // Create the item
1214    Desc.Description = URIDesc;
1215    Desc.Owner = this;
1216    Desc.ShortDesc = ShortDesc;
1217    Desc.URI = URI;
1218
1219    QueueURI(Desc);
1220 }
1221                                                                         /*}}}*/
1222 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
1223 // ---------------------------------------------------------------------
1224 /* The only header we use is the last-modified header. */
1225 string pkgAcqMetaIndex::Custom600Headers()
1226 {
1227    string Final = _config->FindDir("Dir::State::lists");
1228    Final += URItoFileName(RealURI);
1229
1230    struct stat Buf;
1231    if (stat(Final.c_str(),&Buf) != 0)
1232       return "\nIndex-File: true";
1233
1234    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1235 }
1236                                                                         /*}}}*/
1237 void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,string Hash,  /*{{{*/
1238                            pkgAcquire::MethodConfig *Cfg)
1239 {
1240    Item::Done(Message,Size,Hash,Cfg);
1241
1242    // MetaIndexes are done in two passes: one to download the
1243    // metaindex with an appropriate method, and a second to verify it
1244    // with the gpgv method
1245
1246    if (AuthPass == true)
1247    {
1248       AuthDone(Message);
1249
1250       // all cool, move Release file into place
1251       Complete = true;
1252    }
1253    else
1254    {
1255       RetrievalDone(Message);
1256       if (!Complete)
1257          // Still more retrieving to do
1258          return;
1259
1260       if (SigFile == "")
1261       {
1262          // There was no signature file, so we are finished.  Download
1263          // the indexes and do only hashsum verification if possible
1264          MetaIndexParser->Load(DestFile);
1265          QueueIndexes(false);
1266       }
1267       else
1268       {
1269          // There was a signature file, so pass it to gpgv for
1270          // verification
1271
1272          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1273             std::cerr << "Metaindex acquired, queueing gpg verification ("
1274                       << SigFile << "," << DestFile << ")\n";
1275          AuthPass = true;
1276          Desc.URI = "gpgv:" + SigFile;
1277          QueueURI(Desc);
1278          Mode = "gpgv";
1279          return;
1280       }
1281    }
1282
1283    if (Complete == true)
1284    {
1285       string FinalFile = _config->FindDir("Dir::State::lists");
1286       FinalFile += URItoFileName(RealURI);
1287       if (SigFile == DestFile)
1288          SigFile = FinalFile;
1289       Rename(DestFile,FinalFile);
1290       chmod(FinalFile.c_str(),0644);
1291       DestFile = FinalFile;
1292    }
1293 }
1294                                                                         /*}}}*/
1295 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
1296 {
1297    // We have just finished downloading a Release file (it is not
1298    // verified yet)
1299
1300    string FileName = LookupTag(Message,"Filename");
1301    if (FileName.empty() == true)
1302    {
1303       Status = StatError;
1304       ErrorText = "Method gave a blank filename";
1305       return;
1306    }
1307
1308    if (FileName != DestFile)
1309    {
1310       Local = true;
1311       Desc.URI = "copy:" + FileName;
1312       QueueURI(Desc);
1313       return;
1314    }
1315
1316    // make sure to verify against the right file on I-M-S hit
1317    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1318    if(IMSHit)
1319    {
1320       string FinalFile = _config->FindDir("Dir::State::lists");
1321       FinalFile += URItoFileName(RealURI);
1322       if (SigFile == DestFile)
1323       {
1324          SigFile = FinalFile;
1325          // constructor of pkgAcqMetaClearSig moved it out of the way,
1326          // now move it back in on IMS hit for the 'old' file
1327          string const OldClearSig = DestFile + ".reverify";
1328          if (RealFileExists(OldClearSig) == true)
1329             Rename(OldClearSig, FinalFile);
1330       }
1331       DestFile = FinalFile;
1332    }
1333    Complete = true;
1334 }
1335                                                                         /*}}}*/
1336 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1337 {
1338    // At this point, the gpgv method has succeeded, so there is a
1339    // valid signature from a key in the trusted keyring.  We
1340    // perform additional verification of its contents, and use them
1341    // to verify the indexes we are about to download
1342
1343    if (!MetaIndexParser->Load(DestFile))
1344    {
1345       Status = StatAuthError;
1346       ErrorText = MetaIndexParser->ErrorText;
1347       return;
1348    }
1349
1350    if (!VerifyVendor(Message))
1351    {
1352       return;
1353    }
1354
1355    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1356       std::cerr << "Signature verification succeeded: "
1357                 << DestFile << std::endl;
1358
1359    // Download further indexes with verification
1360    QueueIndexes(true);
1361
1362    // is it a clearsigned MetaIndex file?
1363    if (DestFile == SigFile)
1364       return;
1365
1366    // Done, move signature file into position
1367    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1368       URItoFileName(RealURI) + ".gpg";
1369    Rename(SigFile,VerifiedSigFile);
1370    chmod(VerifiedSigFile.c_str(),0644);
1371 }
1372                                                                         /*}}}*/
1373 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1374 {
1375 #if 0
1376    /* Reject invalid, existing Release files (LP: #346386) (Closes: #627642)
1377     * FIXME: Disabled; it breaks unsigned repositories without hashes */
1378    if (!verify && FileExists(DestFile) && !MetaIndexParser->Load(DestFile))
1379    {
1380       Status = StatError;
1381       ErrorText = MetaIndexParser->ErrorText;
1382       return;
1383    }
1384 #endif
1385    bool transInRelease = false;
1386    {
1387       std::vector<std::string> const keys = MetaIndexParser->MetaKeys();
1388       for (std::vector<std::string>::const_iterator k = keys.begin(); k != keys.end(); ++k)
1389          // FIXME: Feels wrong to check for hardcoded string here, but what should we do else…
1390          if (k->find("Translation-") != std::string::npos)
1391          {
1392             transInRelease = true;
1393             break;
1394          }
1395    }
1396
1397    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1398         Target != IndexTargets->end();
1399         ++Target)
1400    {
1401       HashString ExpectedIndexHash;
1402       const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1403       bool compressedAvailable = false;
1404       if (Record == NULL)
1405       {
1406          if ((*Target)->IsOptional() == true)
1407          {
1408             std::vector<std::string> types = APT::Configuration::getCompressionTypes();
1409             for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
1410                if (MetaIndexParser->Exists(string((*Target)->MetaKey).append(".").append(*t)) == true)
1411                {
1412                   compressedAvailable = true;
1413                   break;
1414                }
1415          }
1416          else if (verify == true)
1417          {
1418             Status = StatAuthError;
1419             strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), (*Target)->MetaKey.c_str());
1420             return;
1421          }
1422       }
1423       else
1424       {
1425          ExpectedIndexHash = Record->Hash;
1426          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1427          {
1428             std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1429             std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1430             std::cerr << "For: " << Record->MetaKeyFilename << std::endl;
1431          }
1432          if (verify == true && ExpectedIndexHash.empty() == true && (*Target)->IsOptional() == false)
1433          {
1434             Status = StatAuthError;
1435             strprintf(ErrorText, _("Unable to find hash sum for '%s' in Release file"), (*Target)->MetaKey.c_str());
1436             return;
1437          }
1438       }
1439
1440       if ((*Target)->IsOptional() == true)
1441       {
1442          if ((*Target)->IsSubIndex() == true)
1443             new pkgAcqSubIndex(Owner, (*Target)->URI, (*Target)->Description,
1444                                 (*Target)->ShortDesc, ExpectedIndexHash);
1445          else if (transInRelease == false || Record != NULL || compressedAvailable == true)
1446          {
1447             if (_config->FindB("Acquire::PDiffs",true) == true && transInRelease == true &&
1448                 MetaIndexParser->Exists(string((*Target)->MetaKey).append(".diff/Index")) == true)
1449                new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1450                                    (*Target)->ShortDesc, ExpectedIndexHash);
1451             else
1452                new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1453          }
1454          continue;
1455       }
1456
1457       /* Queue Packages file (either diff or full packages files, depending
1458          on the users option) - we also check if the PDiff Index file is listed
1459          in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
1460          instead, but passing the required info to it is to much hassle */
1461       if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
1462           MetaIndexParser->Exists(string((*Target)->MetaKey).append(".diff/Index")) == true))
1463          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1464                              (*Target)->ShortDesc, ExpectedIndexHash);
1465       else
1466          new pkgAcqIndex(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1467    }
1468 }
1469                                                                         /*}}}*/
1470 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1471 {
1472    string::size_type pos;
1473
1474    // check for missing sigs (that where not fatal because otherwise we had
1475    // bombed earlier)
1476    string missingkeys;
1477    string msg = _("There is no public key available for the "
1478                   "following key IDs:\n");
1479    pos = Message.find("NO_PUBKEY ");
1480    if (pos != std::string::npos)
1481    {
1482       string::size_type start = pos+strlen("NO_PUBKEY ");
1483       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1484       missingkeys += (Fingerprint);
1485    }
1486    if(!missingkeys.empty())
1487       _error->Warning("%s", string(msg+missingkeys).c_str());
1488
1489    string Transformed = MetaIndexParser->GetExpectedDist();
1490
1491    if (Transformed == "../project/experimental")
1492    {
1493       Transformed = "experimental";
1494    }
1495
1496    pos = Transformed.rfind('/');
1497    if (pos != string::npos)
1498    {
1499       Transformed = Transformed.substr(0, pos);
1500    }
1501
1502    if (Transformed == ".")
1503    {
1504       Transformed = "";
1505    }
1506
1507    if (_config->FindB("Acquire::Check-Valid-Until", true) == true &&
1508        MetaIndexParser->GetValidUntil() > 0) {
1509       time_t const invalid_since = time(NULL) - MetaIndexParser->GetValidUntil();
1510       if (invalid_since > 0)
1511          // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1512          // the time since then the file is invalid - formated in the same way as in
1513          // the download progress display (e.g. 7d 3h 42min 1s)
1514          return _error->Error(
1515             _("Release file for %s is expired (invalid since %s). "
1516               "Updates for this repository will not be applied."),
1517             RealURI.c_str(), TimeToStr(invalid_since).c_str());
1518    }
1519
1520    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1521    {
1522       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1523       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1524       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1525    }
1526
1527    if (MetaIndexParser->CheckDist(Transformed) == false)
1528    {
1529       // This might become fatal one day
1530 //       Status = StatAuthError;
1531 //       ErrorText = "Conflicting distribution; expected "
1532 //          + MetaIndexParser->GetExpectedDist() + " but got "
1533 //          + MetaIndexParser->GetDist();
1534 //       return false;
1535       if (!Transformed.empty())
1536       {
1537          _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1538                          Desc.Description.c_str(),
1539                          Transformed.c_str(),
1540                          MetaIndexParser->GetDist().c_str());
1541       }
1542    }
1543
1544    return true;
1545 }
1546                                                                         /*}}}*/
1547 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1548 // ---------------------------------------------------------------------
1549 /* */
1550 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1551 {
1552    if (AuthPass == true)
1553    {
1554       // gpgv method failed, if we have a good signature
1555       string LastGoodSigFile = _config->FindDir("Dir::State::lists").append("partial/").append(URItoFileName(RealURI));
1556       if (DestFile != SigFile)
1557          LastGoodSigFile.append(".gpg");
1558       LastGoodSigFile.append(".reverify");
1559
1560       if(FileExists(LastGoodSigFile))
1561       {
1562          string VerifiedSigFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1563          if (DestFile != SigFile)
1564             VerifiedSigFile.append(".gpg");
1565          Rename(LastGoodSigFile, VerifiedSigFile);
1566          Status = StatTransientNetworkError;
1567          _error->Warning(_("An error occurred during the signature "
1568                            "verification. The repository is not updated "
1569                            "and the previous index files will be used. "
1570                            "GPG error: %s: %s\n"),
1571                          Desc.Description.c_str(),
1572                          LookupTag(Message,"Message").c_str());
1573          RunScripts("APT::Update::Auth-Failure");
1574          return;
1575       } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) {
1576          /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */
1577          _error->Error(_("GPG error: %s: %s"),
1578                          Desc.Description.c_str(),
1579                          LookupTag(Message,"Message").c_str());
1580          return;
1581       } else {
1582          _error->Warning(_("GPG error: %s: %s"),
1583                          Desc.Description.c_str(),
1584                          LookupTag(Message,"Message").c_str());
1585       }
1586       // gpgv method failed
1587       ReportMirrorFailure("GPGFailure");
1588    }
1589
1590    /* Always move the meta index, even if gpgv failed. This ensures
1591     * that PackageFile objects are correctly filled in */
1592    if (FileExists(DestFile)) {
1593       string FinalFile = _config->FindDir("Dir::State::lists");
1594       FinalFile += URItoFileName(RealURI);
1595       /* InRelease files become Release files, otherwise
1596        * they would be considered as trusted later on */
1597       if (SigFile == DestFile) {
1598          RealURI = RealURI.replace(RealURI.rfind("InRelease"), 9,
1599                                        "Release");
1600          FinalFile = FinalFile.replace(FinalFile.rfind("InRelease"), 9,
1601                                        "Release");
1602          SigFile = FinalFile;
1603       }
1604       Rename(DestFile,FinalFile);
1605       chmod(FinalFile.c_str(),0644);
1606
1607       DestFile = FinalFile;
1608    }
1609
1610    // No Release file was present, or verification failed, so fall
1611    // back to queueing Packages files without verification
1612    QueueIndexes(false);
1613 }
1614                                                                         /*}}}*/
1615 pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner,               /*{{{*/
1616                 string const &URI, string const &URIDesc, string const &ShortDesc,
1617                 string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc,
1618                 string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc,
1619                 const vector<struct IndexTarget*>* IndexTargets,
1620                 indexRecords* MetaIndexParser) :
1621         pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, "", IndexTargets, MetaIndexParser),
1622         MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1623         MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc)
1624 {
1625    SigFile = DestFile;
1626
1627    // keep the old InRelease around in case of transistent network errors
1628    string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1629    if (RealFileExists(Final) == true)
1630    {
1631       string const LastGoodSig = DestFile + ".reverify";
1632       Rename(Final,LastGoodSig);
1633    }
1634 }
1635                                                                         /*}}}*/
1636 pkgAcqMetaClearSig::~pkgAcqMetaClearSig()                               /*{{{*/
1637 {
1638    // if the file was never queued undo file-changes done in the constructor
1639    if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false)
1640    {
1641       string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1642       string const LastGoodSig = DestFile + ".reverify";
1643       if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true)
1644          Rename(LastGoodSig, Final);
1645    }
1646 }
1647                                                                         /*}}}*/
1648 // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/
1649 // ---------------------------------------------------------------------
1650 // FIXME: this can go away once the InRelease file is used widely
1651 string pkgAcqMetaClearSig::Custom600Headers()
1652 {
1653    string Final = _config->FindDir("Dir::State::lists");
1654    Final += URItoFileName(RealURI);
1655
1656    struct stat Buf;
1657    if (stat(Final.c_str(),&Buf) != 0)
1658    {
1659       Final = DestFile + ".reverify";
1660       if (stat(Final.c_str(),&Buf) != 0)
1661          return "\nIndex-File: true\nFail-Ignore: true\n";
1662    }
1663
1664    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1665 }
1666                                                                         /*}}}*/
1667 void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/
1668 {
1669    if (AuthPass == false)
1670    {
1671       // Remove the 'old' InRelease file if we try Release.gpg now as otherwise
1672       // the file will stay around and gives a false-auth impression (CVE-2012-0214)
1673       string FinalFile = _config->FindDir("Dir::State::lists");
1674       FinalFile.append(URItoFileName(RealURI));
1675       if (FileExists(FinalFile))
1676          unlink(FinalFile.c_str());
1677
1678       new pkgAcqMetaSig(Owner,
1679                         MetaSigURI, MetaSigURIDesc, MetaSigShortDesc,
1680                         MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1681                         IndexTargets, MetaIndexParser);
1682       if (Cnf->LocalOnly == true ||
1683           StringToBool(LookupTag(Message, "Transient-Failure"), false) == false)
1684          Dequeue();
1685    }
1686    else
1687       pkgAcqMetaIndex::Failed(Message, Cnf);
1688 }
1689                                                                         /*}}}*/
1690 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1691 // ---------------------------------------------------------------------
1692 /* This just sets up the initial fetch environment and queues the first
1693    possibilitiy */
1694 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1695                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1696                              string &StoreFilename) :
1697                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1698                StoreFilename(StoreFilename), Vf(Version.FileList()),
1699                Trusted(false)
1700 {
1701    Retries = _config->FindI("Acquire::Retries",0);
1702
1703    if (Version.Arch() == 0)
1704    {
1705       _error->Error(_("I wasn't able to locate a file for the %s package. "
1706                       "This might mean you need to manually fix this package. "
1707                       "(due to missing arch)"),
1708                     Version.ParentPkg().Name());
1709       return;
1710    }
1711
1712    /* We need to find a filename to determine the extension. We make the
1713       assumption here that all the available sources for this version share
1714       the same extension.. */
1715    // Skip not source sources, they do not have file fields.
1716    for (; Vf.end() == false; ++Vf)
1717    {
1718       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1719          continue;
1720       break;
1721    }
1722
1723    // Does not really matter here.. we are going to fail out below
1724    if (Vf.end() != true)
1725    {
1726       // If this fails to get a file name we will bomb out below.
1727       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1728       if (_error->PendingError() == true)
1729          return;
1730
1731       // Generate the final file name as: package_version_arch.foo
1732       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1733                       QuoteString(Version.VerStr(),"_:") + '_' +
1734                       QuoteString(Version.Arch(),"_:.") +
1735                       "." + flExtension(Parse.FileName());
1736    }
1737
1738    // check if we have one trusted source for the package. if so, switch
1739    // to "TrustedOnly" mode - but only if not in AllowUnauthenticated mode
1740    bool const allowUnauth = _config->FindB("APT::Get::AllowUnauthenticated", false);
1741    bool const debugAuth = _config->FindB("Debug::pkgAcquire::Auth", false);
1742    bool seenUntrusted = false;
1743    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; ++i)
1744    {
1745       pkgIndexFile *Index;
1746       if (Sources->FindIndex(i.File(),Index) == false)
1747          continue;
1748
1749       if (debugAuth == true)
1750          std::cerr << "Checking index: " << Index->Describe()
1751                    << "(Trusted=" << Index->IsTrusted() << ")" << std::endl;
1752
1753       if (Index->IsTrusted() == true)
1754       {
1755          Trusted = true;
1756          if (allowUnauth == false)
1757             break;
1758       }
1759       else
1760          seenUntrusted = true;
1761    }
1762
1763    // "allow-unauthenticated" restores apts old fetching behaviour
1764    // that means that e.g. unauthenticated file:// uris are higher
1765    // priority than authenticated http:// uris
1766    if (allowUnauth == true && seenUntrusted == true)
1767       Trusted = false;
1768
1769    // Select a source
1770    if (QueueNext() == false && _error->PendingError() == false)
1771       _error->Error(_("I wasn't able to locate a file for the %s package. "
1772                     "This might mean you need to manually fix this package."),
1773                     Version.ParentPkg().Name());
1774 }
1775                                                                         /*}}}*/
1776 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1777 // ---------------------------------------------------------------------
1778 /* This queues the next available file version for download. It checks if
1779    the archive is already available in the cache and stashs the MD5 for
1780    checking later. */
1781 bool pkgAcqArchive::QueueNext()
1782 {
1783    string const ForceHash = _config->Find("Acquire::ForceHash");
1784    for (; Vf.end() == false; ++Vf)
1785    {
1786       // Ignore not source sources
1787       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1788          continue;
1789
1790       // Try to cross match against the source list
1791       pkgIndexFile *Index;
1792       if (Sources->FindIndex(Vf.File(),Index) == false)
1793             continue;
1794
1795       // only try to get a trusted package from another source if that source
1796       // is also trusted
1797       if(Trusted && !Index->IsTrusted())
1798          continue;
1799
1800       // Grab the text package record
1801       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1802       if (_error->PendingError() == true)
1803          return false;
1804
1805       string PkgFile = Parse.FileName();
1806       if (ForceHash.empty() == false)
1807       {
1808          if(stringcasecmp(ForceHash, "sha512") == 0)
1809             ExpectedHash = HashString("SHA512", Parse.SHA512Hash());
1810          else if(stringcasecmp(ForceHash, "sha256") == 0)
1811             ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
1812          else if (stringcasecmp(ForceHash, "sha1") == 0)
1813             ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
1814          else
1815             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1816       }
1817       else
1818       {
1819          string Hash;
1820          if ((Hash = Parse.SHA512Hash()).empty() == false)
1821             ExpectedHash = HashString("SHA512", Hash);
1822          else if ((Hash = Parse.SHA256Hash()).empty() == false)
1823             ExpectedHash = HashString("SHA256", Hash);
1824          else if ((Hash = Parse.SHA1Hash()).empty() == false)
1825             ExpectedHash = HashString("SHA1", Hash);
1826          else
1827             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1828       }
1829       if (PkgFile.empty() == true)
1830          return _error->Error(_("The package index files are corrupted. No Filename: "
1831                               "field for package %s."),
1832                               Version.ParentPkg().Name());
1833
1834       Desc.URI = Index->ArchiveURI(PkgFile);
1835       Desc.Description = Index->ArchiveInfo(Version);
1836       Desc.Owner = this;
1837       Desc.ShortDesc = Version.ParentPkg().Name();
1838
1839       // See if we already have the file. (Legacy filenames)
1840       FileSize = Version->Size;
1841       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
1842       struct stat Buf;
1843       if (stat(FinalFile.c_str(),&Buf) == 0)
1844       {
1845          // Make sure the size matches
1846          if ((unsigned long long)Buf.st_size == Version->Size)
1847          {
1848             Complete = true;
1849             Local = true;
1850             Status = StatDone;
1851             StoreFilename = DestFile = FinalFile;
1852             return true;
1853          }
1854
1855          /* Hmm, we have a file and its size does not match, this means it is
1856             an old style mismatched arch */
1857          unlink(FinalFile.c_str());
1858       }
1859
1860       // Check it again using the new style output filenames
1861       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
1862       if (stat(FinalFile.c_str(),&Buf) == 0)
1863       {
1864          // Make sure the size matches
1865          if ((unsigned long long)Buf.st_size == Version->Size)
1866          {
1867             Complete = true;
1868             Local = true;
1869             Status = StatDone;
1870             StoreFilename = DestFile = FinalFile;
1871             return true;
1872          }
1873
1874          /* Hmm, we have a file and its size does not match, this shouldnt
1875             happen.. */
1876          unlink(FinalFile.c_str());
1877       }
1878
1879       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
1880
1881       // Check the destination file
1882       if (stat(DestFile.c_str(),&Buf) == 0)
1883       {
1884          // Hmm, the partial file is too big, erase it
1885          if ((unsigned long long)Buf.st_size > Version->Size)
1886             unlink(DestFile.c_str());
1887          else
1888             PartialSize = Buf.st_size;
1889       }
1890
1891       // Disables download of archives - useful if no real installation follows,
1892       // e.g. if we are just interested in proposed installation order
1893       if (_config->FindB("Debug::pkgAcqArchive::NoQueue", false) == true)
1894       {
1895          Complete = true;
1896          Local = true;
1897          Status = StatDone;
1898          StoreFilename = DestFile = FinalFile;
1899          return true;
1900       }
1901
1902       // Create the item
1903       Local = false;
1904       Desc.URI = Index->ArchiveURI(PkgFile);
1905       Desc.Description = Index->ArchiveInfo(Version);
1906       Desc.Owner = this;
1907       Desc.ShortDesc = Version.ParentPkg().Name();
1908       QueueURI(Desc);
1909
1910       ++Vf;
1911       return true;
1912    }
1913    return false;
1914 }
1915                                                                         /*}}}*/
1916 // AcqArchive::Done - Finished fetching                                 /*{{{*/
1917 // ---------------------------------------------------------------------
1918 /* */
1919 void pkgAcqArchive::Done(string Message,unsigned long long Size,string CalcHash,
1920                          pkgAcquire::MethodConfig *Cfg)
1921 {
1922    Item::Done(Message,Size,CalcHash,Cfg);
1923
1924    // Check the size
1925    if (Size != Version->Size)
1926    {
1927       RenameOnError(SizeMismatch);
1928       return;
1929    }
1930
1931    // Check the hash
1932    if(ExpectedHash.toStr() != CalcHash)
1933    {
1934       RenameOnError(HashSumMismatch);
1935       return;
1936    }
1937
1938    // Grab the output filename
1939    string FileName = LookupTag(Message,"Filename");
1940    if (FileName.empty() == true)
1941    {
1942       Status = StatError;
1943       ErrorText = "Method gave a blank filename";
1944       return;
1945    }
1946
1947    Complete = true;
1948
1949    // Reference filename
1950    if (FileName != DestFile)
1951    {
1952       StoreFilename = DestFile = FileName;
1953       Local = true;
1954       return;
1955    }
1956
1957    // Done, move it into position
1958    string FinalFile = _config->FindDir("Dir::Cache::Archives");
1959    FinalFile += flNotDir(StoreFilename);
1960    Rename(DestFile,FinalFile);
1961
1962    StoreFilename = DestFile = FinalFile;
1963    Complete = true;
1964 }
1965                                                                         /*}}}*/
1966 // AcqArchive::Failed - Failure handler                                 /*{{{*/
1967 // ---------------------------------------------------------------------
1968 /* Here we try other sources */
1969 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1970 {
1971    ErrorText = LookupTag(Message,"Message");
1972
1973    /* We don't really want to retry on failed media swaps, this prevents
1974       that. An interesting observation is that permanent failures are not
1975       recorded. */
1976    if (Cnf->Removable == true &&
1977        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1978    {
1979       // Vf = Version.FileList();
1980       while (Vf.end() == false) ++Vf;
1981       StoreFilename = string();
1982       Item::Failed(Message,Cnf);
1983       return;
1984    }
1985
1986    if (QueueNext() == false)
1987    {
1988       // This is the retry counter
1989       if (Retries != 0 &&
1990           Cnf->LocalOnly == false &&
1991           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1992       {
1993          Retries--;
1994          Vf = Version.FileList();
1995          if (QueueNext() == true)
1996             return;
1997       }
1998
1999       StoreFilename = string();
2000       Item::Failed(Message,Cnf);
2001    }
2002 }
2003                                                                         /*}}}*/
2004 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
2005 // ---------------------------------------------------------------------
2006 bool pkgAcqArchive::IsTrusted()
2007 {
2008    return Trusted;
2009 }
2010                                                                         /*}}}*/
2011 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
2012 // ---------------------------------------------------------------------
2013 /* */
2014 void pkgAcqArchive::Finished()
2015 {
2016    if (Status == pkgAcquire::Item::StatDone &&
2017        Complete == true)
2018       return;
2019    StoreFilename = string();
2020 }
2021                                                                         /*}}}*/
2022 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
2023 // ---------------------------------------------------------------------
2024 /* The file is added to the queue */
2025 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
2026                        unsigned long long Size,string Dsc,string ShortDesc,
2027                        const string &DestDir, const string &DestFilename,
2028                        bool IsIndexFile) :
2029                        Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile)
2030 {
2031    Retries = _config->FindI("Acquire::Retries",0);
2032
2033    if(!DestFilename.empty())
2034       DestFile = DestFilename;
2035    else if(!DestDir.empty())
2036       DestFile = DestDir + "/" + flNotDir(URI);
2037    else
2038       DestFile = flNotDir(URI);
2039
2040    // Create the item
2041    Desc.URI = URI;
2042    Desc.Description = Dsc;
2043    Desc.Owner = this;
2044
2045    // Set the short description to the archive component
2046    Desc.ShortDesc = ShortDesc;
2047
2048    // Get the transfer sizes
2049    FileSize = Size;
2050    struct stat Buf;
2051    if (stat(DestFile.c_str(),&Buf) == 0)
2052    {
2053       // Hmm, the partial file is too big, erase it
2054       if ((unsigned long long)Buf.st_size > Size)
2055          unlink(DestFile.c_str());
2056       else
2057          PartialSize = Buf.st_size;
2058    }
2059
2060    QueueURI(Desc);
2061 }
2062                                                                         /*}}}*/
2063 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
2064 // ---------------------------------------------------------------------
2065 /* */
2066 void pkgAcqFile::Done(string Message,unsigned long long Size,string CalcHash,
2067                       pkgAcquire::MethodConfig *Cnf)
2068 {
2069    Item::Done(Message,Size,CalcHash,Cnf);
2070
2071    // Check the hash
2072    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
2073    {
2074       RenameOnError(HashSumMismatch);
2075       return;
2076    }
2077
2078    string FileName = LookupTag(Message,"Filename");
2079    if (FileName.empty() == true)
2080    {
2081       Status = StatError;
2082       ErrorText = "Method gave a blank filename";
2083       return;
2084    }
2085
2086    Complete = true;
2087
2088    // The files timestamp matches
2089    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
2090       return;
2091
2092    // We have to copy it into place
2093    if (FileName != DestFile)
2094    {
2095       Local = true;
2096       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
2097           Cnf->Removable == true)
2098       {
2099          Desc.URI = "copy:" + FileName;
2100          QueueURI(Desc);
2101          return;
2102       }
2103
2104       // Erase the file if it is a symlink so we can overwrite it
2105       struct stat St;
2106       if (lstat(DestFile.c_str(),&St) == 0)
2107       {
2108          if (S_ISLNK(St.st_mode) != 0)
2109             unlink(DestFile.c_str());
2110       }
2111
2112       // Symlink the file
2113       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
2114       {
2115          ErrorText = "Link to " + DestFile + " failure ";
2116          Status = StatError;
2117          Complete = false;
2118       }
2119    }
2120 }
2121                                                                         /*}}}*/
2122 // AcqFile::Failed - Failure handler                                    /*{{{*/
2123 // ---------------------------------------------------------------------
2124 /* Here we try other sources */
2125 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2126 {
2127    ErrorText = LookupTag(Message,"Message");
2128
2129    // This is the retry counter
2130    if (Retries != 0 &&
2131        Cnf->LocalOnly == false &&
2132        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2133    {
2134       Retries--;
2135       QueueURI(Desc);
2136       return;
2137    }
2138
2139    Item::Failed(Message,Cnf);
2140 }
2141                                                                         /*}}}*/
2142 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
2143 // ---------------------------------------------------------------------
2144 /* The only header we use is the last-modified header. */
2145 string pkgAcqFile::Custom600Headers()
2146 {
2147    if (IsIndexFile)
2148       return "\nIndex-File: true";
2149    return "";
2150 }
2151                                                                         /*}}}*/