apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <config.h>
  17
  18 #include <apt-pkg/acquire-item.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/aptconfiguration.h>
  21 #include <apt-pkg/sourcelist.h>
  22 #include <apt-pkg/error.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/md5.h>
  26 #include <apt-pkg/sha1.h>
  27 #include <apt-pkg/tagfile.h>
  28 #include <apt-pkg/indexrecords.h>
  29 #include <apt-pkg/metaindex.h>
  30
  31 #include <sys/stat.h>
  32 #include <unistd.h>
  33 #include <errno.h>
  34 #include <string>
  35 #include <sstream>
  36 #include <stdio.h>
  37 #include <ctime>
  38
  39 #include <apti18n.h>
  40                                                                         /*}}}*/
  41
  42 using namespace std;
  43
  44 // Acquire::Item::Item - Constructor                                    /*{{{*/
  45 // ---------------------------------------------------------------------
  46 /* */
  47 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  48                        PartialSize(0), Mode(0), ID(0), Complete(false),
  49                        Local(false), QueueCounter(0)
  50 {
  51    Owner->Add(this);
  52    Status = StatIdle;
  53 }
  54                                                                         /*}}}*/
  55 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  56 // ---------------------------------------------------------------------
  57 /* */
  58 pkgAcquire::Item::~Item()
  59 {
  60    Owner->Remove(this);
  61 }
  62                                                                         /*}}}*/
  63 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  64 // ---------------------------------------------------------------------
  65 /* We return to an idle state if there are still other queues that could
  66    fetch this object */
  67 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  68 {
  69    Status = StatIdle;
  70    ErrorText = LookupTag(Message,"Message");
  71    UsedMirror =  LookupTag(Message,"UsedMirror");
  72    if (QueueCounter <= 1)
  73    {
  74       /* This indicates that the file is not available right now but might
  75          be sometime later. If we do a retry cycle then this should be
  76          retried [CDROMs] */
  77       if (Cnf->LocalOnly == true &&
  78           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  79       {
  80          Status = StatIdle;
  81          Dequeue();
  82          return;
  83       }
  84
  85       Status = StatError;
  86       Dequeue();
  87    }
  88
  89    // report mirror failure back to LP if we actually use a mirror
  90    string FailReason = LookupTag(Message, "FailReason");
  91    if(FailReason.size() != 0)
  92       ReportMirrorFailure(FailReason);
  93    else
  94       ReportMirrorFailure(ErrorText);
  95 }
  96                                                                         /*}}}*/
  97 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
  98 // ---------------------------------------------------------------------
  99 /* Stash status and the file size. Note that setting Complete means
 100    sub-phases of the acquire process such as decompresion are operating */
 101 void pkgAcquire::Item::Start(string /*Message*/,unsigned long long Size)
 102 {
 103    Status = StatFetching;
 104    if (FileSize == 0 && Complete == false)
 105       FileSize = Size;
 106 }
 107                                                                         /*}}}*/
 108 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
 109 // ---------------------------------------------------------------------
 110 /* */
 111 void pkgAcquire::Item::Done(string Message,unsigned long long Size,string Hash,
 112                             pkgAcquire::MethodConfig *Cnf)
 113 {
 114    // We just downloaded something..
 115    string FileName = LookupTag(Message,"Filename");
 116    UsedMirror =  LookupTag(Message,"UsedMirror");
 117    if (Complete == false && !Local && FileName == DestFile)
 118    {
 119       if (Owner->Log != 0)
 120          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 121    }
 122
 123    if (FileSize == 0)
 124       FileSize= Size;
 125    Status = StatDone;
 126    ErrorText = string();
 127    Owner->Dequeue(this);
 128 }
 129                                                                         /*}}}*/
 130 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 131 // ---------------------------------------------------------------------
 132 /* This helper function is used by alot of item methods as thier final
 133    step */
 134 void pkgAcquire::Item::Rename(string From,string To)
 135 {
 136    if (rename(From.c_str(),To.c_str()) != 0)
 137    {
 138       char S[300];
 139       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 140               From.c_str(),To.c_str());
 141       Status = StatError;
 142       ErrorText = S;
 143    }
 144 }
 145                                                                         /*}}}*/
 146 // Acquire::Item::ReportMirrorFailure                                   /*{{{*/
 147 // ---------------------------------------------------------------------
 148 void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
 149 {
 150    // we only act if a mirror was used at all
 151    if(UsedMirror.empty())
 152       return;
 153 #if 0
 154    std::cerr << "\nReportMirrorFailure: "
 155              << UsedMirror
 156              << " Uri: " << DescURI()
 157              << " FailCode: "
 158              << FailCode << std::endl;
 159 #endif
 160    const char *Args[40];
 161    unsigned int i = 0;
 162    string report = _config->Find("Methods::Mirror::ProblemReporting",
 163                                  "/usr/lib/apt/apt-report-mirror-failure");
 164    if(!FileExists(report))
 165       return;
 166    Args[i++] = report.c_str();
 167    Args[i++] = UsedMirror.c_str();
 168    Args[i++] = DescURI().c_str();
 169    Args[i++] = FailCode.c_str();
 170    Args[i++] = NULL;
 171    pid_t pid = ExecFork();
 172    if(pid < 0)
 173    {
 174       _error->Error("ReportMirrorFailure Fork failed");
 175       return;
 176    }
 177    else if(pid == 0)
 178    {
 179       execvp(Args[0], (char**)Args);
 180       std::cerr << "Could not exec " << Args[0] << std::endl;
 181       _exit(100);
 182    }
 183    if(!ExecWait(pid, "report-mirror-failure"))
 184    {
 185       _error->Warning("Couldn't report problem to '%s'",
 186                       _config->Find("Methods::Mirror::ProblemReporting").c_str());
 187    }
 188 }
 189                                                                         /*}}}*/
 190 // AcqSubIndex::AcqSubIndex - Constructor                               /*{{{*/
 191 // ---------------------------------------------------------------------
 192 /* Get a sub-index file based on checksums from a 'master' file and
 193    possibly query additional files */
 194 pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, string const &URI,
 195                                  string const &URIDesc, string const &ShortDesc,
 196                                  HashString const &ExpectedHash)
 197    : Item(Owner), ExpectedHash(ExpectedHash)
 198 {
 199    /* XXX: Beware: Currently this class does nothing (of value) anymore ! */
 200    Debug = _config->FindB("Debug::pkgAcquire::SubIndex",false);
 201
 202    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 203    DestFile += URItoFileName(URI);
 204
 205    Desc.URI = URI;
 206    Desc.Description = URIDesc;
 207    Desc.Owner = this;
 208    Desc.ShortDesc = ShortDesc;
 209
 210    QueueURI(Desc);
 211
 212    if(Debug)
 213       std::clog << "pkgAcqSubIndex: " << Desc.URI << std::endl;
 214 }
 215                                                                         /*}}}*/
 216 // AcqSubIndex::Custom600Headers - Insert custom request headers        /*{{{*/
 217 // ---------------------------------------------------------------------
 218 /* The only header we use is the last-modified header. */
 219 string pkgAcqSubIndex::Custom600Headers()
 220 {
 221    string Final = _config->FindDir("Dir::State::lists");
 222    Final += URItoFileName(Desc.URI);
 223
 224    struct stat Buf;
 225    if (stat(Final.c_str(),&Buf) != 0)
 226       return "\nIndex-File: true\nFail-Ignore: true\n";
 227    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 228 }
 229                                                                         /*}}}*/
 230 void pkgAcqSubIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)       /*{{{*/
 231 {
 232    if(Debug)
 233       std::clog << "pkgAcqSubIndex failed: " << Desc.URI << std::endl;
 234
 235    Complete = false;
 236    Status = StatDone;
 237    Dequeue();
 238
 239    // No good Index is provided
 240 }
 241                                                                         /*}}}*/
 242 void pkgAcqSubIndex::Done(string Message,unsigned long long Size,string Md5Hash,        /*{{{*/
 243                            pkgAcquire::MethodConfig *Cnf)
 244 {
 245    if(Debug)
 246       std::clog << "pkgAcqSubIndex::Done(): " << Desc.URI << std::endl;
 247
 248    string FileName = LookupTag(Message,"Filename");
 249    if (FileName.empty() == true)
 250    {
 251       Status = StatError;
 252       ErrorText = "Method gave a blank filename";
 253       return;
 254    }
 255
 256    if (FileName != DestFile)
 257    {
 258       Local = true;
 259       Desc.URI = "copy:" + FileName;
 260       QueueURI(Desc);
 261       return;
 262    }
 263
 264    Item::Done(Message,Size,Md5Hash,Cnf);
 265
 266    string FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(Desc.URI);
 267
 268    /* Downloaded invalid transindex => Error (LP: #346386) (Closes: #627642) */
 269    indexRecords SubIndexParser;
 270    if (FileExists(DestFile) == true && !SubIndexParser.Load(DestFile)) {
 271       Status = StatError;
 272       ErrorText = SubIndexParser.ErrorText;
 273       return;
 274    }
 275
 276    // sucess in downloading the index
 277    // rename the index
 278    if(Debug)
 279       std::clog << "Renaming: " << DestFile << " -> " << FinalFile << std::endl;
 280    Rename(DestFile,FinalFile);
 281    chmod(FinalFile.c_str(),0644);
 282    DestFile = FinalFile;
 283
 284    if(ParseIndex(DestFile) == false)
 285       return Failed("", NULL);
 286
 287    Complete = true;
 288    Status = StatDone;
 289    Dequeue();
 290    return;
 291 }
 292                                                                         /*}}}*/
 293 bool pkgAcqSubIndex::ParseIndex(string const &IndexFile)                /*{{{*/
 294 {
 295    indexRecords SubIndexParser;
 296    if (FileExists(IndexFile) == false || SubIndexParser.Load(IndexFile) == false)
 297       return false;
 298    // so something with the downloaded index
 299    return true;
 300 }
 301                                                                         /*}}}*/
 302 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 303 // ---------------------------------------------------------------------
 304 /* Get the DiffIndex file first and see if there are patches availabe
 305  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 306  * patches. If anything goes wrong in that process, it will fall back to
 307  * the original packages file
 308  */
 309 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 310                                  string URI,string URIDesc,string ShortDesc,
 311                                  HashString ExpectedHash)
 312    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 313      Description(URIDesc)
 314 {
 315
 316    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 317
 318    Desc.Description = URIDesc + "/DiffIndex";
 319    Desc.Owner = this;
 320    Desc.ShortDesc = ShortDesc;
 321    Desc.URI = URI + ".diff/Index";
 322
 323    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 324    DestFile += URItoFileName(URI) + string(".DiffIndex");
 325
 326    if(Debug)
 327       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 328
 329    // look for the current package file
 330    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 331    CurrentPackagesFile += URItoFileName(RealURI);
 332
 333    // FIXME: this file:/ check is a hack to prevent fetching
 334    //        from local sources. this is really silly, and
 335    //        should be fixed cleanly as soon as possible
 336    if(!FileExists(CurrentPackagesFile) ||
 337       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 338    {
 339       // we don't have a pkg file or we don't want to queue
 340       if(Debug)
 341          std::clog << "No index file, local or canceld by user" << std::endl;
 342       Failed("", NULL);
 343       return;
 344    }
 345
 346    if(Debug)
 347       std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): "
 348                 << CurrentPackagesFile << std::endl;
 349
 350    QueueURI(Desc);
 351
 352 }
 353                                                                         /*}}}*/
 354 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 355 // ---------------------------------------------------------------------
 356 /* The only header we use is the last-modified header. */
 357 string pkgAcqDiffIndex::Custom600Headers()
 358 {
 359    string Final = _config->FindDir("Dir::State::lists");
 360    Final += URItoFileName(RealURI) + string(".IndexDiff");
 361
 362    if(Debug)
 363       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 364
 365    struct stat Buf;
 366    if (stat(Final.c_str(),&Buf) != 0)
 367       return "\nIndex-File: true";
 368
 369    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 370 }
 371                                                                         /*}}}*/
 372 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 373 {
 374    if(Debug)
 375       std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile
 376                 << std::endl;
 377
 378    pkgTagSection Tags;
 379    string ServerSha1;
 380    vector<DiffInfo> available_patches;
 381
 382    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 383    pkgTagFile TF(&Fd);
 384    if (_error->PendingError() == true)
 385       return false;
 386
 387    if(TF.Step(Tags) == true)
 388    {
 389       bool found = false;
 390       DiffInfo d;
 391       string size;
 392
 393       string const tmp = Tags.FindS("SHA1-Current");
 394       std::stringstream ss(tmp);
 395       ss >> ServerSha1 >> size;
 396       unsigned long const ServerSize = atol(size.c_str());
 397
 398       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 399       SHA1Summation SHA1;
 400       SHA1.AddFD(fd);
 401       string const local_sha1 = SHA1.Result();
 402
 403       if(local_sha1 == ServerSha1)
 404       {
 405          // we have the same sha1 as the server
 406          if(Debug)
 407             std::clog << "Package file is up-to-date" << std::endl;
 408          // set found to true, this will queue a pkgAcqIndexDiffs with
 409          // a empty availabe_patches
 410          found = true;
 411       }
 412       else
 413       {
 414          if(Debug)
 415             std::clog << "SHA1-Current: " << ServerSha1 << " and we start at "<< fd.Name() << " " << fd.Size() << " " << local_sha1 << std::endl;
 416
 417          // check the historie and see what patches we need
 418          string const history = Tags.FindS("SHA1-History");
 419          std::stringstream hist(history);
 420          while(hist >> d.sha1 >> size >> d.file)
 421          {
 422             // read until the first match is found
 423             // from that point on, we probably need all diffs
 424             if(d.sha1 == local_sha1)
 425                found=true;
 426             else if (found == false)
 427                continue;
 428
 429             if(Debug)
 430                std::clog << "Need to get diff: " << d.file << std::endl;
 431             available_patches.push_back(d);
 432          }
 433
 434          if (available_patches.empty() == false)
 435          {
 436             // patching with too many files is rather slow compared to a fast download
 437             unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 438             if (fileLimit != 0 && fileLimit < available_patches.size())
 439             {
 440                if (Debug)
 441                   std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 442                         << ") so fallback to complete download" << std::endl;
 443                return false;
 444             }
 445
 446             // see if the patches are too big
 447             found = false; // it was true and it will be true again at the end
 448             d = *available_patches.begin();
 449             string const firstPatch = d.file;
 450             unsigned long patchesSize = 0;
 451             std::stringstream patches(Tags.FindS("SHA1-Patches"));
 452             while(patches >> d.sha1 >> size >> d.file)
 453             {
 454                if (firstPatch == d.file)
 455                   found = true;
 456                else if (found == false)
 457                   continue;
 458
 459                patchesSize += atol(size.c_str());
 460             }
 461             unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 462             if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 463             {
 464                if (Debug)
 465                   std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 466                         << ") so fallback to complete download" << std::endl;
 467                return false;
 468             }
 469          }
 470       }
 471
 472       // we have something, queue the next diff
 473       if(found)
 474       {
 475          // queue the diffs
 476          string::size_type const last_space = Description.rfind(" ");
 477          if(last_space != string::npos)
 478             Description.erase(last_space, Description.size()-last_space);
 479          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 480                               ExpectedHash, ServerSha1, available_patches);
 481          Complete = false;
 482          Status = StatDone;
 483          Dequeue();
 484          return true;
 485       }
 486    }
 487
 488    // Nothing found, report and return false
 489    // Failing here is ok, if we return false later, the full
 490    // IndexFile is queued
 491    if(Debug)
 492       std::clog << "Can't find a patch in the index file" << std::endl;
 493    return false;
 494 }
 495                                                                         /*}}}*/
 496 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)      /*{{{*/
 497 {
 498    if(Debug)
 499       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << std::endl
 500                 << "Falling back to normal index file aquire" << std::endl;
 501
 502    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 503                    ExpectedHash);
 504
 505    Complete = false;
 506    Status = StatDone;
 507    Dequeue();
 508 }
 509                                                                         /*}}}*/
 510 void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,string Md5Hash,       /*{{{*/
 511                            pkgAcquire::MethodConfig *Cnf)
 512 {
 513    if(Debug)
 514       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 515
 516    Item::Done(Message,Size,Md5Hash,Cnf);
 517
 518    string FinalFile;
 519    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 520
 521    // sucess in downloading the index
 522    // rename the index
 523    FinalFile += string(".IndexDiff");
 524    if(Debug)
 525       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 526                 << std::endl;
 527    Rename(DestFile,FinalFile);
 528    chmod(FinalFile.c_str(),0644);
 529    DestFile = FinalFile;
 530
 531    if(!ParseDiffIndex(DestFile))
 532       return Failed("", NULL);
 533
 534    Complete = true;
 535    Status = StatDone;
 536    Dequeue();
 537    return;
 538 }
 539                                                                         /*}}}*/
 540 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 541 // ---------------------------------------------------------------------
 542 /* The package diff is added to the queue. one object is constructed
 543  * for each diff and the index
 544  */
 545 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 546                                    string URI,string URIDesc,string ShortDesc,
 547                                    HashString ExpectedHash,
 548                                    string ServerSha1,
 549                                    vector<DiffInfo> diffs)
 550    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 551      available_patches(diffs), ServerSha1(ServerSha1)
 552 {
 553
 554    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 555    DestFile += URItoFileName(URI);
 556
 557    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 558
 559    Description = URIDesc;
 560    Desc.Owner = this;
 561    Desc.ShortDesc = ShortDesc;
 562
 563    if(available_patches.empty() == true)
 564    {
 565       // we are done (yeah!)
 566       Finish(true);
 567    }
 568    else
 569    {
 570       // get the next diff
 571       State = StateFetchDiff;
 572       QueueNextDiff();
 573    }
 574 }
 575                                                                         /*}}}*/
 576 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)     /*{{{*/
 577 {
 578    if(Debug)
 579       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl
 580                 << "Falling back to normal index file aquire" << std::endl;
 581    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 582                    ExpectedHash);
 583    Finish();
 584 }
 585                                                                         /*}}}*/
 586 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 587 void pkgAcqIndexDiffs::Finish(bool allDone)
 588 {
 589    // we restore the original name, this is required, otherwise
 590    // the file will be cleaned
 591    if(allDone)
 592    {
 593       DestFile = _config->FindDir("Dir::State::lists");
 594       DestFile += URItoFileName(RealURI);
 595
 596       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 597       {
 598          Status = StatAuthError;
 599          ErrorText = _("MD5Sum mismatch");
 600          Rename(DestFile,DestFile + ".FAILED");
 601          Dequeue();
 602          return;
 603       }
 604
 605       // this is for the "real" finish
 606       Complete = true;
 607       Status = StatDone;
 608       Dequeue();
 609       if(Debug)
 610          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 611       return;
 612    }
 613
 614    if(Debug)
 615       std::clog << "Finishing: " << Desc.URI << std::endl;
 616    Complete = false;
 617    Status = StatDone;
 618    Dequeue();
 619    return;
 620 }
 621                                                                         /*}}}*/
 622 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 623 {
 624
 625    // calc sha1 of the just patched file
 626    string FinalFile = _config->FindDir("Dir::State::lists");
 627    FinalFile += URItoFileName(RealURI);
 628
 629    FileFd fd(FinalFile, FileFd::ReadOnly);
 630    SHA1Summation SHA1;
 631    SHA1.AddFD(fd);
 632    string local_sha1 = string(SHA1.Result());
 633    if(Debug)
 634       std::clog << "QueueNextDiff: "
 635                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 636
 637    // final file reached before all patches are applied
 638    if(local_sha1 == ServerSha1)
 639    {
 640       Finish(true);
 641       return true;
 642    }
 643
 644    // remove all patches until the next matching patch is found
 645    // this requires the Index file to be ordered
 646    for(vector<DiffInfo>::iterator I=available_patches.begin();
 647        available_patches.empty() == false &&
 648           I != available_patches.end() &&
 649           I->sha1 != local_sha1;
 650        ++I)
 651    {
 652       available_patches.erase(I);
 653    }
 654
 655    // error checking and falling back if no patch was found
 656    if(available_patches.empty() == true)
 657    {
 658       Failed("", NULL);
 659       return false;
 660    }
 661
 662    // queue the right diff
 663    Desc.URI = string(RealURI) + ".diff/" + available_patches[0].file + ".gz";
 664    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 665    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 666    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 667
 668    if(Debug)
 669       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 670
 671    QueueURI(Desc);
 672
 673    return true;
 674 }
 675                                                                         /*}}}*/
 676 void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size,string Md5Hash,      /*{{{*/
 677                             pkgAcquire::MethodConfig *Cnf)
 678 {
 679    if(Debug)
 680       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 681
 682    Item::Done(Message,Size,Md5Hash,Cnf);
 683
 684    string FinalFile;
 685    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 686
 687    // sucess in downloading a diff, enter ApplyDiff state
 688    if(State == StateFetchDiff)
 689    {
 690
 691       // rred excepts the patch as $FinalFile.ed
 692       Rename(DestFile,FinalFile+".ed");
 693
 694       if(Debug)
 695          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 696
 697       State = StateApplyDiff;
 698       Local = true;
 699       Desc.URI = "rred:" + FinalFile;
 700       QueueURI(Desc);
 701       Mode = "rred";
 702       return;
 703    }
 704
 705
 706    // success in download/apply a diff, queue next (if needed)
 707    if(State == StateApplyDiff)
 708    {
 709       // remove the just applied patch
 710       available_patches.erase(available_patches.begin());
 711
 712       // move into place
 713       if(Debug)
 714       {
 715          std::clog << "Moving patched file in place: " << std::endl
 716                    << DestFile << " -> " << FinalFile << std::endl;
 717       }
 718       Rename(DestFile,FinalFile);
 719       chmod(FinalFile.c_str(),0644);
 720
 721       // see if there is more to download
 722       if(available_patches.empty() == false) {
 723          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 724                               ExpectedHash, ServerSha1, available_patches);
 725          return Finish();
 726       } else
 727          return Finish(true);
 728    }
 729 }
 730                                                                         /*}}}*/
 731 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 732 // ---------------------------------------------------------------------
 733 /* The package file is added to the queue and a second class is
 734    instantiated to fetch the revision file */
 735 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 736                          string URI,string URIDesc,string ShortDesc,
 737                          HashString ExpectedHash, string comprExt)
 738    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 739 {
 740    if(comprExt.empty() == true)
 741    {
 742       // autoselect the compression method
 743       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 744       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 745          comprExt.append(*t).append(" ");
 746       if (comprExt.empty() == false)
 747          comprExt.erase(comprExt.end()-1);
 748    }
 749    CompressionExtension = comprExt;
 750
 751    Init(URI, URIDesc, ShortDesc);
 752 }
 753 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target,
 754                          HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
 755    : Item(Owner), RealURI(Target->URI), ExpectedHash(ExpectedHash)
 756 {
 757    // autoselect the compression method
 758    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 759    CompressionExtension = "";
 760    if (ExpectedHash.empty() == false)
 761    {
 762       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 763          if (*t == "uncompressed" || MetaIndexParser->Exists(string(Target->MetaKey).append(".").append(*t)) == true)
 764             CompressionExtension.append(*t).append(" ");
 765    }
 766    else
 767    {
 768       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 769          CompressionExtension.append(*t).append(" ");
 770    }
 771    if (CompressionExtension.empty() == false)
 772       CompressionExtension.erase(CompressionExtension.end()-1);
 773
 774    // only verify non-optional targets, see acquire-item.h for a FIXME
 775    // to make this more flexible
 776    if (Target->IsOptional())
 777      Verify = false;
 778    else
 779      Verify = true;
 780
 781    Init(Target->URI, Target->Description, Target->ShortDesc);
 782 }
 783                                                                         /*}}}*/
 784 // AcqIndex::Init - defered Constructor                                 /*{{{*/
 785 void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &ShortDesc) {
 786    Decompression = false;
 787    Erase = false;
 788
 789    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 790    DestFile += URItoFileName(URI);
 791
 792    std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
 793    if (comprExt == "uncompressed")
 794       Desc.URI = URI;
 795    else
 796       Desc.URI = URI + '.' + comprExt;
 797
 798    Desc.Description = URIDesc;
 799    Desc.Owner = this;
 800    Desc.ShortDesc = ShortDesc;
 801
 802    QueueURI(Desc);
 803 }
 804                                                                         /*}}}*/
 805 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 806 // ---------------------------------------------------------------------
 807 /* The only header we use is the last-modified header. */
 808 string pkgAcqIndex::Custom600Headers()
 809 {
 810    string Final = _config->FindDir("Dir::State::lists");
 811    Final += URItoFileName(RealURI);
 812    if (_config->FindB("Acquire::GzipIndexes",false))
 813       Final += ".gz";
 814
 815    string msg = "\nIndex-File: true";
 816    // FIXME: this really should use "IndexTarget::IsOptional()" but that
 817    //        seems to be difficult without breaking ABI
 818    if (ShortDesc().find("Translation") != 0)
 819       msg += "\nFail-Ignore: true";
 820    struct stat Buf;
 821    if (stat(Final.c_str(),&Buf) == 0)
 822       msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 823
 824    return msg;
 825 }
 826                                                                         /*}}}*/
 827 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
 828 {
 829    size_t const nextExt = CompressionExtension.find(' ');
 830    if (nextExt != std::string::npos)
 831    {
 832       CompressionExtension = CompressionExtension.substr(nextExt+1);
 833       Init(RealURI, Desc.Description, Desc.ShortDesc);
 834       return;
 835    }
 836
 837    // on decompression failure, remove bad versions in partial/
 838    if (Decompression && Erase) {
 839       string s = _config->FindDir("Dir::State::lists") + "partial/";
 840       s.append(URItoFileName(RealURI));
 841       unlink(s.c_str());
 842    }
 843
 844    Item::Failed(Message,Cnf);
 845 }
 846                                                                         /*}}}*/
 847 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
 848 // ---------------------------------------------------------------------
 849 /* This goes through a number of states.. On the initial fetch the
 850    method could possibly return an alternate filename which points
 851    to the uncompressed version of the file. If this is so the file
 852    is copied into the partial directory. In all other cases the file
 853    is decompressed with a gzip uri. */
 854 void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash,
 855                        pkgAcquire::MethodConfig *Cfg)
 856 {
 857    Item::Done(Message,Size,Hash,Cfg);
 858
 859    if (Decompression == true)
 860    {
 861       if (_config->FindB("Debug::pkgAcquire::Auth", false))
 862       {
 863          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
 864          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
 865       }
 866
 867       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
 868       {
 869          Status = StatAuthError;
 870          ErrorText = _("Hash Sum mismatch");
 871          Rename(DestFile,DestFile + ".FAILED");
 872          ReportMirrorFailure("HashChecksumFailure");
 873          return;
 874       }
 875
 876       /* Verify the index file for correctness (all indexes must
 877        * have a Package field) (LP: #346386) (Closes: #627642) */
 878       if (Verify == true)
 879       {
 880          FileFd fd(DestFile, FileFd::ReadOnly);
 881          pkgTagSection sec;
 882          pkgTagFile tag(&fd);
 883
 884          // Only test for correctness if the file is not empty (empty is ok)
 885          if (fd.Size() > 0) {
 886             if (_error->PendingError() || !tag.Step(sec)) {
 887                Status = StatError;
 888                _error->DumpErrors();
 889                Rename(DestFile,DestFile + ".FAILED");
 890                return;
 891             } else if (!sec.Exists("Package")) {
 892                Status = StatError;
 893                ErrorText = ("Encountered a section with no Package: header");
 894                Rename(DestFile,DestFile + ".FAILED");
 895                return;
 896             }
 897          }
 898       }
 899
 900       // Done, move it into position
 901       string FinalFile = _config->FindDir("Dir::State::lists");
 902       FinalFile += URItoFileName(RealURI);
 903       Rename(DestFile,FinalFile);
 904       chmod(FinalFile.c_str(),0644);
 905
 906       /* We restore the original name to DestFile so that the clean operation
 907          will work OK */
 908       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 909       DestFile += URItoFileName(RealURI);
 910
 911       // Remove the compressed version.
 912       if (Erase == true)
 913          unlink(DestFile.c_str());
 914       return;
 915    }
 916
 917    Erase = false;
 918    Complete = true;
 919
 920    // Handle the unzipd case
 921    string FileName = LookupTag(Message,"Alt-Filename");
 922    if (FileName.empty() == false)
 923    {
 924       // The files timestamp matches
 925       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
 926          return;
 927       Decompression = true;
 928       Local = true;
 929       DestFile += ".decomp";
 930       Desc.URI = "copy:" + FileName;
 931       QueueURI(Desc);
 932       Mode = "copy";
 933       return;
 934    }
 935
 936    FileName = LookupTag(Message,"Filename");
 937    if (FileName.empty() == true)
 938    {
 939       Status = StatError;
 940       ErrorText = "Method gave a blank filename";
 941    }
 942
 943    std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
 944
 945    // The files timestamp matches
 946    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) {
 947        if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
 948           // Update DestFile for .gz suffix so that the clean operation keeps it
 949           DestFile += ".gz";
 950       return;
 951     }
 952
 953    if (FileName == DestFile)
 954       Erase = true;
 955    else
 956       Local = true;
 957
 958    string decompProg;
 959
 960    // If we enable compressed indexes and already have gzip, keep it
 961    if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) {
 962       string FinalFile = _config->FindDir("Dir::State::lists");
 963       FinalFile += URItoFileName(RealURI) + ".gz";
 964       Rename(DestFile,FinalFile);
 965       chmod(FinalFile.c_str(),0644);
 966
 967       // Update DestFile for .gz suffix so that the clean operation keeps it
 968       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 969       DestFile += URItoFileName(RealURI) + ".gz";
 970       return;
 971     }
 972
 973    // get the binary name for your used compression type
 974    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
 975    if(decompProg.empty() == false);
 976    else if(compExt == "uncompressed")
 977       decompProg = "copy";
 978    else {
 979       _error->Error("Unsupported extension: %s", compExt.c_str());
 980       return;
 981    }
 982
 983    Decompression = true;
 984    DestFile += ".decomp";
 985    Desc.URI = decompProg + ":" + FileName;
 986    QueueURI(Desc);
 987    Mode = decompProg.c_str();
 988 }
 989                                                                         /*}}}*/
 990 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
 991 // ---------------------------------------------------------------------
 992 /* The Translation file is added to the queue */
 993 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
 994                             string URI,string URIDesc,string ShortDesc)
 995   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
 996 {
 997 }
 998 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const *Target,
 999                          HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
1000   : pkgAcqIndex(Owner, Target, ExpectedHash, MetaIndexParser)
1001 {
1002 }
1003                                                                         /*}}}*/
1004 // AcqIndexTrans::Custom600Headers - Insert custom request headers      /*{{{*/
1005 // ---------------------------------------------------------------------
1006 string pkgAcqIndexTrans::Custom600Headers()
1007 {
1008    string Final = _config->FindDir("Dir::State::lists");
1009    Final += URItoFileName(RealURI);
1010
1011    struct stat Buf;
1012    if (stat(Final.c_str(),&Buf) != 0)
1013       return "\nFail-Ignore: true\nIndex-File: true";
1014    return "\nFail-Ignore: true\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1015 }
1016                                                                         /*}}}*/
1017 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
1018 // ---------------------------------------------------------------------
1019 /* */
1020 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1021 {
1022    size_t const nextExt = CompressionExtension.find(' ');
1023    if (nextExt != std::string::npos)
1024    {
1025       CompressionExtension = CompressionExtension.substr(nextExt+1);
1026       Init(RealURI, Desc.Description, Desc.ShortDesc);
1027       Status = StatIdle;
1028       return;
1029    }
1030
1031    if (Cnf->LocalOnly == true ||
1032        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1033    {
1034       // Ignore this
1035       Status = StatDone;
1036       Complete = false;
1037       Dequeue();
1038       return;
1039    }
1040
1041    Item::Failed(Message,Cnf);
1042 }
1043                                                                         /*}}}*/
1044 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
1045                              string URI,string URIDesc,string ShortDesc,
1046                              string MetaIndexURI, string MetaIndexURIDesc,
1047                              string MetaIndexShortDesc,
1048                              const vector<IndexTarget*>* IndexTargets,
1049                              indexRecords* MetaIndexParser) :
1050    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
1051    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1052    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
1053 {
1054    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1055    DestFile += URItoFileName(URI);
1056
1057    // remove any partial downloaded sig-file in partial/.
1058    // it may confuse proxies and is too small to warrant a
1059    // partial download anyway
1060    unlink(DestFile.c_str());
1061
1062    // Create the item
1063    Desc.Description = URIDesc;
1064    Desc.Owner = this;
1065    Desc.ShortDesc = ShortDesc;
1066    Desc.URI = URI;
1067
1068    string Final = _config->FindDir("Dir::State::lists");
1069    Final += URItoFileName(RealURI);
1070    struct stat Buf;
1071    if (stat(Final.c_str(),&Buf) == 0)
1072    {
1073       // File was already in place.  It needs to be re-downloaded/verified
1074       // because Release might have changed, we do give it a differnt
1075       // name than DestFile because otherwise the http method will
1076       // send If-Range requests and there are too many broken servers
1077       // out there that do not understand them
1078       LastGoodSig = DestFile+".reverify";
1079       Rename(Final,LastGoodSig);
1080    }
1081
1082    QueueURI(Desc);
1083 }
1084                                                                         /*}}}*/
1085 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
1086 // ---------------------------------------------------------------------
1087 /* The only header we use is the last-modified header. */
1088 string pkgAcqMetaSig::Custom600Headers()
1089 {
1090    struct stat Buf;
1091    if (stat(LastGoodSig.c_str(),&Buf) != 0)
1092       return "\nIndex-File: true";
1093
1094    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1095 }
1096
1097 void pkgAcqMetaSig::Done(string Message,unsigned long long Size,string MD5,
1098                          pkgAcquire::MethodConfig *Cfg)
1099 {
1100    Item::Done(Message,Size,MD5,Cfg);
1101
1102    string FileName = LookupTag(Message,"Filename");
1103    if (FileName.empty() == true)
1104    {
1105       Status = StatError;
1106       ErrorText = "Method gave a blank filename";
1107       return;
1108    }
1109
1110    if (FileName != DestFile)
1111    {
1112       // We have to copy it into place
1113       Local = true;
1114       Desc.URI = "copy:" + FileName;
1115       QueueURI(Desc);
1116       return;
1117    }
1118
1119    Complete = true;
1120
1121    // put the last known good file back on i-m-s hit (it will
1122    // be re-verified again)
1123    // Else do nothing, we have the new file in DestFile then
1124    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1125       Rename(LastGoodSig, DestFile);
1126
1127    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
1128    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc,
1129                        MetaIndexShortDesc,  DestFile, IndexTargets,
1130                        MetaIndexParser);
1131
1132 }
1133                                                                         /*}}}*/
1134 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
1135 {
1136    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1137
1138    // if we get a network error we fail gracefully
1139    if(Status == StatTransientNetworkError)
1140    {
1141       Item::Failed(Message,Cnf);
1142       // move the sigfile back on transient network failures
1143       if(FileExists(LastGoodSig))
1144          Rename(LastGoodSig,Final);
1145
1146       // set the status back to , Item::Failed likes to reset it
1147       Status = pkgAcquire::Item::StatTransientNetworkError;
1148       return;
1149    }
1150
1151    // Delete any existing sigfile when the acquire failed
1152    unlink(Final.c_str());
1153
1154    // queue a pkgAcqMetaIndex with no sigfile
1155    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1156                        "", IndexTargets, MetaIndexParser);
1157
1158    if (Cnf->LocalOnly == true ||
1159        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1160    {
1161       // Ignore this
1162       Status = StatDone;
1163       Complete = false;
1164       Dequeue();
1165       return;
1166    }
1167
1168    Item::Failed(Message,Cnf);
1169 }
1170                                                                         /*}}}*/
1171 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
1172                                  string URI,string URIDesc,string ShortDesc,
1173                                  string SigFile,
1174                                  const vector<struct IndexTarget*>* IndexTargets,
1175                                  indexRecords* MetaIndexParser) :
1176    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
1177    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
1178 {
1179    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1180    DestFile += URItoFileName(URI);
1181
1182    // Create the item
1183    Desc.Description = URIDesc;
1184    Desc.Owner = this;
1185    Desc.ShortDesc = ShortDesc;
1186    Desc.URI = URI;
1187
1188    QueueURI(Desc);
1189 }
1190                                                                         /*}}}*/
1191 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
1192 // ---------------------------------------------------------------------
1193 /* The only header we use is the last-modified header. */
1194 string pkgAcqMetaIndex::Custom600Headers()
1195 {
1196    string Final = _config->FindDir("Dir::State::lists");
1197    Final += URItoFileName(RealURI);
1198
1199    struct stat Buf;
1200    if (stat(Final.c_str(),&Buf) != 0)
1201       return "\nIndex-File: true";
1202
1203    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1204 }
1205                                                                         /*}}}*/
1206 void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,string Hash,  /*{{{*/
1207                            pkgAcquire::MethodConfig *Cfg)
1208 {
1209    Item::Done(Message,Size,Hash,Cfg);
1210
1211    // MetaIndexes are done in two passes: one to download the
1212    // metaindex with an appropriate method, and a second to verify it
1213    // with the gpgv method
1214
1215    if (AuthPass == true)
1216    {
1217       AuthDone(Message);
1218
1219       // all cool, move Release file into place
1220       Complete = true;
1221    }
1222    else
1223    {
1224       RetrievalDone(Message);
1225       if (!Complete)
1226          // Still more retrieving to do
1227          return;
1228
1229       if (SigFile == "")
1230       {
1231          // There was no signature file, so we are finished.  Download
1232          // the indexes and do only hashsum verification if possible
1233          MetaIndexParser->Load(DestFile);
1234          QueueIndexes(false);
1235       }
1236       else
1237       {
1238          // FIXME: move this into pkgAcqMetaClearSig::Done on the next
1239          //        ABI break
1240
1241          // if we expect a ClearTextSignature (InRelase), ensure that
1242          // this is what we get and if not fail to queue a
1243          // Release/Release.gpg, see #346386
1244          if (SigFile == DestFile && !StartsWithGPGClearTextSignature(DestFile))
1245          {
1246             Failed(Message, Cfg);
1247             return;
1248          }
1249
1250          // There was a signature file, so pass it to gpgv for
1251          // verification
1252          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1253             std::cerr << "Metaindex acquired, queueing gpg verification ("
1254                       << SigFile << "," << DestFile << ")\n";
1255          AuthPass = true;
1256          Desc.URI = "gpgv:" + SigFile;
1257          QueueURI(Desc);
1258          Mode = "gpgv";
1259          return;
1260       }
1261    }
1262
1263    if (Complete == true)
1264    {
1265       string FinalFile = _config->FindDir("Dir::State::lists");
1266       FinalFile += URItoFileName(RealURI);
1267       if (SigFile == DestFile)
1268          SigFile = FinalFile;
1269       Rename(DestFile,FinalFile);
1270       chmod(FinalFile.c_str(),0644);
1271       DestFile = FinalFile;
1272    }
1273 }
1274                                                                         /*}}}*/
1275 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
1276 {
1277    // We have just finished downloading a Release file (it is not
1278    // verified yet)
1279
1280    string FileName = LookupTag(Message,"Filename");
1281    if (FileName.empty() == true)
1282    {
1283       Status = StatError;
1284       ErrorText = "Method gave a blank filename";
1285       return;
1286    }
1287
1288    if (FileName != DestFile)
1289    {
1290       Local = true;
1291       Desc.URI = "copy:" + FileName;
1292       QueueURI(Desc);
1293       return;
1294    }
1295
1296    // make sure to verify against the right file on I-M-S hit
1297    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1298    if(IMSHit)
1299    {
1300       string FinalFile = _config->FindDir("Dir::State::lists");
1301       FinalFile += URItoFileName(RealURI);
1302       if (SigFile == DestFile)
1303          SigFile = FinalFile;
1304       DestFile = FinalFile;
1305    }
1306    Complete = true;
1307 }
1308                                                                         /*}}}*/
1309 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1310 {
1311    // At this point, the gpgv method has succeeded, so there is a
1312    // valid signature from a key in the trusted keyring.  We
1313    // perform additional verification of its contents, and use them
1314    // to verify the indexes we are about to download
1315
1316    if (!MetaIndexParser->Load(DestFile))
1317    {
1318       Status = StatAuthError;
1319       ErrorText = MetaIndexParser->ErrorText;
1320       return;
1321    }
1322
1323    if (!VerifyVendor(Message))
1324    {
1325       return;
1326    }
1327
1328    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1329       std::cerr << "Signature verification succeeded: "
1330                 << DestFile << std::endl;
1331
1332    // Download further indexes with verification
1333    QueueIndexes(true);
1334
1335    // is it a clearsigned MetaIndex file?
1336    if (DestFile == SigFile)
1337       return;
1338
1339    // Done, move signature file into position
1340    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1341       URItoFileName(RealURI) + ".gpg";
1342    Rename(SigFile,VerifiedSigFile);
1343    chmod(VerifiedSigFile.c_str(),0644);
1344 }
1345                                                                         /*}}}*/
1346 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1347 {
1348 #if 0
1349    /* Reject invalid, existing Release files (LP: #346386) (Closes: #627642)
1350     * FIXME: Disabled; it breaks unsigned repositories without hashes */
1351    if (!verify && FileExists(DestFile) && !MetaIndexParser->Load(DestFile))
1352    {
1353       Status = StatError;
1354       ErrorText = MetaIndexParser->ErrorText;
1355       return;
1356    }
1357 #endif
1358    bool transInRelease = false;
1359    {
1360       std::vector<std::string> const keys = MetaIndexParser->MetaKeys();
1361       for (std::vector<std::string>::const_iterator k = keys.begin(); k != keys.end(); ++k)
1362          // FIXME: Feels wrong to check for hardcoded string here, but what should we do else…
1363          if (k->find("Translation-") != std::string::npos)
1364          {
1365             transInRelease = true;
1366             break;
1367          }
1368    }
1369
1370    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1371         Target != IndexTargets->end();
1372         ++Target)
1373    {
1374       HashString ExpectedIndexHash;
1375       const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1376       if (Record == NULL)
1377       {
1378          if (verify == true && (*Target)->IsOptional() == false)
1379          {
1380             Status = StatAuthError;
1381             strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), (*Target)->MetaKey.c_str());
1382             return;
1383          }
1384       }
1385       else
1386       {
1387          ExpectedIndexHash = Record->Hash;
1388          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1389          {
1390             std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1391             std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1392             std::cerr << "For: " << Record->MetaKeyFilename << std::endl;
1393          }
1394          if (verify == true && ExpectedIndexHash.empty() == true && (*Target)->IsOptional() == false)
1395          {
1396             Status = StatAuthError;
1397             strprintf(ErrorText, _("Unable to find hash sum for '%s' in Release file"), (*Target)->MetaKey.c_str());
1398             return;
1399          }
1400       }
1401
1402       if ((*Target)->IsOptional() == true)
1403       {
1404          if ((*Target)->IsSubIndex() == true)
1405             new pkgAcqSubIndex(Owner, (*Target)->URI, (*Target)->Description,
1406                                 (*Target)->ShortDesc, ExpectedIndexHash);
1407          else if (transInRelease == false || MetaIndexParser->Exists((*Target)->MetaKey) == true)
1408          {
1409             if (_config->FindB("Acquire::PDiffs",true) == true && transInRelease == true &&
1410                 MetaIndexParser->Exists(string((*Target)->MetaKey).append(".diff/Index")) == true)
1411                new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1412                                    (*Target)->ShortDesc, ExpectedIndexHash);
1413             else
1414                new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1415          }
1416          continue;
1417       }
1418
1419       /* Queue Packages file (either diff or full packages files, depending
1420          on the users option) - we also check if the PDiff Index file is listed
1421          in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
1422          instead, but passing the required info to it is to much hassle */
1423       if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
1424           MetaIndexParser->Exists(string((*Target)->MetaKey).append(".diff/Index")) == true))
1425          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1426                              (*Target)->ShortDesc, ExpectedIndexHash);
1427       else
1428          new pkgAcqIndex(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1429    }
1430 }
1431                                                                         /*}}}*/
1432 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1433 {
1434    string::size_type pos;
1435
1436    // check for missing sigs (that where not fatal because otherwise we had
1437    // bombed earlier)
1438    string missingkeys;
1439    string msg = _("There is no public key available for the "
1440                   "following key IDs:\n");
1441    pos = Message.find("NO_PUBKEY ");
1442    if (pos != std::string::npos)
1443    {
1444       string::size_type start = pos+strlen("NO_PUBKEY ");
1445       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1446       missingkeys += (Fingerprint);
1447    }
1448    if(!missingkeys.empty())
1449       _error->Warning("%s", string(msg+missingkeys).c_str());
1450
1451    string Transformed = MetaIndexParser->GetExpectedDist();
1452
1453    if (Transformed == "../project/experimental")
1454    {
1455       Transformed = "experimental";
1456    }
1457
1458    pos = Transformed.rfind('/');
1459    if (pos != string::npos)
1460    {
1461       Transformed = Transformed.substr(0, pos);
1462    }
1463
1464    if (Transformed == ".")
1465    {
1466       Transformed = "";
1467    }
1468
1469    if (_config->FindB("Acquire::Check-Valid-Until", true) == true &&
1470        MetaIndexParser->GetValidUntil() > 0) {
1471       time_t const invalid_since = time(NULL) - MetaIndexParser->GetValidUntil();
1472       if (invalid_since > 0)
1473          // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1474          // the time since then the file is invalid - formated in the same way as in
1475          // the download progress display (e.g. 7d 3h 42min 1s)
1476          return _error->Error(
1477             _("Release file for %s is expired (invalid since %s). "
1478               "Updates for this repository will not be applied."),
1479             RealURI.c_str(), TimeToStr(invalid_since).c_str());
1480    }
1481
1482    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1483    {
1484       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1485       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1486       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1487    }
1488
1489    if (MetaIndexParser->CheckDist(Transformed) == false)
1490    {
1491       // This might become fatal one day
1492 //       Status = StatAuthError;
1493 //       ErrorText = "Conflicting distribution; expected "
1494 //          + MetaIndexParser->GetExpectedDist() + " but got "
1495 //          + MetaIndexParser->GetDist();
1496 //       return false;
1497       if (!Transformed.empty())
1498       {
1499          _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1500                          Desc.Description.c_str(),
1501                          Transformed.c_str(),
1502                          MetaIndexParser->GetDist().c_str());
1503       }
1504    }
1505
1506    return true;
1507 }
1508                                                                         /*}}}*/
1509 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1510 // ---------------------------------------------------------------------
1511 /* */
1512 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1513 {
1514    if (AuthPass == true)
1515    {
1516       // gpgv method failed, if we have a good signature
1517       string LastGoodSigFile = _config->FindDir("Dir::State::lists").append("partial/").append(URItoFileName(RealURI));
1518       if (DestFile != SigFile)
1519          LastGoodSigFile.append(".gpg");
1520       LastGoodSigFile.append(".reverify");
1521
1522       if(FileExists(LastGoodSigFile))
1523       {
1524          string VerifiedSigFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1525          if (DestFile != SigFile)
1526             VerifiedSigFile.append(".gpg");
1527          Rename(LastGoodSigFile, VerifiedSigFile);
1528          Status = StatTransientNetworkError;
1529          _error->Warning(_("A error occurred during the signature "
1530                            "verification. The repository is not updated "
1531                            "and the previous index files will be used. "
1532                            "GPG error: %s: %s\n"),
1533                          Desc.Description.c_str(),
1534                          LookupTag(Message,"Message").c_str());
1535          RunScripts("APT::Update::Auth-Failure");
1536          return;
1537       } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) {
1538          /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */
1539          _error->Error(_("GPG error: %s: %s"),
1540                          Desc.Description.c_str(),
1541                          LookupTag(Message,"Message").c_str());
1542          return;
1543       } else {
1544          _error->Warning(_("GPG error: %s: %s"),
1545                          Desc.Description.c_str(),
1546                          LookupTag(Message,"Message").c_str());
1547       }
1548       // gpgv method failed
1549       ReportMirrorFailure("GPGFailure");
1550    }
1551
1552    /* Always move the meta index, even if gpgv failed. This ensures
1553     * that PackageFile objects are correctly filled in */
1554    if (FileExists(DestFile)) {
1555       string FinalFile = _config->FindDir("Dir::State::lists");
1556       FinalFile += URItoFileName(RealURI);
1557       /* InRelease files become Release files, otherwise
1558        * they would be considered as trusted later on */
1559       if (SigFile == DestFile) {
1560          RealURI = RealURI.replace(RealURI.rfind("InRelease"), 9,
1561                                        "Release");
1562          FinalFile = FinalFile.replace(FinalFile.rfind("InRelease"), 9,
1563                                        "Release");
1564          SigFile = FinalFile;
1565       }
1566       Rename(DestFile,FinalFile);
1567       chmod(FinalFile.c_str(),0644);
1568
1569       DestFile = FinalFile;
1570    }
1571
1572    // No Release file was present, or verification failed, so fall
1573    // back to queueing Packages files without verification
1574    QueueIndexes(false);
1575 }
1576                                                                         /*}}}*/
1577 pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner,               /*{{{*/
1578                 string const &URI, string const &URIDesc, string const &ShortDesc,
1579                 string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc,
1580                 string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc,
1581                 const vector<struct IndexTarget*>* IndexTargets,
1582                 indexRecords* MetaIndexParser) :
1583         pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, "", IndexTargets, MetaIndexParser),
1584         MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1585         MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc)
1586 {
1587    SigFile = DestFile;
1588
1589    // keep the old InRelease around in case of transistent network errors
1590    string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1591    struct stat Buf;
1592    if (stat(Final.c_str(),&Buf) == 0)
1593    {
1594       string const LastGoodSig = DestFile + ".reverify";
1595       Rename(Final,LastGoodSig);
1596    }
1597 }
1598                                                                         /*}}}*/
1599 // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/
1600 // ---------------------------------------------------------------------
1601 // FIXME: this can go away once the InRelease file is used widely
1602 string pkgAcqMetaClearSig::Custom600Headers()
1603 {
1604    string Final = _config->FindDir("Dir::State::lists");
1605    Final += URItoFileName(RealURI);
1606
1607    struct stat Buf;
1608    if (stat(Final.c_str(),&Buf) != 0)
1609       return "\nIndex-File: true\nFail-Ignore: true\n";
1610
1611    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1612 }
1613                                                                         /*}}}*/
1614 void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/
1615 {
1616    if (AuthPass == false)
1617    {
1618       // Remove the 'old' InRelease file if we try Release.gpg now as otherwise
1619       // the file will stay around and gives a false-auth impression (CVE-2012-0214)
1620       string FinalFile = _config->FindDir("Dir::State::lists");
1621       FinalFile.append(URItoFileName(RealURI));
1622       if (FileExists(FinalFile))
1623          unlink(FinalFile.c_str());
1624
1625       new pkgAcqMetaSig(Owner,
1626                         MetaSigURI, MetaSigURIDesc, MetaSigShortDesc,
1627                         MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1628                         IndexTargets, MetaIndexParser);
1629       if (Cnf->LocalOnly == true ||
1630           StringToBool(LookupTag(Message, "Transient-Failure"), false) == false)
1631          Dequeue();
1632    }
1633    else
1634       pkgAcqMetaIndex::Failed(Message, Cnf);
1635 }
1636                                                                         /*}}}*/
1637 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1638 // ---------------------------------------------------------------------
1639 /* This just sets up the initial fetch environment and queues the first
1640    possibilitiy */
1641 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1642                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1643                              string &StoreFilename) :
1644                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1645                StoreFilename(StoreFilename), Vf(Version.FileList()),
1646                Trusted(false)
1647 {
1648    Retries = _config->FindI("Acquire::Retries",0);
1649
1650    if (Version.Arch() == 0)
1651    {
1652       _error->Error(_("I wasn't able to locate a file for the %s package. "
1653                       "This might mean you need to manually fix this package. "
1654                       "(due to missing arch)"),
1655                     Version.ParentPkg().Name());
1656       return;
1657    }
1658
1659    /* We need to find a filename to determine the extension. We make the
1660       assumption here that all the available sources for this version share
1661       the same extension.. */
1662    // Skip not source sources, they do not have file fields.
1663    for (; Vf.end() == false; ++Vf)
1664    {
1665       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1666          continue;
1667       break;
1668    }
1669
1670    // Does not really matter here.. we are going to fail out below
1671    if (Vf.end() != true)
1672    {
1673       // If this fails to get a file name we will bomb out below.
1674       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1675       if (_error->PendingError() == true)
1676          return;
1677
1678       // Generate the final file name as: package_version_arch.foo
1679       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1680                       QuoteString(Version.VerStr(),"_:") + '_' +
1681                       QuoteString(Version.Arch(),"_:.") +
1682                       "." + flExtension(Parse.FileName());
1683    }
1684
1685    // check if we have one trusted source for the package. if so, switch
1686    // to "TrustedOnly" mode
1687    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; ++i)
1688    {
1689       pkgIndexFile *Index;
1690       if (Sources->FindIndex(i.File(),Index) == false)
1691          continue;
1692       if (_config->FindB("Debug::pkgAcquire::Auth", false))
1693       {
1694          std::cerr << "Checking index: " << Index->Describe()
1695                    << "(Trusted=" << Index->IsTrusted() << ")\n";
1696       }
1697       if (Index->IsTrusted()) {
1698          Trusted = true;
1699          break;
1700       }
1701    }
1702
1703    // "allow-unauthenticated" restores apts old fetching behaviour
1704    // that means that e.g. unauthenticated file:// uris are higher
1705    // priority than authenticated http:// uris
1706    if (_config->FindB("APT::Get::AllowUnauthenticated",false) == true)
1707       Trusted = false;
1708
1709    // Select a source
1710    if (QueueNext() == false && _error->PendingError() == false)
1711       _error->Error(_("I wasn't able to locate a file for the %s package. "
1712                     "This might mean you need to manually fix this package."),
1713                     Version.ParentPkg().Name());
1714 }
1715                                                                         /*}}}*/
1716 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1717 // ---------------------------------------------------------------------
1718 /* This queues the next available file version for download. It checks if
1719    the archive is already available in the cache and stashs the MD5 for
1720    checking later. */
1721 bool pkgAcqArchive::QueueNext()
1722 {
1723    string const ForceHash = _config->Find("Acquire::ForceHash");
1724    for (; Vf.end() == false; ++Vf)
1725    {
1726       // Ignore not source sources
1727       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1728          continue;
1729
1730       // Try to cross match against the source list
1731       pkgIndexFile *Index;
1732       if (Sources->FindIndex(Vf.File(),Index) == false)
1733             continue;
1734
1735       // only try to get a trusted package from another source if that source
1736       // is also trusted
1737       if(Trusted && !Index->IsTrusted())
1738          continue;
1739
1740       // Grab the text package record
1741       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1742       if (_error->PendingError() == true)
1743          return false;
1744
1745       string PkgFile = Parse.FileName();
1746       if (ForceHash.empty() == false)
1747       {
1748          if(stringcasecmp(ForceHash, "sha512") == 0)
1749             ExpectedHash = HashString("SHA512", Parse.SHA512Hash());
1750          else if(stringcasecmp(ForceHash, "sha256") == 0)
1751             ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
1752          else if (stringcasecmp(ForceHash, "sha1") == 0)
1753             ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
1754          else
1755             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1756       }
1757       else
1758       {
1759          string Hash;
1760          if ((Hash = Parse.SHA512Hash()).empty() == false)
1761             ExpectedHash = HashString("SHA512", Hash);
1762          else if ((Hash = Parse.SHA256Hash()).empty() == false)
1763             ExpectedHash = HashString("SHA256", Hash);
1764          else if ((Hash = Parse.SHA1Hash()).empty() == false)
1765             ExpectedHash = HashString("SHA1", Hash);
1766          else
1767             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1768       }
1769       if (PkgFile.empty() == true)
1770          return _error->Error(_("The package index files are corrupted. No Filename: "
1771                               "field for package %s."),
1772                               Version.ParentPkg().Name());
1773
1774       Desc.URI = Index->ArchiveURI(PkgFile);
1775       Desc.Description = Index->ArchiveInfo(Version);
1776       Desc.Owner = this;
1777       Desc.ShortDesc = Version.ParentPkg().Name();
1778
1779       // See if we already have the file. (Legacy filenames)
1780       FileSize = Version->Size;
1781       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
1782       struct stat Buf;
1783       if (stat(FinalFile.c_str(),&Buf) == 0)
1784       {
1785          // Make sure the size matches
1786          if ((unsigned long long)Buf.st_size == Version->Size)
1787          {
1788             Complete = true;
1789             Local = true;
1790             Status = StatDone;
1791             StoreFilename = DestFile = FinalFile;
1792             return true;
1793          }
1794
1795          /* Hmm, we have a file and its size does not match, this means it is
1796             an old style mismatched arch */
1797          unlink(FinalFile.c_str());
1798       }
1799
1800       // Check it again using the new style output filenames
1801       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
1802       if (stat(FinalFile.c_str(),&Buf) == 0)
1803       {
1804          // Make sure the size matches
1805          if ((unsigned long long)Buf.st_size == Version->Size)
1806          {
1807             Complete = true;
1808             Local = true;
1809             Status = StatDone;
1810             StoreFilename = DestFile = FinalFile;
1811             return true;
1812          }
1813
1814          /* Hmm, we have a file and its size does not match, this shouldnt
1815             happen.. */
1816          unlink(FinalFile.c_str());
1817       }
1818
1819       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
1820
1821       // Check the destination file
1822       if (stat(DestFile.c_str(),&Buf) == 0)
1823       {
1824          // Hmm, the partial file is too big, erase it
1825          if ((unsigned long long)Buf.st_size > Version->Size)
1826             unlink(DestFile.c_str());
1827          else
1828             PartialSize = Buf.st_size;
1829       }
1830
1831       // Disables download of archives - useful if no real installation follows,
1832       // e.g. if we are just interested in proposed installation order
1833       if (_config->FindB("Debug::pkgAcqArchive::NoQueue", false) == true)
1834       {
1835          Complete = true;
1836          Local = true;
1837          Status = StatDone;
1838          StoreFilename = DestFile = FinalFile;
1839          return true;
1840       }
1841
1842       // Create the item
1843       Local = false;
1844       Desc.URI = Index->ArchiveURI(PkgFile);
1845       Desc.Description = Index->ArchiveInfo(Version);
1846       Desc.Owner = this;
1847       Desc.ShortDesc = Version.ParentPkg().Name();
1848       QueueURI(Desc);
1849
1850       ++Vf;
1851       return true;
1852    }
1853    return false;
1854 }
1855                                                                         /*}}}*/
1856 // AcqArchive::Done - Finished fetching                                 /*{{{*/
1857 // ---------------------------------------------------------------------
1858 /* */
1859 void pkgAcqArchive::Done(string Message,unsigned long long Size,string CalcHash,
1860                          pkgAcquire::MethodConfig *Cfg)
1861 {
1862    Item::Done(Message,Size,CalcHash,Cfg);
1863
1864    // Check the size
1865    if (Size != Version->Size)
1866    {
1867       Status = StatError;
1868       ErrorText = _("Size mismatch");
1869       return;
1870    }
1871
1872    // Check the hash
1873    if(ExpectedHash.toStr() != CalcHash)
1874    {
1875       Status = StatError;
1876       ErrorText = _("Hash Sum mismatch");
1877       if(FileExists(DestFile))
1878          Rename(DestFile,DestFile + ".FAILED");
1879       return;
1880    }
1881
1882    // Grab the output filename
1883    string FileName = LookupTag(Message,"Filename");
1884    if (FileName.empty() == true)
1885    {
1886       Status = StatError;
1887       ErrorText = "Method gave a blank filename";
1888       return;
1889    }
1890
1891    Complete = true;
1892
1893    // Reference filename
1894    if (FileName != DestFile)
1895    {
1896       StoreFilename = DestFile = FileName;
1897       Local = true;
1898       return;
1899    }
1900
1901    // Done, move it into position
1902    string FinalFile = _config->FindDir("Dir::Cache::Archives");
1903    FinalFile += flNotDir(StoreFilename);
1904    Rename(DestFile,FinalFile);
1905
1906    StoreFilename = DestFile = FinalFile;
1907    Complete = true;
1908 }
1909                                                                         /*}}}*/
1910 // AcqArchive::Failed - Failure handler                                 /*{{{*/
1911 // ---------------------------------------------------------------------
1912 /* Here we try other sources */
1913 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1914 {
1915    ErrorText = LookupTag(Message,"Message");
1916
1917    /* We don't really want to retry on failed media swaps, this prevents
1918       that. An interesting observation is that permanent failures are not
1919       recorded. */
1920    if (Cnf->Removable == true &&
1921        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1922    {
1923       // Vf = Version.FileList();
1924       while (Vf.end() == false) ++Vf;
1925       StoreFilename = string();
1926       Item::Failed(Message,Cnf);
1927       return;
1928    }
1929
1930    if (QueueNext() == false)
1931    {
1932       // This is the retry counter
1933       if (Retries != 0 &&
1934           Cnf->LocalOnly == false &&
1935           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1936       {
1937          Retries--;
1938          Vf = Version.FileList();
1939          if (QueueNext() == true)
1940             return;
1941       }
1942
1943       StoreFilename = string();
1944       Item::Failed(Message,Cnf);
1945    }
1946 }
1947                                                                         /*}}}*/
1948 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
1949 // ---------------------------------------------------------------------
1950 bool pkgAcqArchive::IsTrusted()
1951 {
1952    return Trusted;
1953 }
1954                                                                         /*}}}*/
1955 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
1956 // ---------------------------------------------------------------------
1957 /* */
1958 void pkgAcqArchive::Finished()
1959 {
1960    if (Status == pkgAcquire::Item::StatDone &&
1961        Complete == true)
1962       return;
1963    StoreFilename = string();
1964 }
1965                                                                         /*}}}*/
1966 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
1967 // ---------------------------------------------------------------------
1968 /* The file is added to the queue */
1969 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
1970                        unsigned long long Size,string Dsc,string ShortDesc,
1971                        const string &DestDir, const string &DestFilename,
1972                        bool IsIndexFile) :
1973                        Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile)
1974 {
1975    Retries = _config->FindI("Acquire::Retries",0);
1976
1977    if(!DestFilename.empty())
1978       DestFile = DestFilename;
1979    else if(!DestDir.empty())
1980       DestFile = DestDir + "/" + flNotDir(URI);
1981    else
1982       DestFile = flNotDir(URI);
1983
1984    // Create the item
1985    Desc.URI = URI;
1986    Desc.Description = Dsc;
1987    Desc.Owner = this;
1988
1989    // Set the short description to the archive component
1990    Desc.ShortDesc = ShortDesc;
1991
1992    // Get the transfer sizes
1993    FileSize = Size;
1994    struct stat Buf;
1995    if (stat(DestFile.c_str(),&Buf) == 0)
1996    {
1997       // Hmm, the partial file is too big, erase it
1998       if ((unsigned long long)Buf.st_size > Size)
1999          unlink(DestFile.c_str());
2000       else
2001          PartialSize = Buf.st_size;
2002    }
2003
2004    QueueURI(Desc);
2005 }
2006                                                                         /*}}}*/
2007 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
2008 // ---------------------------------------------------------------------
2009 /* */
2010 void pkgAcqFile::Done(string Message,unsigned long long Size,string CalcHash,
2011                       pkgAcquire::MethodConfig *Cnf)
2012 {
2013    Item::Done(Message,Size,CalcHash,Cnf);
2014
2015    // Check the hash
2016    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
2017    {
2018       Status = StatError;
2019       ErrorText = _("Hash Sum mismatch");
2020       Rename(DestFile,DestFile + ".FAILED");
2021       return;
2022    }
2023
2024    string FileName = LookupTag(Message,"Filename");
2025    if (FileName.empty() == true)
2026    {
2027       Status = StatError;
2028       ErrorText = "Method gave a blank filename";
2029       return;
2030    }
2031
2032    Complete = true;
2033
2034    // The files timestamp matches
2035    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
2036       return;
2037
2038    // We have to copy it into place
2039    if (FileName != DestFile)
2040    {
2041       Local = true;
2042       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
2043           Cnf->Removable == true)
2044       {
2045          Desc.URI = "copy:" + FileName;
2046          QueueURI(Desc);
2047          return;
2048       }
2049
2050       // Erase the file if it is a symlink so we can overwrite it
2051       struct stat St;
2052       if (lstat(DestFile.c_str(),&St) == 0)
2053       {
2054          if (S_ISLNK(St.st_mode) != 0)
2055             unlink(DestFile.c_str());
2056       }
2057
2058       // Symlink the file
2059       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
2060       {
2061          ErrorText = "Link to " + DestFile + " failure ";
2062          Status = StatError;
2063          Complete = false;
2064       }
2065    }
2066 }
2067                                                                         /*}}}*/
2068 // AcqFile::Failed - Failure handler                                    /*{{{*/
2069 // ---------------------------------------------------------------------
2070 /* Here we try other sources */
2071 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2072 {
2073    ErrorText = LookupTag(Message,"Message");
2074
2075    // This is the retry counter
2076    if (Retries != 0 &&
2077        Cnf->LocalOnly == false &&
2078        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2079    {
2080       Retries--;
2081       QueueURI(Desc);
2082       return;
2083    }
2084
2085    Item::Failed(Message,Cnf);
2086 }
2087                                                                         /*}}}*/
2088 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
2089 // ---------------------------------------------------------------------
2090 /* The only header we use is the last-modified header. */
2091 string pkgAcqFile::Custom600Headers()
2092 {
2093    if (IsIndexFile)
2094       return "\nIndex-File: true";
2095    return "";
2096 }
2097                                                                         /*}}}*/