apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <config.h>
  17
  18 #include <apt-pkg/acquire-item.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/aptconfiguration.h>
  21 #include <apt-pkg/sourcelist.h>
  22 #include <apt-pkg/error.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/md5.h>
  26 #include <apt-pkg/sha1.h>
  27 #include <apt-pkg/tagfile.h>
  28 #include <apt-pkg/indexrecords.h>
  29 #include <apt-pkg/metaindex.h>
  30
  31 #include <sys/stat.h>
  32 #include <unistd.h>
  33 #include <errno.h>
  34 #include <string>
  35 #include <sstream>
  36 #include <stdio.h>
  37 #include <ctime>
  38
  39 #include <apti18n.h>
  40                                                                         /*}}}*/
  41
  42 using namespace std;
  43
  44 // Acquire::Item::Item - Constructor                                    /*{{{*/
  45 // ---------------------------------------------------------------------
  46 /* */
  47 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  48                        PartialSize(0), Mode(0), ID(0), Complete(false),
  49                        Local(false), QueueCounter(0)
  50 {
  51    Owner->Add(this);
  52    Status = StatIdle;
  53 }
  54                                                                         /*}}}*/
  55 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  56 // ---------------------------------------------------------------------
  57 /* */
  58 pkgAcquire::Item::~Item()
  59 {
  60    Owner->Remove(this);
  61 }
  62                                                                         /*}}}*/
  63 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  64 // ---------------------------------------------------------------------
  65 /* We return to an idle state if there are still other queues that could
  66    fetch this object */
  67 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  68 {
  69    Status = StatIdle;
  70    ErrorText = LookupTag(Message,"Message");
  71    UsedMirror =  LookupTag(Message,"UsedMirror");
  72    if (QueueCounter <= 1)
  73    {
  74       /* This indicates that the file is not available right now but might
  75          be sometime later. If we do a retry cycle then this should be
  76          retried [CDROMs] */
  77       if (Cnf->LocalOnly == true &&
  78           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  79       {
  80          Status = StatIdle;
  81          Dequeue();
  82          return;
  83       }
  84
  85       Status = StatError;
  86       Dequeue();
  87    }
  88
  89    // report mirror failure back to LP if we actually use a mirror
  90    string FailReason = LookupTag(Message, "FailReason");
  91    if(FailReason.size() != 0)
  92       ReportMirrorFailure(FailReason);
  93    else
  94       ReportMirrorFailure(ErrorText);
  95 }
  96                                                                         /*}}}*/
  97 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
  98 // ---------------------------------------------------------------------
  99 /* Stash status and the file size. Note that setting Complete means
 100    sub-phases of the acquire process such as decompresion are operating */
 101 void pkgAcquire::Item::Start(string /*Message*/,unsigned long long Size)
 102 {
 103    Status = StatFetching;
 104    if (FileSize == 0 && Complete == false)
 105       FileSize = Size;
 106 }
 107                                                                         /*}}}*/
 108 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
 109 // ---------------------------------------------------------------------
 110 /* */
 111 void pkgAcquire::Item::Done(string Message,unsigned long long Size,string Hash,
 112                             pkgAcquire::MethodConfig *Cnf)
 113 {
 114    // We just downloaded something..
 115    string FileName = LookupTag(Message,"Filename");
 116    UsedMirror =  LookupTag(Message,"UsedMirror");
 117    if (Complete == false && !Local && FileName == DestFile)
 118    {
 119       if (Owner->Log != 0)
 120          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 121    }
 122
 123    if (FileSize == 0)
 124       FileSize= Size;
 125    Status = StatDone;
 126    ErrorText = string();
 127    Owner->Dequeue(this);
 128 }
 129                                                                         /*}}}*/
 130 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 131 // ---------------------------------------------------------------------
 132 /* This helper function is used by alot of item methods as thier final
 133    step */
 134 void pkgAcquire::Item::Rename(string From,string To)
 135 {
 136    if (rename(From.c_str(),To.c_str()) != 0)
 137    {
 138       char S[300];
 139       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 140               From.c_str(),To.c_str());
 141       Status = StatError;
 142       ErrorText = S;
 143    }
 144 }
 145                                                                         /*}}}*/
 146 bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const error)/*{{{*/
 147 {
 148    if(FileExists(DestFile))
 149       Rename(DestFile, DestFile + ".FAILED");
 150
 151    switch (error)
 152    {
 153       case HashSumMismatch:
 154          ErrorText = _("Hash Sum mismatch");
 155          Status = StatAuthError;
 156          ReportMirrorFailure("HashChecksumFailure");
 157          break;
 158       case SizeMismatch:
 159          ErrorText = _("Size mismatch");
 160          Status = StatAuthError;
 161          ReportMirrorFailure("SizeFailure");
 162          break;
 163       case InvalidFormat:
 164          ErrorText = _("Invalid file format");
 165          Status = StatError;
 166          // do not report as usually its not the mirrors fault, but Portal/Proxy
 167          break;
 168    }
 169    return false;
 170 }
 171                                                                         /*}}}*/
 172 // Acquire::Item::ReportMirrorFailure                                   /*{{{*/
 173 // ---------------------------------------------------------------------
 174 void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
 175 {
 176    // we only act if a mirror was used at all
 177    if(UsedMirror.empty())
 178       return;
 179 #if 0
 180    std::cerr << "\nReportMirrorFailure: "
 181              << UsedMirror
 182              << " Uri: " << DescURI()
 183              << " FailCode: "
 184              << FailCode << std::endl;
 185 #endif
 186    const char *Args[40];
 187    unsigned int i = 0;
 188    string report = _config->Find("Methods::Mirror::ProblemReporting",
 189                                  "/usr/lib/apt/apt-report-mirror-failure");
 190    if(!FileExists(report))
 191       return;
 192    Args[i++] = report.c_str();
 193    Args[i++] = UsedMirror.c_str();
 194    Args[i++] = DescURI().c_str();
 195    Args[i++] = FailCode.c_str();
 196    Args[i++] = NULL;
 197    pid_t pid = ExecFork();
 198    if(pid < 0)
 199    {
 200       _error->Error("ReportMirrorFailure Fork failed");
 201       return;
 202    }
 203    else if(pid == 0)
 204    {
 205       execvp(Args[0], (char**)Args);
 206       std::cerr << "Could not exec " << Args[0] << std::endl;
 207       _exit(100);
 208    }
 209    if(!ExecWait(pid, "report-mirror-failure"))
 210    {
 211       _error->Warning("Couldn't report problem to '%s'",
 212                       _config->Find("Methods::Mirror::ProblemReporting").c_str());
 213    }
 214 }
 215                                                                         /*}}}*/
 216 // AcqSubIndex::AcqSubIndex - Constructor                               /*{{{*/
 217 // ---------------------------------------------------------------------
 218 /* Get a sub-index file based on checksums from a 'master' file and
 219    possibly query additional files */
 220 pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, string const &URI,
 221                                  string const &URIDesc, string const &ShortDesc,
 222                                  HashString const &ExpectedHash)
 223    : Item(Owner), ExpectedHash(ExpectedHash)
 224 {
 225    /* XXX: Beware: Currently this class does nothing (of value) anymore ! */
 226    Debug = _config->FindB("Debug::pkgAcquire::SubIndex",false);
 227
 228    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 229    DestFile += URItoFileName(URI);
 230
 231    Desc.URI = URI;
 232    Desc.Description = URIDesc;
 233    Desc.Owner = this;
 234    Desc.ShortDesc = ShortDesc;
 235
 236    QueueURI(Desc);
 237
 238    if(Debug)
 239       std::clog << "pkgAcqSubIndex: " << Desc.URI << std::endl;
 240 }
 241                                                                         /*}}}*/
 242 // AcqSubIndex::Custom600Headers - Insert custom request headers        /*{{{*/
 243 // ---------------------------------------------------------------------
 244 /* The only header we use is the last-modified header. */
 245 string pkgAcqSubIndex::Custom600Headers()
 246 {
 247    string Final = _config->FindDir("Dir::State::lists");
 248    Final += URItoFileName(Desc.URI);
 249
 250    struct stat Buf;
 251    if (stat(Final.c_str(),&Buf) != 0)
 252       return "\nIndex-File: true\nFail-Ignore: true\n";
 253    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 254 }
 255                                                                         /*}}}*/
 256 void pkgAcqSubIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)       /*{{{*/
 257 {
 258    if(Debug)
 259       std::clog << "pkgAcqSubIndex failed: " << Desc.URI << std::endl;
 260
 261    Complete = false;
 262    Status = StatDone;
 263    Dequeue();
 264
 265    // No good Index is provided
 266 }
 267                                                                         /*}}}*/
 268 void pkgAcqSubIndex::Done(string Message,unsigned long long Size,string Md5Hash,        /*{{{*/
 269                            pkgAcquire::MethodConfig *Cnf)
 270 {
 271    if(Debug)
 272       std::clog << "pkgAcqSubIndex::Done(): " << Desc.URI << std::endl;
 273
 274    string FileName = LookupTag(Message,"Filename");
 275    if (FileName.empty() == true)
 276    {
 277       Status = StatError;
 278       ErrorText = "Method gave a blank filename";
 279       return;
 280    }
 281
 282    if (FileName != DestFile)
 283    {
 284       Local = true;
 285       Desc.URI = "copy:" + FileName;
 286       QueueURI(Desc);
 287       return;
 288    }
 289
 290    Item::Done(Message,Size,Md5Hash,Cnf);
 291
 292    string FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(Desc.URI);
 293
 294    /* Downloaded invalid transindex => Error (LP: #346386) (Closes: #627642) */
 295    indexRecords SubIndexParser;
 296    if (FileExists(DestFile) == true && !SubIndexParser.Load(DestFile)) {
 297       Status = StatError;
 298       ErrorText = SubIndexParser.ErrorText;
 299       return;
 300    }
 301
 302    // sucess in downloading the index
 303    // rename the index
 304    if(Debug)
 305       std::clog << "Renaming: " << DestFile << " -> " << FinalFile << std::endl;
 306    Rename(DestFile,FinalFile);
 307    chmod(FinalFile.c_str(),0644);
 308    DestFile = FinalFile;
 309
 310    if(ParseIndex(DestFile) == false)
 311       return Failed("", NULL);
 312
 313    Complete = true;
 314    Status = StatDone;
 315    Dequeue();
 316    return;
 317 }
 318                                                                         /*}}}*/
 319 bool pkgAcqSubIndex::ParseIndex(string const &IndexFile)                /*{{{*/
 320 {
 321    indexRecords SubIndexParser;
 322    if (FileExists(IndexFile) == false || SubIndexParser.Load(IndexFile) == false)
 323       return false;
 324    // so something with the downloaded index
 325    return true;
 326 }
 327                                                                         /*}}}*/
 328 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 329 // ---------------------------------------------------------------------
 330 /* Get the DiffIndex file first and see if there are patches availabe
 331  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 332  * patches. If anything goes wrong in that process, it will fall back to
 333  * the original packages file
 334  */
 335 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 336                                  string URI,string URIDesc,string ShortDesc,
 337                                  HashString ExpectedHash)
 338    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 339      Description(URIDesc)
 340 {
 341
 342    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 343
 344    Desc.Description = URIDesc + "/DiffIndex";
 345    Desc.Owner = this;
 346    Desc.ShortDesc = ShortDesc;
 347    Desc.URI = URI + ".diff/Index";
 348
 349    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 350    DestFile += URItoFileName(URI) + string(".DiffIndex");
 351
 352    if(Debug)
 353       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 354
 355    // look for the current package file
 356    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 357    CurrentPackagesFile += URItoFileName(RealURI);
 358
 359    // FIXME: this file:/ check is a hack to prevent fetching
 360    //        from local sources. this is really silly, and
 361    //        should be fixed cleanly as soon as possible
 362    if(!FileExists(CurrentPackagesFile) ||
 363       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 364    {
 365       // we don't have a pkg file or we don't want to queue
 366       if(Debug)
 367          std::clog << "No index file, local or canceld by user" << std::endl;
 368       Failed("", NULL);
 369       return;
 370    }
 371
 372    if(Debug)
 373       std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): "
 374                 << CurrentPackagesFile << std::endl;
 375
 376    QueueURI(Desc);
 377
 378 }
 379                                                                         /*}}}*/
 380 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 381 // ---------------------------------------------------------------------
 382 /* The only header we use is the last-modified header. */
 383 string pkgAcqDiffIndex::Custom600Headers()
 384 {
 385    string Final = _config->FindDir("Dir::State::lists");
 386    Final += URItoFileName(RealURI) + string(".IndexDiff");
 387
 388    if(Debug)
 389       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 390
 391    struct stat Buf;
 392    if (stat(Final.c_str(),&Buf) != 0)
 393       return "\nIndex-File: true";
 394
 395    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 396 }
 397                                                                         /*}}}*/
 398 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 399 {
 400    if(Debug)
 401       std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile
 402                 << std::endl;
 403
 404    pkgTagSection Tags;
 405    string ServerSha1;
 406    vector<DiffInfo> available_patches;
 407
 408    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 409    pkgTagFile TF(&Fd);
 410    if (_error->PendingError() == true)
 411       return false;
 412
 413    if(TF.Step(Tags) == true)
 414    {
 415       bool found = false;
 416       DiffInfo d;
 417       string size;
 418
 419       string const tmp = Tags.FindS("SHA1-Current");
 420       std::stringstream ss(tmp);
 421       ss >> ServerSha1 >> size;
 422       unsigned long const ServerSize = atol(size.c_str());
 423
 424       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 425       SHA1Summation SHA1;
 426       SHA1.AddFD(fd);
 427       string const local_sha1 = SHA1.Result();
 428
 429       if(local_sha1 == ServerSha1)
 430       {
 431          // we have the same sha1 as the server so we are done here
 432          if(Debug)
 433             std::clog << "Package file is up-to-date" << std::endl;
 434          // list cleanup needs to know that this file as well as the already
 435          // present index is ours, so we create an empty diff to save it for us
 436          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 437                ExpectedHash, ServerSha1, available_patches);
 438          return true;
 439       }
 440       else
 441       {
 442          if(Debug)
 443             std::clog << "SHA1-Current: " << ServerSha1 << " and we start at "<< fd.Name() << " " << fd.Size() << " " << local_sha1 << std::endl;
 444
 445          // check the historie and see what patches we need
 446          string const history = Tags.FindS("SHA1-History");
 447          std::stringstream hist(history);
 448          while(hist >> d.sha1 >> size >> d.file)
 449          {
 450             // read until the first match is found
 451             // from that point on, we probably need all diffs
 452             if(d.sha1 == local_sha1)
 453                found=true;
 454             else if (found == false)
 455                continue;
 456
 457             if(Debug)
 458                std::clog << "Need to get diff: " << d.file << std::endl;
 459             available_patches.push_back(d);
 460          }
 461
 462          if (available_patches.empty() == false)
 463          {
 464             // patching with too many files is rather slow compared to a fast download
 465             unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 20);
 466             if (fileLimit != 0 && fileLimit < available_patches.size())
 467             {
 468                if (Debug)
 469                   std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 470                         << ") so fallback to complete download" << std::endl;
 471                return false;
 472             }
 473
 474             // see if the patches are too big
 475             found = false; // it was true and it will be true again at the end
 476             d = *available_patches.begin();
 477             string const firstPatch = d.file;
 478             unsigned long patchesSize = 0;
 479             std::stringstream patches(Tags.FindS("SHA1-Patches"));
 480             while(patches >> d.sha1 >> size >> d.file)
 481             {
 482                if (firstPatch == d.file)
 483                   found = true;
 484                else if (found == false)
 485                   continue;
 486
 487                patchesSize += atol(size.c_str());
 488             }
 489             unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 490             if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 491             {
 492                if (Debug)
 493                   std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 494                         << ") so fallback to complete download" << std::endl;
 495                return false;
 496             }
 497          }
 498       }
 499
 500       // we have something, queue the next diff
 501       if(found)
 502       {
 503          // queue the diffs
 504          string::size_type const last_space = Description.rfind(" ");
 505          if(last_space != string::npos)
 506             Description.erase(last_space, Description.size()-last_space);
 507          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 508                               ExpectedHash, ServerSha1, available_patches);
 509          Complete = false;
 510          Status = StatDone;
 511          Dequeue();
 512          return true;
 513       }
 514    }
 515
 516    // Nothing found, report and return false
 517    // Failing here is ok, if we return false later, the full
 518    // IndexFile is queued
 519    if(Debug)
 520       std::clog << "Can't find a patch in the index file" << std::endl;
 521    return false;
 522 }
 523                                                                         /*}}}*/
 524 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)      /*{{{*/
 525 {
 526    if(Debug)
 527       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << std::endl
 528                 << "Falling back to normal index file aquire" << std::endl;
 529
 530    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 531                    ExpectedHash);
 532
 533    Complete = false;
 534    Status = StatDone;
 535    Dequeue();
 536 }
 537                                                                         /*}}}*/
 538 void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,string Md5Hash,       /*{{{*/
 539                            pkgAcquire::MethodConfig *Cnf)
 540 {
 541    if(Debug)
 542       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 543
 544    Item::Done(Message,Size,Md5Hash,Cnf);
 545
 546    string FinalFile;
 547    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 548
 549    // sucess in downloading the index
 550    // rename the index
 551    FinalFile += string(".IndexDiff");
 552    if(Debug)
 553       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 554                 << std::endl;
 555    Rename(DestFile,FinalFile);
 556    chmod(FinalFile.c_str(),0644);
 557    DestFile = FinalFile;
 558
 559    if(!ParseDiffIndex(DestFile))
 560       return Failed("", NULL);
 561
 562    Complete = true;
 563    Status = StatDone;
 564    Dequeue();
 565    return;
 566 }
 567                                                                         /*}}}*/
 568 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 569 // ---------------------------------------------------------------------
 570 /* The package diff is added to the queue. one object is constructed
 571  * for each diff and the index
 572  */
 573 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 574                                    string URI,string URIDesc,string ShortDesc,
 575                                    HashString ExpectedHash,
 576                                    string ServerSha1,
 577                                    vector<DiffInfo> diffs)
 578    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 579      available_patches(diffs), ServerSha1(ServerSha1)
 580 {
 581
 582    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 583    DestFile += URItoFileName(URI);
 584
 585    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 586
 587    Description = URIDesc;
 588    Desc.Owner = this;
 589    Desc.ShortDesc = ShortDesc;
 590
 591    if(available_patches.empty() == true)
 592    {
 593       // we are done (yeah!)
 594       Finish(true);
 595    }
 596    else
 597    {
 598       // get the next diff
 599       State = StateFetchDiff;
 600       QueueNextDiff();
 601    }
 602 }
 603                                                                         /*}}}*/
 604 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)     /*{{{*/
 605 {
 606    if(Debug)
 607       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl
 608                 << "Falling back to normal index file aquire" << std::endl;
 609    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 610                    ExpectedHash);
 611    Finish();
 612 }
 613                                                                         /*}}}*/
 614 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 615 void pkgAcqIndexDiffs::Finish(bool allDone)
 616 {
 617    // we restore the original name, this is required, otherwise
 618    // the file will be cleaned
 619    if(allDone)
 620    {
 621       DestFile = _config->FindDir("Dir::State::lists");
 622       DestFile += URItoFileName(RealURI);
 623
 624       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 625       {
 626          RenameOnError(HashSumMismatch);
 627          Dequeue();
 628          return;
 629       }
 630
 631       // this is for the "real" finish
 632       Complete = true;
 633       Status = StatDone;
 634       Dequeue();
 635       if(Debug)
 636          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 637       return;
 638    }
 639
 640    if(Debug)
 641       std::clog << "Finishing: " << Desc.URI << std::endl;
 642    Complete = false;
 643    Status = StatDone;
 644    Dequeue();
 645    return;
 646 }
 647                                                                         /*}}}*/
 648 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 649 {
 650
 651    // calc sha1 of the just patched file
 652    string FinalFile = _config->FindDir("Dir::State::lists");
 653    FinalFile += URItoFileName(RealURI);
 654
 655    FileFd fd(FinalFile, FileFd::ReadOnly);
 656    SHA1Summation SHA1;
 657    SHA1.AddFD(fd);
 658    string local_sha1 = string(SHA1.Result());
 659    if(Debug)
 660       std::clog << "QueueNextDiff: "
 661                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 662
 663    // final file reached before all patches are applied
 664    if(local_sha1 == ServerSha1)
 665    {
 666       Finish(true);
 667       return true;
 668    }
 669
 670    // remove all patches until the next matching patch is found
 671    // this requires the Index file to be ordered
 672    for(vector<DiffInfo>::iterator I=available_patches.begin();
 673        available_patches.empty() == false &&
 674           I != available_patches.end() &&
 675           I->sha1 != local_sha1;
 676        ++I)
 677    {
 678       available_patches.erase(I);
 679    }
 680
 681    // error checking and falling back if no patch was found
 682    if(available_patches.empty() == true)
 683    {
 684       Failed("", NULL);
 685       return false;
 686    }
 687
 688    // queue the right diff
 689    Desc.URI = string(RealURI) + ".diff/" + available_patches[0].file + ".gz";
 690    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 691    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 692    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 693
 694    if(Debug)
 695       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 696
 697    QueueURI(Desc);
 698
 699    return true;
 700 }
 701                                                                         /*}}}*/
 702 void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size,string Md5Hash,      /*{{{*/
 703                             pkgAcquire::MethodConfig *Cnf)
 704 {
 705    if(Debug)
 706       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 707
 708    Item::Done(Message,Size,Md5Hash,Cnf);
 709
 710    string FinalFile;
 711    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 712
 713    // sucess in downloading a diff, enter ApplyDiff state
 714    if(State == StateFetchDiff)
 715    {
 716
 717       // rred excepts the patch as $FinalFile.ed
 718       Rename(DestFile,FinalFile+".ed");
 719
 720       if(Debug)
 721          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 722
 723       State = StateApplyDiff;
 724       Local = true;
 725       Desc.URI = "rred:" + FinalFile;
 726       QueueURI(Desc);
 727       Mode = "rred";
 728       return;
 729    }
 730
 731
 732    // success in download/apply a diff, queue next (if needed)
 733    if(State == StateApplyDiff)
 734    {
 735       // remove the just applied patch
 736       available_patches.erase(available_patches.begin());
 737
 738       // move into place
 739       if(Debug)
 740       {
 741          std::clog << "Moving patched file in place: " << std::endl
 742                    << DestFile << " -> " << FinalFile << std::endl;
 743       }
 744       Rename(DestFile,FinalFile);
 745       chmod(FinalFile.c_str(),0644);
 746
 747       // see if there is more to download
 748       if(available_patches.empty() == false) {
 749          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 750                               ExpectedHash, ServerSha1, available_patches);
 751          return Finish();
 752       } else
 753          return Finish(true);
 754    }
 755 }
 756                                                                         /*}}}*/
 757 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 758 // ---------------------------------------------------------------------
 759 /* The package file is added to the queue and a second class is
 760    instantiated to fetch the revision file */
 761 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 762                          string URI,string URIDesc,string ShortDesc,
 763                          HashString ExpectedHash, string comprExt)
 764    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 765 {
 766    if(comprExt.empty() == true)
 767    {
 768       // autoselect the compression method
 769       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 770       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 771          comprExt.append(*t).append(" ");
 772       if (comprExt.empty() == false)
 773          comprExt.erase(comprExt.end()-1);
 774    }
 775    CompressionExtension = comprExt;
 776
 777    Init(URI, URIDesc, ShortDesc);
 778 }
 779 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target,
 780                          HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
 781    : Item(Owner), RealURI(Target->URI), ExpectedHash(ExpectedHash)
 782 {
 783    // autoselect the compression method
 784    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 785    CompressionExtension = "";
 786    if (ExpectedHash.empty() == false)
 787    {
 788       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 789          if (*t == "uncompressed" || MetaIndexParser->Exists(string(Target->MetaKey).append(".").append(*t)) == true)
 790             CompressionExtension.append(*t).append(" ");
 791    }
 792    else
 793    {
 794       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 795          CompressionExtension.append(*t).append(" ");
 796    }
 797    if (CompressionExtension.empty() == false)
 798       CompressionExtension.erase(CompressionExtension.end()-1);
 799
 800    // only verify non-optional targets, see acquire-item.h for a FIXME
 801    // to make this more flexible
 802    if (Target->IsOptional())
 803      Verify = false;
 804    else
 805      Verify = true;
 806
 807    Init(Target->URI, Target->Description, Target->ShortDesc);
 808 }
 809                                                                         /*}}}*/
 810 // AcqIndex::Init - defered Constructor                                 /*{{{*/
 811 void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &ShortDesc) {
 812    Decompression = false;
 813    Erase = false;
 814
 815    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 816    DestFile += URItoFileName(URI);
 817
 818    std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
 819    if (comprExt == "uncompressed")
 820       Desc.URI = URI;
 821    else
 822       Desc.URI = URI + '.' + comprExt;
 823
 824    Desc.Description = URIDesc;
 825    Desc.Owner = this;
 826    Desc.ShortDesc = ShortDesc;
 827
 828    QueueURI(Desc);
 829 }
 830                                                                         /*}}}*/
 831 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 832 // ---------------------------------------------------------------------
 833 /* The only header we use is the last-modified header. */
 834 string pkgAcqIndex::Custom600Headers()
 835 {
 836    string Final = _config->FindDir("Dir::State::lists");
 837    Final += URItoFileName(RealURI);
 838    if (_config->FindB("Acquire::GzipIndexes",false))
 839       Final += ".gz";
 840
 841    string msg = "\nIndex-File: true";
 842    // FIXME: this really should use "IndexTarget::IsOptional()" but that
 843    //        seems to be difficult without breaking ABI
 844    if (ShortDesc().find("Translation") != 0)
 845       msg += "\nFail-Ignore: true";
 846    struct stat Buf;
 847    if (stat(Final.c_str(),&Buf) == 0)
 848       msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 849
 850    return msg;
 851 }
 852                                                                         /*}}}*/
 853 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
 854 {
 855    size_t const nextExt = CompressionExtension.find(' ');
 856    if (nextExt != std::string::npos)
 857    {
 858       CompressionExtension = CompressionExtension.substr(nextExt+1);
 859       Init(RealURI, Desc.Description, Desc.ShortDesc);
 860       return;
 861    }
 862
 863    // on decompression failure, remove bad versions in partial/
 864    if (Decompression && Erase) {
 865       string s = _config->FindDir("Dir::State::lists") + "partial/";
 866       s.append(URItoFileName(RealURI));
 867       unlink(s.c_str());
 868    }
 869
 870    Item::Failed(Message,Cnf);
 871 }
 872                                                                         /*}}}*/
 873 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
 874 // ---------------------------------------------------------------------
 875 /* This goes through a number of states.. On the initial fetch the
 876    method could possibly return an alternate filename which points
 877    to the uncompressed version of the file. If this is so the file
 878    is copied into the partial directory. In all other cases the file
 879    is decompressed with a gzip uri. */
 880 void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash,
 881                        pkgAcquire::MethodConfig *Cfg)
 882 {
 883    Item::Done(Message,Size,Hash,Cfg);
 884
 885    if (Decompression == true)
 886    {
 887       if (_config->FindB("Debug::pkgAcquire::Auth", false))
 888       {
 889          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
 890          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
 891       }
 892
 893       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
 894       {
 895          RenameOnError(HashSumMismatch);
 896          return;
 897       }
 898
 899       /* Verify the index file for correctness (all indexes must
 900        * have a Package field) (LP: #346386) (Closes: #627642) */
 901       if (Verify == true)
 902       {
 903          FileFd fd(DestFile, FileFd::ReadOnly);
 904          // Only test for correctness if the file is not empty (empty is ok)
 905          if (fd.FileSize() > 0)
 906          {
 907             pkgTagSection sec;
 908             pkgTagFile tag(&fd);
 909
 910             // all our current indexes have a field 'Package' in each section
 911             if (_error->PendingError() == true || tag.Step(sec) == false || sec.Exists("Package") == false)
 912             {
 913                RenameOnError(InvalidFormat);
 914                return;
 915             }
 916          }
 917       }
 918
 919       // Done, move it into position
 920       string FinalFile = _config->FindDir("Dir::State::lists");
 921       FinalFile += URItoFileName(RealURI);
 922       Rename(DestFile,FinalFile);
 923       chmod(FinalFile.c_str(),0644);
 924
 925       /* We restore the original name to DestFile so that the clean operation
 926          will work OK */
 927       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 928       DestFile += URItoFileName(RealURI);
 929
 930       // Remove the compressed version.
 931       if (Erase == true)
 932          unlink(DestFile.c_str());
 933       return;
 934    }
 935
 936    Erase = false;
 937    Complete = true;
 938
 939    // Handle the unzipd case
 940    string FileName = LookupTag(Message,"Alt-Filename");
 941    if (FileName.empty() == false)
 942    {
 943       // The files timestamp matches
 944       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
 945          return;
 946       Decompression = true;
 947       Local = true;
 948       DestFile += ".decomp";
 949       Desc.URI = "copy:" + FileName;
 950       QueueURI(Desc);
 951       Mode = "copy";
 952       return;
 953    }
 954
 955    FileName = LookupTag(Message,"Filename");
 956    if (FileName.empty() == true)
 957    {
 958       Status = StatError;
 959       ErrorText = "Method gave a blank filename";
 960    }
 961
 962    std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
 963
 964    // The files timestamp matches
 965    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) {
 966        if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
 967           // Update DestFile for .gz suffix so that the clean operation keeps it
 968           DestFile += ".gz";
 969       return;
 970     }
 971
 972    if (FileName == DestFile)
 973       Erase = true;
 974    else
 975       Local = true;
 976
 977    string decompProg;
 978
 979    // If we enable compressed indexes and already have gzip, keep it
 980    if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) {
 981       string FinalFile = _config->FindDir("Dir::State::lists");
 982       FinalFile += URItoFileName(RealURI) + ".gz";
 983       Rename(DestFile,FinalFile);
 984       chmod(FinalFile.c_str(),0644);
 985
 986       // Update DestFile for .gz suffix so that the clean operation keeps it
 987       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 988       DestFile += URItoFileName(RealURI) + ".gz";
 989       return;
 990     }
 991
 992    // get the binary name for your used compression type
 993    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
 994    if(decompProg.empty() == false);
 995    else if(compExt == "uncompressed")
 996       decompProg = "copy";
 997    else {
 998       _error->Error("Unsupported extension: %s", compExt.c_str());
 999       return;
1000    }
1001
1002    Decompression = true;
1003    DestFile += ".decomp";
1004    Desc.URI = decompProg + ":" + FileName;
1005    QueueURI(Desc);
1006
1007    // FIXME: this points to a c++ string that goes out of scope
1008    Mode = decompProg.c_str();
1009 }
1010                                                                         /*}}}*/
1011 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
1012 // ---------------------------------------------------------------------
1013 /* The Translation file is added to the queue */
1014 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
1015                             string URI,string URIDesc,string ShortDesc)
1016   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
1017 {
1018 }
1019 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const *Target,
1020                          HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
1021   : pkgAcqIndex(Owner, Target, ExpectedHash, MetaIndexParser)
1022 {
1023 }
1024                                                                         /*}}}*/
1025 // AcqIndexTrans::Custom600Headers - Insert custom request headers      /*{{{*/
1026 // ---------------------------------------------------------------------
1027 string pkgAcqIndexTrans::Custom600Headers()
1028 {
1029    string Final = _config->FindDir("Dir::State::lists");
1030    Final += URItoFileName(RealURI);
1031
1032    struct stat Buf;
1033    if (stat(Final.c_str(),&Buf) != 0)
1034       return "\nFail-Ignore: true\nIndex-File: true";
1035    return "\nFail-Ignore: true\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1036 }
1037                                                                         /*}}}*/
1038 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
1039 // ---------------------------------------------------------------------
1040 /* */
1041 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1042 {
1043    size_t const nextExt = CompressionExtension.find(' ');
1044    if (nextExt != std::string::npos)
1045    {
1046       CompressionExtension = CompressionExtension.substr(nextExt+1);
1047       Init(RealURI, Desc.Description, Desc.ShortDesc);
1048       Status = StatIdle;
1049       return;
1050    }
1051
1052    if (Cnf->LocalOnly == true ||
1053        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1054    {
1055       // Ignore this
1056       Status = StatDone;
1057       Complete = false;
1058       Dequeue();
1059       return;
1060    }
1061
1062    Item::Failed(Message,Cnf);
1063 }
1064                                                                         /*}}}*/
1065 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
1066                              string URI,string URIDesc,string ShortDesc,
1067                              string MetaIndexURI, string MetaIndexURIDesc,
1068                              string MetaIndexShortDesc,
1069                              const vector<IndexTarget*>* IndexTargets,
1070                              indexRecords* MetaIndexParser) :
1071    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
1072    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1073    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
1074 {
1075    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1076    DestFile += URItoFileName(URI);
1077
1078    // remove any partial downloaded sig-file in partial/.
1079    // it may confuse proxies and is too small to warrant a
1080    // partial download anyway
1081    unlink(DestFile.c_str());
1082
1083    // Create the item
1084    Desc.Description = URIDesc;
1085    Desc.Owner = this;
1086    Desc.ShortDesc = ShortDesc;
1087    Desc.URI = URI;
1088
1089    string Final = _config->FindDir("Dir::State::lists");
1090    Final += URItoFileName(RealURI);
1091    if (RealFileExists(Final) == true)
1092    {
1093       // File was already in place.  It needs to be re-downloaded/verified
1094       // because Release might have changed, we do give it a differnt
1095       // name than DestFile because otherwise the http method will
1096       // send If-Range requests and there are too many broken servers
1097       // out there that do not understand them
1098       LastGoodSig = DestFile+".reverify";
1099       Rename(Final,LastGoodSig);
1100    }
1101
1102    QueueURI(Desc);
1103 }
1104                                                                         /*}}}*/
1105 pkgAcqMetaSig::~pkgAcqMetaSig()                                         /*{{{*/
1106 {
1107    // if the file was never queued undo file-changes done in the constructor
1108    if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false &&
1109          LastGoodSig.empty() == false)
1110    {
1111       string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1112       if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true)
1113          Rename(LastGoodSig, Final);
1114    }
1115
1116 }
1117                                                                         /*}}}*/
1118 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
1119 // ---------------------------------------------------------------------
1120 /* The only header we use is the last-modified header. */
1121 string pkgAcqMetaSig::Custom600Headers()
1122 {
1123    struct stat Buf;
1124    if (stat(LastGoodSig.c_str(),&Buf) != 0)
1125       return "\nIndex-File: true";
1126
1127    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1128 }
1129
1130 void pkgAcqMetaSig::Done(string Message,unsigned long long Size,string MD5,
1131                          pkgAcquire::MethodConfig *Cfg)
1132 {
1133    Item::Done(Message,Size,MD5,Cfg);
1134
1135    string FileName = LookupTag(Message,"Filename");
1136    if (FileName.empty() == true)
1137    {
1138       Status = StatError;
1139       ErrorText = "Method gave a blank filename";
1140       return;
1141    }
1142
1143    if (FileName != DestFile)
1144    {
1145       // We have to copy it into place
1146       Local = true;
1147       Desc.URI = "copy:" + FileName;
1148       QueueURI(Desc);
1149       return;
1150    }
1151
1152    Complete = true;
1153
1154    // put the last known good file back on i-m-s hit (it will
1155    // be re-verified again)
1156    // Else do nothing, we have the new file in DestFile then
1157    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1158       Rename(LastGoodSig, DestFile);
1159
1160    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
1161    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc,
1162                        MetaIndexShortDesc,  DestFile, IndexTargets,
1163                        MetaIndexParser);
1164
1165 }
1166                                                                         /*}}}*/
1167 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
1168 {
1169    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1170
1171    // if we get a network error we fail gracefully
1172    if(Status == StatTransientNetworkError)
1173    {
1174       Item::Failed(Message,Cnf);
1175       // move the sigfile back on transient network failures
1176       if(FileExists(LastGoodSig))
1177          Rename(LastGoodSig,Final);
1178
1179       // set the status back to , Item::Failed likes to reset it
1180       Status = pkgAcquire::Item::StatTransientNetworkError;
1181       return;
1182    }
1183
1184    // Delete any existing sigfile when the acquire failed
1185    unlink(Final.c_str());
1186
1187    // queue a pkgAcqMetaIndex with no sigfile
1188    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1189                        "", IndexTargets, MetaIndexParser);
1190
1191    if (Cnf->LocalOnly == true ||
1192        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1193    {
1194       // Ignore this
1195       Status = StatDone;
1196       Complete = false;
1197       Dequeue();
1198       return;
1199    }
1200
1201    Item::Failed(Message,Cnf);
1202 }
1203                                                                         /*}}}*/
1204 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
1205                                  string URI,string URIDesc,string ShortDesc,
1206                                  string SigFile,
1207                                  const vector<struct IndexTarget*>* IndexTargets,
1208                                  indexRecords* MetaIndexParser) :
1209    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
1210    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
1211 {
1212    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1213    DestFile += URItoFileName(URI);
1214
1215    // Create the item
1216    Desc.Description = URIDesc;
1217    Desc.Owner = this;
1218    Desc.ShortDesc = ShortDesc;
1219    Desc.URI = URI;
1220
1221    QueueURI(Desc);
1222 }
1223                                                                         /*}}}*/
1224 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
1225 // ---------------------------------------------------------------------
1226 /* The only header we use is the last-modified header. */
1227 string pkgAcqMetaIndex::Custom600Headers()
1228 {
1229    string Final = _config->FindDir("Dir::State::lists");
1230    Final += URItoFileName(RealURI);
1231
1232    struct stat Buf;
1233    if (stat(Final.c_str(),&Buf) != 0)
1234       return "\nIndex-File: true";
1235
1236    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1237 }
1238                                                                         /*}}}*/
1239 void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,string Hash,  /*{{{*/
1240                            pkgAcquire::MethodConfig *Cfg)
1241 {
1242    Item::Done(Message,Size,Hash,Cfg);
1243
1244    // MetaIndexes are done in two passes: one to download the
1245    // metaindex with an appropriate method, and a second to verify it
1246    // with the gpgv method
1247
1248    if (AuthPass == true)
1249    {
1250       AuthDone(Message);
1251
1252       // all cool, move Release file into place
1253       Complete = true;
1254    }
1255    else
1256    {
1257       RetrievalDone(Message);
1258       if (!Complete)
1259          // Still more retrieving to do
1260          return;
1261
1262       if (SigFile == "")
1263       {
1264          // There was no signature file, so we are finished.  Download
1265          // the indexes and do only hashsum verification if possible
1266          MetaIndexParser->Load(DestFile);
1267          QueueIndexes(false);
1268       }
1269       else
1270       {
1271          // There was a signature file, so pass it to gpgv for
1272          // verification
1273
1274          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1275             std::cerr << "Metaindex acquired, queueing gpg verification ("
1276                       << SigFile << "," << DestFile << ")\n";
1277          AuthPass = true;
1278          Desc.URI = "gpgv:" + SigFile;
1279          QueueURI(Desc);
1280          Mode = "gpgv";
1281          return;
1282       }
1283    }
1284
1285    if (Complete == true)
1286    {
1287       string FinalFile = _config->FindDir("Dir::State::lists");
1288       FinalFile += URItoFileName(RealURI);
1289       if (SigFile == DestFile)
1290          SigFile = FinalFile;
1291       Rename(DestFile,FinalFile);
1292       chmod(FinalFile.c_str(),0644);
1293       DestFile = FinalFile;
1294    }
1295 }
1296                                                                         /*}}}*/
1297 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
1298 {
1299    // We have just finished downloading a Release file (it is not
1300    // verified yet)
1301
1302    string FileName = LookupTag(Message,"Filename");
1303    if (FileName.empty() == true)
1304    {
1305       Status = StatError;
1306       ErrorText = "Method gave a blank filename";
1307       return;
1308    }
1309
1310    if (FileName != DestFile)
1311    {
1312       Local = true;
1313       Desc.URI = "copy:" + FileName;
1314       QueueURI(Desc);
1315       return;
1316    }
1317
1318    // make sure to verify against the right file on I-M-S hit
1319    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1320    if(IMSHit)
1321    {
1322       string FinalFile = _config->FindDir("Dir::State::lists");
1323       FinalFile += URItoFileName(RealURI);
1324       if (SigFile == DestFile)
1325       {
1326          SigFile = FinalFile;
1327          // constructor of pkgAcqMetaClearSig moved it out of the way,
1328          // now move it back in on IMS hit for the 'old' file
1329          string const OldClearSig = DestFile + ".reverify";
1330          if (RealFileExists(OldClearSig) == true)
1331             Rename(OldClearSig, FinalFile);
1332       }
1333       DestFile = FinalFile;
1334    }
1335    Complete = true;
1336 }
1337                                                                         /*}}}*/
1338 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1339 {
1340    // At this point, the gpgv method has succeeded, so there is a
1341    // valid signature from a key in the trusted keyring.  We
1342    // perform additional verification of its contents, and use them
1343    // to verify the indexes we are about to download
1344
1345    if (!MetaIndexParser->Load(DestFile))
1346    {
1347       Status = StatAuthError;
1348       ErrorText = MetaIndexParser->ErrorText;
1349       return;
1350    }
1351
1352    if (!VerifyVendor(Message))
1353    {
1354       return;
1355    }
1356
1357    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1358       std::cerr << "Signature verification succeeded: "
1359                 << DestFile << std::endl;
1360
1361    // Download further indexes with verification
1362    QueueIndexes(true);
1363
1364    // is it a clearsigned MetaIndex file?
1365    if (DestFile == SigFile)
1366       return;
1367
1368    // Done, move signature file into position
1369    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1370       URItoFileName(RealURI) + ".gpg";
1371    Rename(SigFile,VerifiedSigFile);
1372    chmod(VerifiedSigFile.c_str(),0644);
1373 }
1374                                                                         /*}}}*/
1375 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1376 {
1377 #if 0
1378    /* Reject invalid, existing Release files (LP: #346386) (Closes: #627642)
1379     * FIXME: Disabled; it breaks unsigned repositories without hashes */
1380    if (!verify && FileExists(DestFile) && !MetaIndexParser->Load(DestFile))
1381    {
1382       Status = StatError;
1383       ErrorText = MetaIndexParser->ErrorText;
1384       return;
1385    }
1386 #endif
1387    bool transInRelease = false;
1388    {
1389       std::vector<std::string> const keys = MetaIndexParser->MetaKeys();
1390       for (std::vector<std::string>::const_iterator k = keys.begin(); k != keys.end(); ++k)
1391          // FIXME: Feels wrong to check for hardcoded string here, but what should we do else…
1392          if (k->find("Translation-") != std::string::npos)
1393          {
1394             transInRelease = true;
1395             break;
1396          }
1397    }
1398
1399    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1400         Target != IndexTargets->end();
1401         ++Target)
1402    {
1403       HashString ExpectedIndexHash;
1404       const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1405       bool compressedAvailable = false;
1406       if (Record == NULL)
1407       {
1408          if ((*Target)->IsOptional() == true)
1409          {
1410             std::vector<std::string> types = APT::Configuration::getCompressionTypes();
1411             for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
1412                if (MetaIndexParser->Exists(string((*Target)->MetaKey).append(".").append(*t)) == true)
1413                {
1414                   compressedAvailable = true;
1415                   break;
1416                }
1417          }
1418          else if (verify == true)
1419          {
1420             Status = StatAuthError;
1421             strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), (*Target)->MetaKey.c_str());
1422             return;
1423          }
1424       }
1425       else
1426       {
1427          ExpectedIndexHash = Record->Hash;
1428          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1429          {
1430             std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1431             std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1432             std::cerr << "For: " << Record->MetaKeyFilename << std::endl;
1433          }
1434          if (verify == true && ExpectedIndexHash.empty() == true && (*Target)->IsOptional() == false)
1435          {
1436             Status = StatAuthError;
1437             strprintf(ErrorText, _("Unable to find hash sum for '%s' in Release file"), (*Target)->MetaKey.c_str());
1438             return;
1439          }
1440       }
1441
1442       if ((*Target)->IsOptional() == true)
1443       {
1444          if ((*Target)->IsSubIndex() == true)
1445             new pkgAcqSubIndex(Owner, (*Target)->URI, (*Target)->Description,
1446                                 (*Target)->ShortDesc, ExpectedIndexHash);
1447          else if (transInRelease == false || Record != NULL || compressedAvailable == true)
1448          {
1449             if (_config->FindB("Acquire::PDiffs",true) == true && transInRelease == true &&
1450                 MetaIndexParser->Exists(string((*Target)->MetaKey).append(".diff/Index")) == true)
1451                new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1452                                    (*Target)->ShortDesc, ExpectedIndexHash);
1453             else
1454                new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1455          }
1456          continue;
1457       }
1458
1459       /* Queue Packages file (either diff or full packages files, depending
1460          on the users option) - we also check if the PDiff Index file is listed
1461          in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
1462          instead, but passing the required info to it is to much hassle */
1463       if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
1464           MetaIndexParser->Exists(string((*Target)->MetaKey).append(".diff/Index")) == true))
1465          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1466                              (*Target)->ShortDesc, ExpectedIndexHash);
1467       else
1468          new pkgAcqIndex(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1469    }
1470 }
1471                                                                         /*}}}*/
1472 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1473 {
1474    string::size_type pos;
1475
1476    // check for missing sigs (that where not fatal because otherwise we had
1477    // bombed earlier)
1478    string missingkeys;
1479    string msg = _("There is no public key available for the "
1480                   "following key IDs:\n");
1481    pos = Message.find("NO_PUBKEY ");
1482    if (pos != std::string::npos)
1483    {
1484       string::size_type start = pos+strlen("NO_PUBKEY ");
1485       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1486       missingkeys += (Fingerprint);
1487    }
1488    if(!missingkeys.empty())
1489       _error->Warning("%s", string(msg+missingkeys).c_str());
1490
1491    string Transformed = MetaIndexParser->GetExpectedDist();
1492
1493    if (Transformed == "../project/experimental")
1494    {
1495       Transformed = "experimental";
1496    }
1497
1498    pos = Transformed.rfind('/');
1499    if (pos != string::npos)
1500    {
1501       Transformed = Transformed.substr(0, pos);
1502    }
1503
1504    if (Transformed == ".")
1505    {
1506       Transformed = "";
1507    }
1508
1509    if (_config->FindB("Acquire::Check-Valid-Until", true) == true &&
1510        MetaIndexParser->GetValidUntil() > 0) {
1511       time_t const invalid_since = time(NULL) - MetaIndexParser->GetValidUntil();
1512       if (invalid_since > 0)
1513          // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1514          // the time since then the file is invalid - formated in the same way as in
1515          // the download progress display (e.g. 7d 3h 42min 1s)
1516          return _error->Error(
1517             _("Release file for %s is expired (invalid since %s). "
1518               "Updates for this repository will not be applied."),
1519             RealURI.c_str(), TimeToStr(invalid_since).c_str());
1520    }
1521
1522    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1523    {
1524       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1525       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1526       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1527    }
1528
1529    if (MetaIndexParser->CheckDist(Transformed) == false)
1530    {
1531       // This might become fatal one day
1532 //       Status = StatAuthError;
1533 //       ErrorText = "Conflicting distribution; expected "
1534 //          + MetaIndexParser->GetExpectedDist() + " but got "
1535 //          + MetaIndexParser->GetDist();
1536 //       return false;
1537       if (!Transformed.empty())
1538       {
1539          _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1540                          Desc.Description.c_str(),
1541                          Transformed.c_str(),
1542                          MetaIndexParser->GetDist().c_str());
1543       }
1544    }
1545
1546    return true;
1547 }
1548                                                                         /*}}}*/
1549 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1550 // ---------------------------------------------------------------------
1551 /* */
1552 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1553 {
1554    if (AuthPass == true)
1555    {
1556       // gpgv method failed, if we have a good signature
1557       string LastGoodSigFile = _config->FindDir("Dir::State::lists").append("partial/").append(URItoFileName(RealURI));
1558       if (DestFile != SigFile)
1559          LastGoodSigFile.append(".gpg");
1560       LastGoodSigFile.append(".reverify");
1561
1562       if(FileExists(LastGoodSigFile))
1563       {
1564          string VerifiedSigFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1565          if (DestFile != SigFile)
1566             VerifiedSigFile.append(".gpg");
1567          Rename(LastGoodSigFile, VerifiedSigFile);
1568          Status = StatTransientNetworkError;
1569          _error->Warning(_("An error occurred during the signature "
1570                            "verification. The repository is not updated "
1571                            "and the previous index files will be used. "
1572                            "GPG error: %s: %s\n"),
1573                          Desc.Description.c_str(),
1574                          LookupTag(Message,"Message").c_str());
1575          RunScripts("APT::Update::Auth-Failure");
1576          return;
1577       } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) {
1578          /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */
1579          _error->Error(_("GPG error: %s: %s"),
1580                          Desc.Description.c_str(),
1581                          LookupTag(Message,"Message").c_str());
1582          return;
1583       } else {
1584          _error->Warning(_("GPG error: %s: %s"),
1585                          Desc.Description.c_str(),
1586                          LookupTag(Message,"Message").c_str());
1587       }
1588       // gpgv method failed
1589       ReportMirrorFailure("GPGFailure");
1590    }
1591
1592    /* Always move the meta index, even if gpgv failed. This ensures
1593     * that PackageFile objects are correctly filled in */
1594    if (FileExists(DestFile)) {
1595       string FinalFile = _config->FindDir("Dir::State::lists");
1596       FinalFile += URItoFileName(RealURI);
1597       /* InRelease files become Release files, otherwise
1598        * they would be considered as trusted later on */
1599       if (SigFile == DestFile) {
1600          RealURI = RealURI.replace(RealURI.rfind("InRelease"), 9,
1601                                        "Release");
1602          FinalFile = FinalFile.replace(FinalFile.rfind("InRelease"), 9,
1603                                        "Release");
1604          SigFile = FinalFile;
1605       }
1606       Rename(DestFile,FinalFile);
1607       chmod(FinalFile.c_str(),0644);
1608
1609       DestFile = FinalFile;
1610    }
1611
1612    // No Release file was present, or verification failed, so fall
1613    // back to queueing Packages files without verification
1614    QueueIndexes(false);
1615 }
1616                                                                         /*}}}*/
1617 pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner,               /*{{{*/
1618                 string const &URI, string const &URIDesc, string const &ShortDesc,
1619                 string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc,
1620                 string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc,
1621                 const vector<struct IndexTarget*>* IndexTargets,
1622                 indexRecords* MetaIndexParser) :
1623         pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, "", IndexTargets, MetaIndexParser),
1624         MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1625         MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc)
1626 {
1627    SigFile = DestFile;
1628
1629    // keep the old InRelease around in case of transistent network errors
1630    string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1631    if (RealFileExists(Final) == true)
1632    {
1633       string const LastGoodSig = DestFile + ".reverify";
1634       Rename(Final,LastGoodSig);
1635    }
1636 }
1637                                                                         /*}}}*/
1638 pkgAcqMetaClearSig::~pkgAcqMetaClearSig()                               /*{{{*/
1639 {
1640    // if the file was never queued undo file-changes done in the constructor
1641    if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false)
1642    {
1643       string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1644       string const LastGoodSig = DestFile + ".reverify";
1645       if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true)
1646          Rename(LastGoodSig, Final);
1647    }
1648 }
1649                                                                         /*}}}*/
1650 // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/
1651 // ---------------------------------------------------------------------
1652 // FIXME: this can go away once the InRelease file is used widely
1653 string pkgAcqMetaClearSig::Custom600Headers()
1654 {
1655    string Final = _config->FindDir("Dir::State::lists");
1656    Final += URItoFileName(RealURI);
1657
1658    struct stat Buf;
1659    if (stat(Final.c_str(),&Buf) != 0)
1660    {
1661       Final = DestFile + ".reverify";
1662       if (stat(Final.c_str(),&Buf) != 0)
1663          return "\nIndex-File: true\nFail-Ignore: true\n";
1664    }
1665
1666    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1667 }
1668                                                                         /*}}}*/
1669 void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/
1670 {
1671    if (AuthPass == false)
1672    {
1673       // Remove the 'old' InRelease file if we try Release.gpg now as otherwise
1674       // the file will stay around and gives a false-auth impression (CVE-2012-0214)
1675       string FinalFile = _config->FindDir("Dir::State::lists");
1676       FinalFile.append(URItoFileName(RealURI));
1677       if (FileExists(FinalFile))
1678          unlink(FinalFile.c_str());
1679
1680       new pkgAcqMetaSig(Owner,
1681                         MetaSigURI, MetaSigURIDesc, MetaSigShortDesc,
1682                         MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1683                         IndexTargets, MetaIndexParser);
1684       if (Cnf->LocalOnly == true ||
1685           StringToBool(LookupTag(Message, "Transient-Failure"), false) == false)
1686          Dequeue();
1687    }
1688    else
1689       pkgAcqMetaIndex::Failed(Message, Cnf);
1690 }
1691                                                                         /*}}}*/
1692 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1693 // ---------------------------------------------------------------------
1694 /* This just sets up the initial fetch environment and queues the first
1695    possibilitiy */
1696 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1697                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1698                              string &StoreFilename) :
1699                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1700                StoreFilename(StoreFilename), Vf(Version.FileList()),
1701                Trusted(false)
1702 {
1703    Retries = _config->FindI("Acquire::Retries",0);
1704
1705    if (Version.Arch() == 0)
1706    {
1707       _error->Error(_("I wasn't able to locate a file for the %s package. "
1708                       "This might mean you need to manually fix this package. "
1709                       "(due to missing arch)"),
1710                     Version.ParentPkg().Name());
1711       return;
1712    }
1713
1714    /* We need to find a filename to determine the extension. We make the
1715       assumption here that all the available sources for this version share
1716       the same extension.. */
1717    // Skip not source sources, they do not have file fields.
1718    for (; Vf.end() == false; ++Vf)
1719    {
1720       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1721          continue;
1722       break;
1723    }
1724
1725    // Does not really matter here.. we are going to fail out below
1726    if (Vf.end() != true)
1727    {
1728       // If this fails to get a file name we will bomb out below.
1729       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1730       if (_error->PendingError() == true)
1731          return;
1732
1733       // Generate the final file name as: package_version_arch.foo
1734       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1735                       QuoteString(Version.VerStr(),"_:") + '_' +
1736                       QuoteString(Version.Arch(),"_:.") +
1737                       "." + flExtension(Parse.FileName());
1738    }
1739
1740    // check if we have one trusted source for the package. if so, switch
1741    // to "TrustedOnly" mode - but only if not in AllowUnauthenticated mode
1742    bool const allowUnauth = _config->FindB("APT::Get::AllowUnauthenticated", false);
1743    bool const debugAuth = _config->FindB("Debug::pkgAcquire::Auth", false);
1744    bool seenUntrusted = false;
1745    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; ++i)
1746    {
1747       pkgIndexFile *Index;
1748       if (Sources->FindIndex(i.File(),Index) == false)
1749          continue;
1750
1751       if (debugAuth == true)
1752          std::cerr << "Checking index: " << Index->Describe()
1753                    << "(Trusted=" << Index->IsTrusted() << ")" << std::endl;
1754
1755       if (Index->IsTrusted() == true)
1756       {
1757          Trusted = true;
1758          if (allowUnauth == false)
1759             break;
1760       }
1761       else
1762          seenUntrusted = true;
1763    }
1764
1765    // "allow-unauthenticated" restores apts old fetching behaviour
1766    // that means that e.g. unauthenticated file:// uris are higher
1767    // priority than authenticated http:// uris
1768    if (allowUnauth == true && seenUntrusted == true)
1769       Trusted = false;
1770
1771    // Select a source
1772    if (QueueNext() == false && _error->PendingError() == false)
1773       _error->Error(_("Can't find a source to download version '%s' of '%s'"),
1774                     Version.VerStr(), Version.ParentPkg().FullName(false).c_str());
1775 }
1776                                                                         /*}}}*/
1777 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1778 // ---------------------------------------------------------------------
1779 /* This queues the next available file version for download. It checks if
1780    the archive is already available in the cache and stashs the MD5 for
1781    checking later. */
1782 bool pkgAcqArchive::QueueNext()
1783 {
1784    string const ForceHash = _config->Find("Acquire::ForceHash");
1785    for (; Vf.end() == false; ++Vf)
1786    {
1787       // Ignore not source sources
1788       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1789          continue;
1790
1791       // Try to cross match against the source list
1792       pkgIndexFile *Index;
1793       if (Sources->FindIndex(Vf.File(),Index) == false)
1794             continue;
1795
1796       // only try to get a trusted package from another source if that source
1797       // is also trusted
1798       if(Trusted && !Index->IsTrusted())
1799          continue;
1800
1801       // Grab the text package record
1802       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1803       if (_error->PendingError() == true)
1804          return false;
1805
1806       string PkgFile = Parse.FileName();
1807       if (ForceHash.empty() == false)
1808       {
1809          if(stringcasecmp(ForceHash, "sha512") == 0)
1810             ExpectedHash = HashString("SHA512", Parse.SHA512Hash());
1811          else if(stringcasecmp(ForceHash, "sha256") == 0)
1812             ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
1813          else if (stringcasecmp(ForceHash, "sha1") == 0)
1814             ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
1815          else
1816             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1817       }
1818       else
1819       {
1820          string Hash;
1821          if ((Hash = Parse.SHA512Hash()).empty() == false)
1822             ExpectedHash = HashString("SHA512", Hash);
1823          else if ((Hash = Parse.SHA256Hash()).empty() == false)
1824             ExpectedHash = HashString("SHA256", Hash);
1825          else if ((Hash = Parse.SHA1Hash()).empty() == false)
1826             ExpectedHash = HashString("SHA1", Hash);
1827          else
1828             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1829       }
1830       if (PkgFile.empty() == true)
1831          return _error->Error(_("The package index files are corrupted. No Filename: "
1832                               "field for package %s."),
1833                               Version.ParentPkg().Name());
1834
1835       Desc.URI = Index->ArchiveURI(PkgFile);
1836       Desc.Description = Index->ArchiveInfo(Version);
1837       Desc.Owner = this;
1838       Desc.ShortDesc = Version.ParentPkg().Name();
1839
1840       // See if we already have the file. (Legacy filenames)
1841       FileSize = Version->Size;
1842       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
1843       struct stat Buf;
1844       if (stat(FinalFile.c_str(),&Buf) == 0)
1845       {
1846          // Make sure the size matches
1847          if ((unsigned long long)Buf.st_size == Version->Size)
1848          {
1849             Complete = true;
1850             Local = true;
1851             Status = StatDone;
1852             StoreFilename = DestFile = FinalFile;
1853             return true;
1854          }
1855
1856          /* Hmm, we have a file and its size does not match, this means it is
1857             an old style mismatched arch */
1858          unlink(FinalFile.c_str());
1859       }
1860
1861       // Check it again using the new style output filenames
1862       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
1863       if (stat(FinalFile.c_str(),&Buf) == 0)
1864       {
1865          // Make sure the size matches
1866          if ((unsigned long long)Buf.st_size == Version->Size)
1867          {
1868             Complete = true;
1869             Local = true;
1870             Status = StatDone;
1871             StoreFilename = DestFile = FinalFile;
1872             return true;
1873          }
1874
1875          /* Hmm, we have a file and its size does not match, this shouldnt
1876             happen.. */
1877          unlink(FinalFile.c_str());
1878       }
1879
1880       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
1881
1882       // Check the destination file
1883       if (stat(DestFile.c_str(),&Buf) == 0)
1884       {
1885          // Hmm, the partial file is too big, erase it
1886          if ((unsigned long long)Buf.st_size > Version->Size)
1887             unlink(DestFile.c_str());
1888          else
1889             PartialSize = Buf.st_size;
1890       }
1891
1892       // Disables download of archives - useful if no real installation follows,
1893       // e.g. if we are just interested in proposed installation order
1894       if (_config->FindB("Debug::pkgAcqArchive::NoQueue", false) == true)
1895       {
1896          Complete = true;
1897          Local = true;
1898          Status = StatDone;
1899          StoreFilename = DestFile = FinalFile;
1900          return true;
1901       }
1902
1903       // Create the item
1904       Local = false;
1905       Desc.URI = Index->ArchiveURI(PkgFile);
1906       Desc.Description = Index->ArchiveInfo(Version);
1907       Desc.Owner = this;
1908       Desc.ShortDesc = Version.ParentPkg().Name();
1909       QueueURI(Desc);
1910
1911       ++Vf;
1912       return true;
1913    }
1914    return false;
1915 }
1916                                                                         /*}}}*/
1917 // AcqArchive::Done - Finished fetching                                 /*{{{*/
1918 // ---------------------------------------------------------------------
1919 /* */
1920 void pkgAcqArchive::Done(string Message,unsigned long long Size,string CalcHash,
1921                          pkgAcquire::MethodConfig *Cfg)
1922 {
1923    Item::Done(Message,Size,CalcHash,Cfg);
1924
1925    // Check the size
1926    if (Size != Version->Size)
1927    {
1928       RenameOnError(SizeMismatch);
1929       return;
1930    }
1931
1932    // Check the hash
1933    if(ExpectedHash.toStr() != CalcHash)
1934    {
1935       RenameOnError(HashSumMismatch);
1936       return;
1937    }
1938
1939    // Grab the output filename
1940    string FileName = LookupTag(Message,"Filename");
1941    if (FileName.empty() == true)
1942    {
1943       Status = StatError;
1944       ErrorText = "Method gave a blank filename";
1945       return;
1946    }
1947
1948    Complete = true;
1949
1950    // Reference filename
1951    if (FileName != DestFile)
1952    {
1953       StoreFilename = DestFile = FileName;
1954       Local = true;
1955       return;
1956    }
1957
1958    // Done, move it into position
1959    string FinalFile = _config->FindDir("Dir::Cache::Archives");
1960    FinalFile += flNotDir(StoreFilename);
1961    Rename(DestFile,FinalFile);
1962
1963    StoreFilename = DestFile = FinalFile;
1964    Complete = true;
1965 }
1966                                                                         /*}}}*/
1967 // AcqArchive::Failed - Failure handler                                 /*{{{*/
1968 // ---------------------------------------------------------------------
1969 /* Here we try other sources */
1970 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1971 {
1972    ErrorText = LookupTag(Message,"Message");
1973
1974    /* We don't really want to retry on failed media swaps, this prevents
1975       that. An interesting observation is that permanent failures are not
1976       recorded. */
1977    if (Cnf->Removable == true &&
1978        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1979    {
1980       // Vf = Version.FileList();
1981       while (Vf.end() == false) ++Vf;
1982       StoreFilename = string();
1983       Item::Failed(Message,Cnf);
1984       return;
1985    }
1986
1987    if (QueueNext() == false)
1988    {
1989       // This is the retry counter
1990       if (Retries != 0 &&
1991           Cnf->LocalOnly == false &&
1992           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1993       {
1994          Retries--;
1995          Vf = Version.FileList();
1996          if (QueueNext() == true)
1997             return;
1998       }
1999
2000       StoreFilename = string();
2001       Item::Failed(Message,Cnf);
2002    }
2003 }
2004                                                                         /*}}}*/
2005 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
2006 // ---------------------------------------------------------------------
2007 bool pkgAcqArchive::IsTrusted()
2008 {
2009    return Trusted;
2010 }
2011                                                                         /*}}}*/
2012 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
2013 // ---------------------------------------------------------------------
2014 /* */
2015 void pkgAcqArchive::Finished()
2016 {
2017    if (Status == pkgAcquire::Item::StatDone &&
2018        Complete == true)
2019       return;
2020    StoreFilename = string();
2021 }
2022                                                                         /*}}}*/
2023 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
2024 // ---------------------------------------------------------------------
2025 /* The file is added to the queue */
2026 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
2027                        unsigned long long Size,string Dsc,string ShortDesc,
2028                        const string &DestDir, const string &DestFilename,
2029                        bool IsIndexFile) :
2030                        Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile)
2031 {
2032    Retries = _config->FindI("Acquire::Retries",0);
2033
2034    if(!DestFilename.empty())
2035       DestFile = DestFilename;
2036    else if(!DestDir.empty())
2037       DestFile = DestDir + "/" + flNotDir(URI);
2038    else
2039       DestFile = flNotDir(URI);
2040
2041    // Create the item
2042    Desc.URI = URI;
2043    Desc.Description = Dsc;
2044    Desc.Owner = this;
2045
2046    // Set the short description to the archive component
2047    Desc.ShortDesc = ShortDesc;
2048
2049    // Get the transfer sizes
2050    FileSize = Size;
2051    struct stat Buf;
2052    if (stat(DestFile.c_str(),&Buf) == 0)
2053    {
2054       // Hmm, the partial file is too big, erase it
2055       if ((unsigned long long)Buf.st_size > Size)
2056          unlink(DestFile.c_str());
2057       else
2058          PartialSize = Buf.st_size;
2059    }
2060
2061    QueueURI(Desc);
2062 }
2063                                                                         /*}}}*/
2064 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
2065 // ---------------------------------------------------------------------
2066 /* */
2067 void pkgAcqFile::Done(string Message,unsigned long long Size,string CalcHash,
2068                       pkgAcquire::MethodConfig *Cnf)
2069 {
2070    Item::Done(Message,Size,CalcHash,Cnf);
2071
2072    // Check the hash
2073    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
2074    {
2075       RenameOnError(HashSumMismatch);
2076       return;
2077    }
2078
2079    string FileName = LookupTag(Message,"Filename");
2080    if (FileName.empty() == true)
2081    {
2082       Status = StatError;
2083       ErrorText = "Method gave a blank filename";
2084       return;
2085    }
2086
2087    Complete = true;
2088
2089    // The files timestamp matches
2090    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
2091       return;
2092
2093    // We have to copy it into place
2094    if (FileName != DestFile)
2095    {
2096       Local = true;
2097       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
2098           Cnf->Removable == true)
2099       {
2100          Desc.URI = "copy:" + FileName;
2101          QueueURI(Desc);
2102          return;
2103       }
2104
2105       // Erase the file if it is a symlink so we can overwrite it
2106       struct stat St;
2107       if (lstat(DestFile.c_str(),&St) == 0)
2108       {
2109          if (S_ISLNK(St.st_mode) != 0)
2110             unlink(DestFile.c_str());
2111       }
2112
2113       // Symlink the file
2114       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
2115       {
2116          ErrorText = "Link to " + DestFile + " failure ";
2117          Status = StatError;
2118          Complete = false;
2119       }
2120    }
2121 }
2122                                                                         /*}}}*/
2123 // AcqFile::Failed - Failure handler                                    /*{{{*/
2124 // ---------------------------------------------------------------------
2125 /* Here we try other sources */
2126 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2127 {
2128    ErrorText = LookupTag(Message,"Message");
2129
2130    // This is the retry counter
2131    if (Retries != 0 &&
2132        Cnf->LocalOnly == false &&
2133        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2134    {
2135       Retries--;
2136       QueueURI(Desc);
2137       return;
2138    }
2139
2140    Item::Failed(Message,Cnf);
2141 }
2142                                                                         /*}}}*/
2143 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
2144 // ---------------------------------------------------------------------
2145 /* The only header we use is the last-modified header. */
2146 string pkgAcqFile::Custom600Headers()
2147 {
2148    if (IsIndexFile)
2149       return "\nIndex-File: true";
2150    return "";
2151 }
2152                                                                         /*}}}*/