apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <apt-pkg/acquire-item.h>
  17 #include <apt-pkg/configuration.h>
  18 #include <apt-pkg/aptconfiguration.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/vendorlist.h>
  21 #include <apt-pkg/error.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <apt-pkg/fileutl.h>
  24 #include <apt-pkg/md5.h>
  25 #include <apt-pkg/sha1.h>
  26 #include <apt-pkg/tagfile.h>
  27
  28 #include <apti18n.h>
  29
  30 #include <sys/stat.h>
  31 #include <unistd.h>
  32 #include <errno.h>
  33 #include <string>
  34 #include <sstream>
  35 #include <stdio.h>
  36 #include <ctime>
  37                                                                         /*}}}*/
  38
  39 using namespace std;
  40
  41 // Acquire::Item::Item - Constructor                                    /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* */
  44 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  45                        PartialSize(0), Mode(0), ID(0), Complete(false),
  46                        Local(false), QueueCounter(0)
  47 {
  48    Owner->Add(this);
  49    Status = StatIdle;
  50 }
  51                                                                         /*}}}*/
  52 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  53 // ---------------------------------------------------------------------
  54 /* */
  55 pkgAcquire::Item::~Item()
  56 {
  57    Owner->Remove(this);
  58 }
  59                                                                         /*}}}*/
  60 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  61 // ---------------------------------------------------------------------
  62 /* We return to an idle state if there are still other queues that could
  63    fetch this object */
  64 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  65 {
  66    Status = StatIdle;
  67    ErrorText = LookupTag(Message,"Message");
  68    UsedMirror =  LookupTag(Message,"UsedMirror");
  69    if (QueueCounter <= 1)
  70    {
  71       /* This indicates that the file is not available right now but might
  72          be sometime later. If we do a retry cycle then this should be
  73          retried [CDROMs] */
  74       if (Cnf->LocalOnly == true &&
  75           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  76       {
  77          Status = StatIdle;
  78          Dequeue();
  79          return;
  80       }
  81
  82       Status = StatError;
  83       Dequeue();
  84    }
  85
  86    // report mirror failure back to LP if we actually use a mirror
  87    string FailReason = LookupTag(Message, "FailReason");
  88    if(FailReason.size() != 0)
  89       ReportMirrorFailure(FailReason);
  90    else
  91       ReportMirrorFailure(ErrorText);
  92 }
  93                                                                         /*}}}*/
  94 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
  95 // ---------------------------------------------------------------------
  96 /* Stash status and the file size. Note that setting Complete means
  97    sub-phases of the acquire process such as decompresion are operating */
  98 void pkgAcquire::Item::Start(string /*Message*/,unsigned long Size)
  99 {
 100    Status = StatFetching;
 101    if (FileSize == 0 && Complete == false)
 102       FileSize = Size;
 103 }
 104                                                                         /*}}}*/
 105 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
 106 // ---------------------------------------------------------------------
 107 /* */
 108 void pkgAcquire::Item::Done(string Message,unsigned long Size,string Hash,
 109                             pkgAcquire::MethodConfig *Cnf)
 110 {
 111    // We just downloaded something..
 112    string FileName = LookupTag(Message,"Filename");
 113    UsedMirror =  LookupTag(Message,"UsedMirror");
 114    if (Complete == false && !Local && FileName == DestFile)
 115    {
 116       if (Owner->Log != 0)
 117          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 118    }
 119
 120    if (FileSize == 0)
 121       FileSize= Size;
 122    Status = StatDone;
 123    ErrorText = string();
 124    Owner->Dequeue(this);
 125 }
 126                                                                         /*}}}*/
 127 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 128 // ---------------------------------------------------------------------
 129 /* This helper function is used by alot of item methods as thier final
 130    step */
 131 void pkgAcquire::Item::Rename(string From,string To)
 132 {
 133    if (rename(From.c_str(),To.c_str()) != 0)
 134    {
 135       char S[300];
 136       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 137               From.c_str(),To.c_str());
 138       Status = StatError;
 139       ErrorText = S;
 140    }
 141 }
 142                                                                         /*}}}*/
 143 // Acquire::Item::ReportMirrorFailure                                   /*{{{*/
 144 // ---------------------------------------------------------------------
 145 void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
 146 {
 147    // we only act if a mirror was used at all
 148    if(UsedMirror.empty())
 149       return;
 150 #if 0
 151    std::cerr << "\nReportMirrorFailure: "
 152              << UsedMirror
 153              << " Uri: " << DescURI()
 154              << " FailCode: "
 155              << FailCode << std::endl;
 156 #endif
 157    const char *Args[40];
 158    unsigned int i = 0;
 159    string report = _config->Find("Methods::Mirror::ProblemReporting",
 160                                  "/usr/lib/apt/apt-report-mirror-failure");
 161    if(!FileExists(report))
 162       return;
 163    Args[i++] = report.c_str();
 164    Args[i++] = UsedMirror.c_str();
 165    Args[i++] = DescURI().c_str();
 166    Args[i++] = FailCode.c_str();
 167    Args[i++] = NULL;
 168    pid_t pid = ExecFork();
 169    if(pid < 0)
 170    {
 171       _error->Error("ReportMirrorFailure Fork failed");
 172       return;
 173    }
 174    else if(pid == 0)
 175    {
 176       execvp(Args[0], (char**)Args);
 177       std::cerr << "Could not exec " << Args[0] << std::endl;
 178       _exit(100);
 179    }
 180    if(!ExecWait(pid, "report-mirror-failure"))
 181    {
 182       _error->Warning("Couldn't report problem to '%s'",
 183                       _config->Find("Methods::Mirror::ProblemReporting").c_str());
 184    }
 185 }
 186                                                                         /*}}}*/
 187 // AcqSubIndex::AcqSubIndex - Constructor                               /*{{{*/
 188 // ---------------------------------------------------------------------
 189 /* Get the Index file first and see if there are languages available
 190  * If so, create a pkgAcqIndexTrans for the found language(s).
 191  */
 192 pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, string const &URI,
 193                                  string const &URIDesc, string const &ShortDesc,
 194                                  HashString const &ExpectedHash)
 195    : Item(Owner), ExpectedHash(ExpectedHash)
 196 {
 197    Debug = _config->FindB("Debug::pkgAcquire::SubIndex",false);
 198
 199    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 200    DestFile += URItoFileName(URI);
 201
 202    Desc.URI = URI;
 203    Desc.Description = URIDesc;
 204    Desc.Owner = this;
 205    Desc.ShortDesc = ShortDesc;
 206
 207    QueueURI(Desc);
 208
 209    if(Debug)
 210       std::clog << "pkgAcqSubIndex: " << Desc.URI << std::endl;
 211 }
 212                                                                         /*}}}*/
 213 // AcqSubIndex::Custom600Headers - Insert custom request headers        /*{{{*/
 214 // ---------------------------------------------------------------------
 215 /* The only header we use is the last-modified header. */
 216 string pkgAcqSubIndex::Custom600Headers()
 217 {
 218    string Final = _config->FindDir("Dir::State::lists");
 219    Final += URItoFileName(Desc.URI);
 220
 221    struct stat Buf;
 222    if (stat(Final.c_str(),&Buf) != 0)
 223       return "\nIndex-File: true\nFail-Ignore: true\n";
 224    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 225 }
 226                                                                         /*}}}*/
 227 void pkgAcqSubIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)       /*{{{*/
 228 {
 229    if(Debug)
 230       std::clog << "pkgAcqSubIndex failed: " << Desc.URI << std::endl;
 231
 232    Complete = false;
 233    Status = StatDone;
 234    Dequeue();
 235
 236    // No good Index is provided, so try guessing
 237    std::vector<std::string> langs = APT::Configuration::getLanguages(true);
 238    for (std::vector<std::string>::const_iterator l = langs.begin();
 239         l != langs.end(); ++l)
 240    {
 241       if (*l == "none") continue;
 242       string const file = "Translation-" + *l;
 243       new pkgAcqIndexTrans(Owner, Desc.URI.substr(0, Desc.URI.rfind('/')+1).append(file),
 244                 Desc.Description.erase(Desc.Description.rfind(' ')+1).append(file),
 245                 file);
 246    }
 247 }
 248                                                                         /*}}}*/
 249 void pkgAcqSubIndex::Done(string Message,unsigned long Size,string Md5Hash,     /*{{{*/
 250                            pkgAcquire::MethodConfig *Cnf)
 251 {
 252    if(Debug)
 253       std::clog << "pkgAcqSubIndex::Done(): " << Desc.URI << std::endl;
 254
 255    string FileName = LookupTag(Message,"Filename");
 256    if (FileName.empty() == true)
 257    {
 258       Status = StatError;
 259       ErrorText = "Method gave a blank filename";
 260       return;
 261    }
 262
 263    if (FileName != DestFile)
 264    {
 265       Local = true;
 266       Desc.URI = "copy:" + FileName;
 267       QueueURI(Desc);
 268       return;
 269    }
 270
 271    Item::Done(Message,Size,Md5Hash,Cnf);
 272
 273    string FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(Desc.URI);
 274
 275    // sucess in downloading the index
 276    // rename the index
 277    if(Debug)
 278       std::clog << "Renaming: " << DestFile << " -> " << FinalFile << std::endl;
 279    Rename(DestFile,FinalFile);
 280    chmod(FinalFile.c_str(),0644);
 281    DestFile = FinalFile;
 282
 283    if(ParseIndex(DestFile) == false)
 284       return Failed("", NULL);
 285
 286    Complete = true;
 287    Status = StatDone;
 288    Dequeue();
 289    return;
 290 }
 291                                                                         /*}}}*/
 292 bool pkgAcqSubIndex::ParseIndex(string const &IndexFile)                /*{{{*/
 293 {
 294    indexRecords SubIndexParser;
 295    if (FileExists(IndexFile) == false || SubIndexParser.Load(IndexFile) == false)
 296       return false;
 297
 298    std::vector<std::string> lang = APT::Configuration::getLanguages(true);
 299    for (std::vector<std::string>::const_iterator l = lang.begin();
 300         l != lang.end(); ++l)
 301    {
 302       if (*l == "none")
 303          continue;
 304
 305       string file = "Translation-" + *l;
 306       indexRecords::checkSum const *Record = SubIndexParser.Lookup(file);
 307       HashString expected;
 308       if (Record == NULL)
 309       {
 310          // FIXME: the Index file provided by debian currently only includes bz2 records
 311          Record = SubIndexParser.Lookup(file + ".bz2");
 312          if (Record == NULL)
 313             continue;
 314       }
 315       else
 316       {
 317          expected = Record->Hash;
 318          if (expected.empty() == true)
 319             continue;
 320       }
 321
 322       IndexTarget target;
 323       target.Description = Desc.Description.erase(Desc.Description.rfind(' ')+1).append(file);
 324       target.MetaKey = file;
 325       target.ShortDesc = file;
 326       target.URI = Desc.URI.substr(0, Desc.URI.rfind('/')+1).append(file);
 327       new pkgAcqIndexTrans(Owner, &target, expected, &SubIndexParser);
 328    }
 329    return true;
 330 }
 331                                                                         /*}}}*/
 332 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 333 // ---------------------------------------------------------------------
 334 /* Get the DiffIndex file first and see if there are patches availabe
 335  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 336  * patches. If anything goes wrong in that process, it will fall back to
 337  * the original packages file
 338  */
 339 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 340                                  string URI,string URIDesc,string ShortDesc,
 341                                  HashString ExpectedHash)
 342    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 343      Description(URIDesc)
 344 {
 345
 346    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 347
 348    Desc.Description = URIDesc + "/DiffIndex";
 349    Desc.Owner = this;
 350    Desc.ShortDesc = ShortDesc;
 351    Desc.URI = URI + ".diff/Index";
 352
 353    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 354    DestFile += URItoFileName(URI) + string(".DiffIndex");
 355
 356    if(Debug)
 357       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 358
 359    // look for the current package file
 360    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 361    CurrentPackagesFile += URItoFileName(RealURI);
 362
 363    // FIXME: this file:/ check is a hack to prevent fetching
 364    //        from local sources. this is really silly, and
 365    //        should be fixed cleanly as soon as possible
 366    if(!FileExists(CurrentPackagesFile) ||
 367       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 368    {
 369       // we don't have a pkg file or we don't want to queue
 370       if(Debug)
 371          std::clog << "No index file, local or canceld by user" << std::endl;
 372       Failed("", NULL);
 373       return;
 374    }
 375
 376    if(Debug)
 377       std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): "
 378                 << CurrentPackagesFile << std::endl;
 379
 380    QueueURI(Desc);
 381
 382 }
 383                                                                         /*}}}*/
 384 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 385 // ---------------------------------------------------------------------
 386 /* The only header we use is the last-modified header. */
 387 string pkgAcqDiffIndex::Custom600Headers()
 388 {
 389    string Final = _config->FindDir("Dir::State::lists");
 390    Final += URItoFileName(RealURI) + string(".IndexDiff");
 391
 392    if(Debug)
 393       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 394
 395    struct stat Buf;
 396    if (stat(Final.c_str(),&Buf) != 0)
 397       return "\nIndex-File: true";
 398
 399    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 400 }
 401                                                                         /*}}}*/
 402 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 403 {
 404    if(Debug)
 405       std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile
 406                 << std::endl;
 407
 408    pkgTagSection Tags;
 409    string ServerSha1;
 410    vector<DiffInfo> available_patches;
 411
 412    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 413    pkgTagFile TF(&Fd);
 414    if (_error->PendingError() == true)
 415       return false;
 416
 417    if(TF.Step(Tags) == true)
 418    {
 419       bool found = false;
 420       DiffInfo d;
 421       string size;
 422
 423       string const tmp = Tags.FindS("SHA1-Current");
 424       std::stringstream ss(tmp);
 425       ss >> ServerSha1 >> size;
 426       unsigned long const ServerSize = atol(size.c_str());
 427
 428       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 429       SHA1Summation SHA1;
 430       SHA1.AddFD(fd.Fd(), fd.Size());
 431       string const local_sha1 = SHA1.Result();
 432
 433       if(local_sha1 == ServerSha1)
 434       {
 435          // we have the same sha1 as the server
 436          if(Debug)
 437             std::clog << "Package file is up-to-date" << std::endl;
 438          // set found to true, this will queue a pkgAcqIndexDiffs with
 439          // a empty availabe_patches
 440          found = true;
 441       }
 442       else
 443       {
 444          if(Debug)
 445             std::clog << "SHA1-Current: " << ServerSha1 << " and we start at "<< fd.Name() << " " << fd.Size() << " " << local_sha1 << std::endl;
 446
 447          // check the historie and see what patches we need
 448          string const history = Tags.FindS("SHA1-History");
 449          std::stringstream hist(history);
 450          while(hist >> d.sha1 >> size >> d.file)
 451          {
 452             // read until the first match is found
 453             // from that point on, we probably need all diffs
 454             if(d.sha1 == local_sha1)
 455                found=true;
 456             else if (found == false)
 457                continue;
 458
 459             if(Debug)
 460                std::clog << "Need to get diff: " << d.file << std::endl;
 461             available_patches.push_back(d);
 462          }
 463
 464          if (available_patches.empty() == false)
 465          {
 466             // patching with too many files is rather slow compared to a fast download
 467             unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 468             if (fileLimit != 0 && fileLimit < available_patches.size())
 469             {
 470                if (Debug)
 471                   std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 472                         << ") so fallback to complete download" << std::endl;
 473                return false;
 474             }
 475
 476             // see if the patches are too big
 477             found = false; // it was true and it will be true again at the end
 478             d = *available_patches.begin();
 479             string const firstPatch = d.file;
 480             unsigned long patchesSize = 0;
 481             std::stringstream patches(Tags.FindS("SHA1-Patches"));
 482             while(patches >> d.sha1 >> size >> d.file)
 483             {
 484                if (firstPatch == d.file)
 485                   found = true;
 486                else if (found == false)
 487                   continue;
 488
 489                patchesSize += atol(size.c_str());
 490             }
 491             unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 492             if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 493             {
 494                if (Debug)
 495                   std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 496                         << ") so fallback to complete download" << std::endl;
 497                return false;
 498             }
 499          }
 500       }
 501
 502       // we have something, queue the next diff
 503       if(found)
 504       {
 505          // queue the diffs
 506          string::size_type const last_space = Description.rfind(" ");
 507          if(last_space != string::npos)
 508             Description.erase(last_space, Description.size()-last_space);
 509          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 510                               ExpectedHash, ServerSha1, available_patches);
 511          Complete = false;
 512          Status = StatDone;
 513          Dequeue();
 514          return true;
 515       }
 516    }
 517
 518    // Nothing found, report and return false
 519    // Failing here is ok, if we return false later, the full
 520    // IndexFile is queued
 521    if(Debug)
 522       std::clog << "Can't find a patch in the index file" << std::endl;
 523    return false;
 524 }
 525                                                                         /*}}}*/
 526 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)      /*{{{*/
 527 {
 528    if(Debug)
 529       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << std::endl
 530                 << "Falling back to normal index file aquire" << std::endl;
 531
 532    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 533                    ExpectedHash);
 534
 535    Complete = false;
 536    Status = StatDone;
 537    Dequeue();
 538 }
 539                                                                         /*}}}*/
 540 void pkgAcqDiffIndex::Done(string Message,unsigned long Size,string Md5Hash,    /*{{{*/
 541                            pkgAcquire::MethodConfig *Cnf)
 542 {
 543    if(Debug)
 544       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 545
 546    Item::Done(Message,Size,Md5Hash,Cnf);
 547
 548    string FinalFile;
 549    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 550
 551    // sucess in downloading the index
 552    // rename the index
 553    FinalFile += string(".IndexDiff");
 554    if(Debug)
 555       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 556                 << std::endl;
 557    Rename(DestFile,FinalFile);
 558    chmod(FinalFile.c_str(),0644);
 559    DestFile = FinalFile;
 560
 561    if(!ParseDiffIndex(DestFile))
 562       return Failed("", NULL);
 563
 564    Complete = true;
 565    Status = StatDone;
 566    Dequeue();
 567    return;
 568 }
 569                                                                         /*}}}*/
 570 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 571 // ---------------------------------------------------------------------
 572 /* The package diff is added to the queue. one object is constructed
 573  * for each diff and the index
 574  */
 575 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 576                                    string URI,string URIDesc,string ShortDesc,
 577                                    HashString ExpectedHash,
 578                                    string ServerSha1,
 579                                    vector<DiffInfo> diffs)
 580    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 581      available_patches(diffs), ServerSha1(ServerSha1)
 582 {
 583
 584    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 585    DestFile += URItoFileName(URI);
 586
 587    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 588
 589    Description = URIDesc;
 590    Desc.Owner = this;
 591    Desc.ShortDesc = ShortDesc;
 592
 593    if(available_patches.size() == 0)
 594    {
 595       // we are done (yeah!)
 596       Finish(true);
 597    }
 598    else
 599    {
 600       // get the next diff
 601       State = StateFetchDiff;
 602       QueueNextDiff();
 603    }
 604 }
 605                                                                         /*}}}*/
 606 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)     /*{{{*/
 607 {
 608    if(Debug)
 609       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl
 610                 << "Falling back to normal index file aquire" << std::endl;
 611    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 612                    ExpectedHash);
 613    Finish();
 614 }
 615                                                                         /*}}}*/
 616 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 617 void pkgAcqIndexDiffs::Finish(bool allDone)
 618 {
 619    // we restore the original name, this is required, otherwise
 620    // the file will be cleaned
 621    if(allDone)
 622    {
 623       DestFile = _config->FindDir("Dir::State::lists");
 624       DestFile += URItoFileName(RealURI);
 625
 626       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 627       {
 628          Status = StatAuthError;
 629          ErrorText = _("MD5Sum mismatch");
 630          Rename(DestFile,DestFile + ".FAILED");
 631          Dequeue();
 632          return;
 633       }
 634
 635       // this is for the "real" finish
 636       Complete = true;
 637       Status = StatDone;
 638       Dequeue();
 639       if(Debug)
 640          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 641       return;
 642    }
 643
 644    if(Debug)
 645       std::clog << "Finishing: " << Desc.URI << std::endl;
 646    Complete = false;
 647    Status = StatDone;
 648    Dequeue();
 649    return;
 650 }
 651                                                                         /*}}}*/
 652 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 653 {
 654
 655    // calc sha1 of the just patched file
 656    string FinalFile = _config->FindDir("Dir::State::lists");
 657    FinalFile += URItoFileName(RealURI);
 658
 659    FileFd fd(FinalFile, FileFd::ReadOnly);
 660    SHA1Summation SHA1;
 661    SHA1.AddFD(fd.Fd(), fd.Size());
 662    string local_sha1 = string(SHA1.Result());
 663    if(Debug)
 664       std::clog << "QueueNextDiff: "
 665                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 666
 667    // final file reached before all patches are applied
 668    if(local_sha1 == ServerSha1)
 669    {
 670       Finish(true);
 671       return true;
 672    }
 673
 674    // remove all patches until the next matching patch is found
 675    // this requires the Index file to be ordered
 676    for(vector<DiffInfo>::iterator I=available_patches.begin();
 677        available_patches.size() > 0 &&
 678           I != available_patches.end() &&
 679           (*I).sha1 != local_sha1;
 680        I++)
 681    {
 682       available_patches.erase(I);
 683    }
 684
 685    // error checking and falling back if no patch was found
 686    if(available_patches.size() == 0)
 687    {
 688       Failed("", NULL);
 689       return false;
 690    }
 691
 692    // queue the right diff
 693    Desc.URI = string(RealURI) + ".diff/" + available_patches[0].file + ".gz";
 694    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 695    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 696    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 697
 698    if(Debug)
 699       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 700
 701    QueueURI(Desc);
 702
 703    return true;
 704 }
 705                                                                         /*}}}*/
 706 void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash,   /*{{{*/
 707                             pkgAcquire::MethodConfig *Cnf)
 708 {
 709    if(Debug)
 710       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 711
 712    Item::Done(Message,Size,Md5Hash,Cnf);
 713
 714    string FinalFile;
 715    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 716
 717    // sucess in downloading a diff, enter ApplyDiff state
 718    if(State == StateFetchDiff)
 719    {
 720
 721       // rred excepts the patch as $FinalFile.ed
 722       Rename(DestFile,FinalFile+".ed");
 723
 724       if(Debug)
 725          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 726
 727       State = StateApplyDiff;
 728       Local = true;
 729       Desc.URI = "rred:" + FinalFile;
 730       QueueURI(Desc);
 731       Mode = "rred";
 732       return;
 733    }
 734
 735
 736    // success in download/apply a diff, queue next (if needed)
 737    if(State == StateApplyDiff)
 738    {
 739       // remove the just applied patch
 740       available_patches.erase(available_patches.begin());
 741
 742       // move into place
 743       if(Debug)
 744       {
 745          std::clog << "Moving patched file in place: " << std::endl
 746                    << DestFile << " -> " << FinalFile << std::endl;
 747       }
 748       Rename(DestFile,FinalFile);
 749       chmod(FinalFile.c_str(),0644);
 750
 751       // see if there is more to download
 752       if(available_patches.size() > 0) {
 753          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 754                               ExpectedHash, ServerSha1, available_patches);
 755          return Finish();
 756       } else
 757          return Finish(true);
 758    }
 759 }
 760                                                                         /*}}}*/
 761 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 762 // ---------------------------------------------------------------------
 763 /* The package file is added to the queue and a second class is
 764    instantiated to fetch the revision file */
 765 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 766                          string URI,string URIDesc,string ShortDesc,
 767                          HashString ExpectedHash, string comprExt)
 768    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 769 {
 770    if(comprExt.empty() == true)
 771    {
 772       // autoselect the compression method
 773       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 774       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 775          comprExt.append(*t).append(" ");
 776       if (comprExt.empty() == false)
 777          comprExt.erase(comprExt.end()-1);
 778    }
 779    CompressionExtension = comprExt;
 780
 781    Init(URI, URIDesc, ShortDesc);
 782 }
 783 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target,
 784                          HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
 785    : Item(Owner), RealURI(Target->URI), ExpectedHash(ExpectedHash)
 786 {
 787    // autoselect the compression method
 788    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 789    CompressionExtension = "";
 790    if (ExpectedHash.empty() == false)
 791    {
 792       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 793          if (*t == "uncompressed" || MetaIndexParser->Exists(string(Target->MetaKey).append(".").append(*t)) == true)
 794             CompressionExtension.append(*t).append(" ");
 795    }
 796    else
 797    {
 798       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 799          CompressionExtension.append(*t).append(" ");
 800    }
 801    if (CompressionExtension.empty() == false)
 802       CompressionExtension.erase(CompressionExtension.end()-1);
 803
 804    Init(Target->URI, Target->Description, Target->ShortDesc);
 805 }
 806                                                                         /*}}}*/
 807 // AcqIndex::Init - defered Constructor                                 /*{{{*/
 808 void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &ShortDesc) {
 809    Decompression = false;
 810    Erase = false;
 811
 812    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 813    DestFile += URItoFileName(URI);
 814
 815    std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
 816    if (comprExt == "uncompressed")
 817       Desc.URI = URI;
 818    else
 819       Desc.URI = URI + '.' + comprExt;
 820
 821    Desc.Description = URIDesc;
 822    Desc.Owner = this;
 823    Desc.ShortDesc = ShortDesc;
 824
 825    QueueURI(Desc);
 826 }
 827                                                                         /*}}}*/
 828 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 829 // ---------------------------------------------------------------------
 830 /* The only header we use is the last-modified header. */
 831 string pkgAcqIndex::Custom600Headers()
 832 {
 833    string Final = _config->FindDir("Dir::State::lists");
 834    Final += URItoFileName(RealURI);
 835    if (_config->FindB("Acquire::GzipIndexes",false))
 836       Final += ".gz";
 837
 838    string msg = "\nIndex-File: true";
 839    // FIXME: this really should use "IndexTarget::IsOptional()" but that
 840    //        seems to be difficult without breaking ABI
 841    if (ShortDesc().find("Translation") != 0)
 842       msg += "\nFail-Ignore: true";
 843    struct stat Buf;
 844    if (stat(Final.c_str(),&Buf) != 0)
 845       msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 846
 847    return msg;
 848 }
 849                                                                         /*}}}*/
 850 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
 851 {
 852    size_t const nextExt = CompressionExtension.find(' ');
 853    if (nextExt != std::string::npos)
 854    {
 855       CompressionExtension = CompressionExtension.substr(nextExt+1);
 856       Init(RealURI, Desc.Description, Desc.ShortDesc);
 857       return;
 858    }
 859
 860    // on decompression failure, remove bad versions in partial/
 861    if (Decompression && Erase) {
 862       string s = _config->FindDir("Dir::State::lists") + "partial/";
 863       s.append(URItoFileName(RealURI));
 864       unlink(s.c_str());
 865    }
 866
 867    Item::Failed(Message,Cnf);
 868 }
 869                                                                         /*}}}*/
 870 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
 871 // ---------------------------------------------------------------------
 872 /* This goes through a number of states.. On the initial fetch the
 873    method could possibly return an alternate filename which points
 874    to the uncompressed version of the file. If this is so the file
 875    is copied into the partial directory. In all other cases the file
 876    is decompressed with a gzip uri. */
 877 void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash,
 878                        pkgAcquire::MethodConfig *Cfg)
 879 {
 880    Item::Done(Message,Size,Hash,Cfg);
 881
 882    if (Decompression == true)
 883    {
 884       if (_config->FindB("Debug::pkgAcquire::Auth", false))
 885       {
 886          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
 887          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
 888       }
 889
 890       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
 891       {
 892          Status = StatAuthError;
 893          ErrorText = _("Hash Sum mismatch");
 894          Rename(DestFile,DestFile + ".FAILED");
 895          ReportMirrorFailure("HashChecksumFailure");
 896          return;
 897       }
 898       // Done, move it into position
 899       string FinalFile = _config->FindDir("Dir::State::lists");
 900       FinalFile += URItoFileName(RealURI);
 901       Rename(DestFile,FinalFile);
 902       chmod(FinalFile.c_str(),0644);
 903
 904       /* We restore the original name to DestFile so that the clean operation
 905          will work OK */
 906       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 907       DestFile += URItoFileName(RealURI);
 908
 909       // Remove the compressed version.
 910       if (Erase == true)
 911          unlink(DestFile.c_str());
 912       return;
 913    }
 914
 915    Erase = false;
 916    Complete = true;
 917
 918    // Handle the unzipd case
 919    string FileName = LookupTag(Message,"Alt-Filename");
 920    if (FileName.empty() == false)
 921    {
 922       // The files timestamp matches
 923       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
 924          return;
 925       Decompression = true;
 926       Local = true;
 927       DestFile += ".decomp";
 928       Desc.URI = "copy:" + FileName;
 929       QueueURI(Desc);
 930       Mode = "copy";
 931       return;
 932    }
 933
 934    FileName = LookupTag(Message,"Filename");
 935    if (FileName.empty() == true)
 936    {
 937       Status = StatError;
 938       ErrorText = "Method gave a blank filename";
 939    }
 940
 941    std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
 942
 943    // The files timestamp matches
 944    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) {
 945        if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
 946           // Update DestFile for .gz suffix so that the clean operation keeps it
 947           DestFile += ".gz";
 948       return;
 949     }
 950
 951    if (FileName == DestFile)
 952       Erase = true;
 953    else
 954       Local = true;
 955
 956    string decompProg;
 957
 958    // If we enable compressed indexes and already have gzip, keep it
 959    if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) {
 960       string FinalFile = _config->FindDir("Dir::State::lists");
 961       FinalFile += URItoFileName(RealURI) + ".gz";
 962       Rename(DestFile,FinalFile);
 963       chmod(FinalFile.c_str(),0644);
 964
 965       // Update DestFile for .gz suffix so that the clean operation keeps it
 966       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 967       DestFile += URItoFileName(RealURI) + ".gz";
 968       return;
 969     }
 970
 971    // get the binary name for your used compression type
 972    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
 973    if(decompProg.empty() == false);
 974    else if(compExt == "uncompressed")
 975       decompProg = "copy";
 976    else {
 977       _error->Error("Unsupported extension: %s", compExt.c_str());
 978       return;
 979    }
 980
 981    Decompression = true;
 982    DestFile += ".decomp";
 983    Desc.URI = decompProg + ":" + FileName;
 984    QueueURI(Desc);
 985    Mode = decompProg.c_str();
 986 }
 987                                                                         /*}}}*/
 988 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
 989 // ---------------------------------------------------------------------
 990 /* The Translation file is added to the queue */
 991 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
 992                             string URI,string URIDesc,string ShortDesc)
 993   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
 994 {
 995 }
 996 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const *Target,
 997                          HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
 998   : pkgAcqIndex(Owner, Target, ExpectedHash, MetaIndexParser)
 999 {
1000 }
1001                                                                         /*}}}*/
1002 // AcqIndexTrans::Custom600Headers - Insert custom request headers      /*{{{*/
1003 // ---------------------------------------------------------------------
1004 string pkgAcqIndexTrans::Custom600Headers()
1005 {
1006    string Final = _config->FindDir("Dir::State::lists");
1007    Final += URItoFileName(RealURI);
1008
1009    struct stat Buf;
1010    if (stat(Final.c_str(),&Buf) != 0)
1011       return "\nFail-Ignore: true\nIndex-File: true";
1012    return "\nFail-Ignore: true\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1013 }
1014                                                                         /*}}}*/
1015 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
1016 // ---------------------------------------------------------------------
1017 /* */
1018 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1019 {
1020    size_t const nextExt = CompressionExtension.find(' ');
1021    if (nextExt != std::string::npos)
1022    {
1023       CompressionExtension = CompressionExtension.substr(nextExt+1);
1024       Init(RealURI, Desc.Description, Desc.ShortDesc);
1025       Status = StatIdle;
1026       return;
1027    }
1028
1029    if (Cnf->LocalOnly == true ||
1030        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1031    {
1032       // Ignore this
1033       Status = StatDone;
1034       Complete = false;
1035       Dequeue();
1036       return;
1037    }
1038
1039    Item::Failed(Message,Cnf);
1040 }
1041                                                                         /*}}}*/
1042 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
1043                              string URI,string URIDesc,string ShortDesc,
1044                              string MetaIndexURI, string MetaIndexURIDesc,
1045                              string MetaIndexShortDesc,
1046                              const vector<IndexTarget*>* IndexTargets,
1047                              indexRecords* MetaIndexParser) :
1048    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
1049    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1050    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
1051 {
1052    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1053    DestFile += URItoFileName(URI);
1054
1055    // remove any partial downloaded sig-file in partial/.
1056    // it may confuse proxies and is too small to warrant a
1057    // partial download anyway
1058    unlink(DestFile.c_str());
1059
1060    // Create the item
1061    Desc.Description = URIDesc;
1062    Desc.Owner = this;
1063    Desc.ShortDesc = ShortDesc;
1064    Desc.URI = URI;
1065
1066    string Final = _config->FindDir("Dir::State::lists");
1067    Final += URItoFileName(RealURI);
1068    struct stat Buf;
1069    if (stat(Final.c_str(),&Buf) == 0)
1070    {
1071       // File was already in place.  It needs to be re-downloaded/verified
1072       // because Release might have changed, we do give it a differnt
1073       // name than DestFile because otherwise the http method will
1074       // send If-Range requests and there are too many broken servers
1075       // out there that do not understand them
1076       LastGoodSig = DestFile+".reverify";
1077       Rename(Final,LastGoodSig);
1078    }
1079
1080    QueueURI(Desc);
1081 }
1082                                                                         /*}}}*/
1083 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
1084 // ---------------------------------------------------------------------
1085 /* The only header we use is the last-modified header. */
1086 string pkgAcqMetaSig::Custom600Headers()
1087 {
1088    struct stat Buf;
1089    if (stat(LastGoodSig.c_str(),&Buf) != 0)
1090       return "\nIndex-File: true";
1091
1092    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1093 }
1094
1095 void pkgAcqMetaSig::Done(string Message,unsigned long Size,string MD5,
1096                          pkgAcquire::MethodConfig *Cfg)
1097 {
1098    Item::Done(Message,Size,MD5,Cfg);
1099
1100    string FileName = LookupTag(Message,"Filename");
1101    if (FileName.empty() == true)
1102    {
1103       Status = StatError;
1104       ErrorText = "Method gave a blank filename";
1105       return;
1106    }
1107
1108    if (FileName != DestFile)
1109    {
1110       // We have to copy it into place
1111       Local = true;
1112       Desc.URI = "copy:" + FileName;
1113       QueueURI(Desc);
1114       return;
1115    }
1116
1117    Complete = true;
1118
1119    // put the last known good file back on i-m-s hit (it will
1120    // be re-verified again)
1121    // Else do nothing, we have the new file in DestFile then
1122    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1123       Rename(LastGoodSig, DestFile);
1124
1125    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
1126    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc,
1127                        MetaIndexShortDesc,  DestFile, IndexTargets,
1128                        MetaIndexParser);
1129
1130 }
1131                                                                         /*}}}*/
1132 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
1133 {
1134    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1135
1136    // if we get a network error we fail gracefully
1137    if(Status == StatTransientNetworkError)
1138    {
1139       Item::Failed(Message,Cnf);
1140       // move the sigfile back on transient network failures
1141       if(FileExists(LastGoodSig))
1142          Rename(LastGoodSig,Final);
1143
1144       // set the status back to , Item::Failed likes to reset it
1145       Status = pkgAcquire::Item::StatTransientNetworkError;
1146       return;
1147    }
1148
1149    // Delete any existing sigfile when the acquire failed
1150    unlink(Final.c_str());
1151
1152    // queue a pkgAcqMetaIndex with no sigfile
1153    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1154                        "", IndexTargets, MetaIndexParser);
1155
1156    if (Cnf->LocalOnly == true ||
1157        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1158    {
1159       // Ignore this
1160       Status = StatDone;
1161       Complete = false;
1162       Dequeue();
1163       return;
1164    }
1165
1166    Item::Failed(Message,Cnf);
1167 }
1168                                                                         /*}}}*/
1169 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
1170                                  string URI,string URIDesc,string ShortDesc,
1171                                  string SigFile,
1172                                  const vector<struct IndexTarget*>* IndexTargets,
1173                                  indexRecords* MetaIndexParser) :
1174    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
1175    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
1176 {
1177    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1178    DestFile += URItoFileName(URI);
1179
1180    // Create the item
1181    Desc.Description = URIDesc;
1182    Desc.Owner = this;
1183    Desc.ShortDesc = ShortDesc;
1184    Desc.URI = URI;
1185
1186    QueueURI(Desc);
1187 }
1188                                                                         /*}}}*/
1189 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
1190 // ---------------------------------------------------------------------
1191 /* The only header we use is the last-modified header. */
1192 string pkgAcqMetaIndex::Custom600Headers()
1193 {
1194    string Final = _config->FindDir("Dir::State::lists");
1195    Final += URItoFileName(RealURI);
1196
1197    struct stat Buf;
1198    if (stat(Final.c_str(),&Buf) != 0)
1199       return "\nIndex-File: true";
1200
1201    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1202 }
1203                                                                         /*}}}*/
1204 void pkgAcqMetaIndex::Done(string Message,unsigned long Size,string Hash,       /*{{{*/
1205                            pkgAcquire::MethodConfig *Cfg)
1206 {
1207    Item::Done(Message,Size,Hash,Cfg);
1208
1209    // MetaIndexes are done in two passes: one to download the
1210    // metaindex with an appropriate method, and a second to verify it
1211    // with the gpgv method
1212
1213    if (AuthPass == true)
1214    {
1215       AuthDone(Message);
1216
1217       // all cool, move Release file into place
1218       Complete = true;
1219    }
1220    else
1221    {
1222       RetrievalDone(Message);
1223       if (!Complete)
1224          // Still more retrieving to do
1225          return;
1226
1227       if (SigFile == "")
1228       {
1229          // There was no signature file, so we are finished.  Download
1230          // the indexes without verification.
1231          QueueIndexes(false);
1232       }
1233       else
1234       {
1235          // There was a signature file, so pass it to gpgv for
1236          // verification
1237
1238          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1239             std::cerr << "Metaindex acquired, queueing gpg verification ("
1240                       << SigFile << "," << DestFile << ")\n";
1241          AuthPass = true;
1242          Desc.URI = "gpgv:" + SigFile;
1243          QueueURI(Desc);
1244          Mode = "gpgv";
1245          return;
1246       }
1247    }
1248
1249    if (Complete == true)
1250    {
1251       string FinalFile = _config->FindDir("Dir::State::lists");
1252       FinalFile += URItoFileName(RealURI);
1253       if (SigFile == DestFile)
1254          SigFile = FinalFile;
1255       Rename(DestFile,FinalFile);
1256       chmod(FinalFile.c_str(),0644);
1257       DestFile = FinalFile;
1258    }
1259 }
1260                                                                         /*}}}*/
1261 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
1262 {
1263    // We have just finished downloading a Release file (it is not
1264    // verified yet)
1265
1266    string FileName = LookupTag(Message,"Filename");
1267    if (FileName.empty() == true)
1268    {
1269       Status = StatError;
1270       ErrorText = "Method gave a blank filename";
1271       return;
1272    }
1273
1274    if (FileName != DestFile)
1275    {
1276       Local = true;
1277       Desc.URI = "copy:" + FileName;
1278       QueueURI(Desc);
1279       return;
1280    }
1281
1282    // make sure to verify against the right file on I-M-S hit
1283    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1284    if(IMSHit)
1285    {
1286       string FinalFile = _config->FindDir("Dir::State::lists");
1287       FinalFile += URItoFileName(RealURI);
1288       if (SigFile == DestFile)
1289          SigFile = FinalFile;
1290       DestFile = FinalFile;
1291    }
1292    Complete = true;
1293 }
1294                                                                         /*}}}*/
1295 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1296 {
1297    // At this point, the gpgv method has succeeded, so there is a
1298    // valid signature from a key in the trusted keyring.  We
1299    // perform additional verification of its contents, and use them
1300    // to verify the indexes we are about to download
1301
1302    if (!MetaIndexParser->Load(DestFile))
1303    {
1304       Status = StatAuthError;
1305       ErrorText = MetaIndexParser->ErrorText;
1306       return;
1307    }
1308
1309    if (!VerifyVendor(Message))
1310    {
1311       return;
1312    }
1313
1314    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1315       std::cerr << "Signature verification succeeded: "
1316                 << DestFile << std::endl;
1317
1318    // Download further indexes with verification
1319    QueueIndexes(true);
1320
1321    // is it a clearsigned MetaIndex file?
1322    if (DestFile == SigFile)
1323       return;
1324
1325    // Done, move signature file into position
1326    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1327       URItoFileName(RealURI) + ".gpg";
1328    Rename(SigFile,VerifiedSigFile);
1329    chmod(VerifiedSigFile.c_str(),0644);
1330 }
1331                                                                         /*}}}*/
1332 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1333 {
1334    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1335         Target != IndexTargets->end();
1336         Target++)
1337    {
1338       HashString ExpectedIndexHash;
1339       if (verify)
1340       {
1341          const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1342          if (Record == NULL)
1343          {
1344             if ((*Target)->IsOptional() == false)
1345             {
1346                Status = StatAuthError;
1347                strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), (*Target)->MetaKey.c_str());
1348                return;
1349             }
1350          }
1351          else
1352          {
1353             ExpectedIndexHash = Record->Hash;
1354             if (_config->FindB("Debug::pkgAcquire::Auth", false))
1355             {
1356                std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1357                std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1358             }
1359             if (ExpectedIndexHash.empty() == true && (*Target)->IsOptional() == false)
1360             {
1361                Status = StatAuthError;
1362                strprintf(ErrorText, _("Unable to find hash sum for '%s' in Release file"), (*Target)->MetaKey.c_str());
1363                return;
1364             }
1365          }
1366       }
1367
1368       if ((*Target)->IsOptional() == true)
1369       {
1370          if ((*Target)->IsSubIndex() == true)
1371             new pkgAcqSubIndex(Owner, (*Target)->URI, (*Target)->Description,
1372                                 (*Target)->ShortDesc, ExpectedIndexHash);
1373          else
1374             new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1375          continue;
1376       }
1377
1378       /* Queue Packages file (either diff or full packages files, depending
1379          on the users option) - we also check if the PDiff Index file is listed
1380          in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
1381          instead, but passing the required info to it is to much hassle */
1382       if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
1383           MetaIndexParser->Exists(string((*Target)->MetaKey).append(".diff/Index")) == true))
1384          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1385                              (*Target)->ShortDesc, ExpectedIndexHash);
1386       else
1387          new pkgAcqIndex(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1388    }
1389 }
1390                                                                         /*}}}*/
1391 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1392 {
1393 //    // Maybe this should be made available from above so we don't have
1394 //    // to read and parse it every time?
1395 //    pkgVendorList List;
1396 //    List.ReadMainList();
1397
1398 //    const Vendor* Vndr = NULL;
1399 //    for (std::vector<string>::const_iterator I = GPGVOutput.begin(); I != GPGVOutput.end(); I++)
1400 //    {
1401 //       string::size_type pos = (*I).find("VALIDSIG ");
1402 //       if (_config->FindB("Debug::Vendor", false))
1403 //          std::cerr << "Looking for VALIDSIG in \"" << (*I) << "\": pos " << pos
1404 //                    << std::endl;
1405 //       if (pos != std::string::npos)
1406 //       {
1407 //          string Fingerprint = (*I).substr(pos+sizeof("VALIDSIG"));
1408 //          if (_config->FindB("Debug::Vendor", false))
1409 //             std::cerr << "Looking for \"" << Fingerprint << "\" in vendor..." <<
1410 //                std::endl;
1411 //          Vndr = List.FindVendor(Fingerprint) != "";
1412 //          if (Vndr != NULL);
1413 //          break;
1414 //       }
1415 //    }
1416    string::size_type pos;
1417
1418    // check for missing sigs (that where not fatal because otherwise we had
1419    // bombed earlier)
1420    string missingkeys;
1421    string msg = _("There is no public key available for the "
1422                   "following key IDs:\n");
1423    pos = Message.find("NO_PUBKEY ");
1424    if (pos != std::string::npos)
1425    {
1426       string::size_type start = pos+strlen("NO_PUBKEY ");
1427       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1428       missingkeys += (Fingerprint);
1429    }
1430    if(!missingkeys.empty())
1431       _error->Warning("%s", string(msg+missingkeys).c_str());
1432
1433    string Transformed = MetaIndexParser->GetExpectedDist();
1434
1435    if (Transformed == "../project/experimental")
1436    {
1437       Transformed = "experimental";
1438    }
1439
1440    pos = Transformed.rfind('/');
1441    if (pos != string::npos)
1442    {
1443       Transformed = Transformed.substr(0, pos);
1444    }
1445
1446    if (Transformed == ".")
1447    {
1448       Transformed = "";
1449    }
1450
1451    if (_config->FindB("Acquire::Check-Valid-Until", true) == true &&
1452        MetaIndexParser->GetValidUntil() > 0) {
1453       time_t const invalid_since = time(NULL) - MetaIndexParser->GetValidUntil();
1454       if (invalid_since > 0)
1455          // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1456          // the time since then the file is invalid - formated in the same way as in
1457          // the download progress display (e.g. 7d 3h 42min 1s)
1458          return _error->Error(_("Release file expired, ignoring %s (invalid since %s)"),
1459                               RealURI.c_str(), TimeToStr(invalid_since).c_str());
1460    }
1461
1462    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1463    {
1464       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1465       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1466       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1467    }
1468
1469    if (MetaIndexParser->CheckDist(Transformed) == false)
1470    {
1471       // This might become fatal one day
1472 //       Status = StatAuthError;
1473 //       ErrorText = "Conflicting distribution; expected "
1474 //          + MetaIndexParser->GetExpectedDist() + " but got "
1475 //          + MetaIndexParser->GetDist();
1476 //       return false;
1477       if (!Transformed.empty())
1478       {
1479          _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1480                          Desc.Description.c_str(),
1481                          Transformed.c_str(),
1482                          MetaIndexParser->GetDist().c_str());
1483       }
1484    }
1485
1486    return true;
1487 }
1488                                                                         /*}}}*/
1489 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1490 // ---------------------------------------------------------------------
1491 /* */
1492 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1493 {
1494    if (AuthPass == true)
1495    {
1496       // gpgv method failed, if we have a good signature
1497       string LastGoodSigFile = _config->FindDir("Dir::State::lists");
1498       if (DestFile == SigFile)
1499          LastGoodSigFile.append(URItoFileName(RealURI));
1500       else
1501          LastGoodSigFile.append("partial/").append(URItoFileName(RealURI)).append(".gpg.reverify");
1502
1503       if(FileExists(LastGoodSigFile))
1504       {
1505          if (DestFile != SigFile)
1506          {
1507             string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1508                                         URItoFileName(RealURI) + ".gpg";
1509             Rename(LastGoodSigFile,VerifiedSigFile);
1510          }
1511          Status = StatTransientNetworkError;
1512          _error->Warning(_("A error occurred during the signature "
1513                            "verification. The repository is not updated "
1514                            "and the previous index files will be used. "
1515                            "GPG error: %s: %s\n"),
1516                          Desc.Description.c_str(),
1517                          LookupTag(Message,"Message").c_str());
1518          RunScripts("APT::Update::Auth-Failure");
1519          return;
1520       } else {
1521          _error->Warning(_("GPG error: %s: %s"),
1522                          Desc.Description.c_str(),
1523                          LookupTag(Message,"Message").c_str());
1524       }
1525       // gpgv method failed
1526       ReportMirrorFailure("GPGFailure");
1527    }
1528
1529    // No Release file was present, or verification failed, so fall
1530    // back to queueing Packages files without verification
1531    QueueIndexes(false);
1532 }
1533                                                                         /*}}}*/
1534 pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner,               /*{{{*/
1535                 string const &URI, string const &URIDesc, string const &ShortDesc,
1536                 string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc,
1537                 string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc,
1538                 const vector<struct IndexTarget*>* IndexTargets,
1539                 indexRecords* MetaIndexParser) :
1540         pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, "", IndexTargets, MetaIndexParser),
1541         MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1542         MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc)
1543 {
1544    SigFile = DestFile;
1545 }
1546                                                                         /*}}}*/
1547 // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/
1548 // ---------------------------------------------------------------------
1549 // FIXME: this can go away once the InRelease file is used widely
1550 string pkgAcqMetaClearSig::Custom600Headers()
1551 {
1552    string Final = _config->FindDir("Dir::State::lists");
1553    Final += URItoFileName(RealURI);
1554
1555    struct stat Buf;
1556    if (stat(Final.c_str(),&Buf) != 0)
1557       return "\nIndex-File: true\nFail-Ignore: true\n";
1558
1559    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1560 }
1561                                                                         /*}}}*/
1562 void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/
1563 {
1564    if (AuthPass == false)
1565    {
1566       new pkgAcqMetaSig(Owner,
1567                         MetaSigURI, MetaSigURIDesc, MetaSigShortDesc,
1568                         MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1569                         IndexTargets, MetaIndexParser);
1570       if (Cnf->LocalOnly == true ||
1571           StringToBool(LookupTag(Message, "Transient-Failure"), false) == false)
1572          Dequeue();
1573    }
1574    else
1575       pkgAcqMetaIndex::Failed(Message, Cnf);
1576 }
1577                                                                         /*}}}*/
1578 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1579 // ---------------------------------------------------------------------
1580 /* This just sets up the initial fetch environment and queues the first
1581    possibilitiy */
1582 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1583                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1584                              string &StoreFilename) :
1585                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1586                StoreFilename(StoreFilename), Vf(Version.FileList()),
1587                Trusted(false)
1588 {
1589    Retries = _config->FindI("Acquire::Retries",0);
1590
1591    if (Version.Arch() == 0)
1592    {
1593       _error->Error(_("I wasn't able to locate a file for the %s package. "
1594                       "This might mean you need to manually fix this package. "
1595                       "(due to missing arch)"),
1596                     Version.ParentPkg().Name());
1597       return;
1598    }
1599
1600    /* We need to find a filename to determine the extension. We make the
1601       assumption here that all the available sources for this version share
1602       the same extension.. */
1603    // Skip not source sources, they do not have file fields.
1604    for (; Vf.end() == false; Vf++)
1605    {
1606       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1607          continue;
1608       break;
1609    }
1610
1611    // Does not really matter here.. we are going to fail out below
1612    if (Vf.end() != true)
1613    {
1614       // If this fails to get a file name we will bomb out below.
1615       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1616       if (_error->PendingError() == true)
1617          return;
1618
1619       // Generate the final file name as: package_version_arch.foo
1620       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1621                       QuoteString(Version.VerStr(),"_:") + '_' +
1622                       QuoteString(Version.Arch(),"_:.") +
1623                       "." + flExtension(Parse.FileName());
1624    }
1625
1626    // check if we have one trusted source for the package. if so, switch
1627    // to "TrustedOnly" mode
1628    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; i++)
1629    {
1630       pkgIndexFile *Index;
1631       if (Sources->FindIndex(i.File(),Index) == false)
1632          continue;
1633       if (_config->FindB("Debug::pkgAcquire::Auth", false))
1634       {
1635          std::cerr << "Checking index: " << Index->Describe()
1636                    << "(Trusted=" << Index->IsTrusted() << ")\n";
1637       }
1638       if (Index->IsTrusted()) {
1639          Trusted = true;
1640          break;
1641       }
1642    }
1643
1644    // "allow-unauthenticated" restores apts old fetching behaviour
1645    // that means that e.g. unauthenticated file:// uris are higher
1646    // priority than authenticated http:// uris
1647    if (_config->FindB("APT::Get::AllowUnauthenticated",false) == true)
1648       Trusted = false;
1649
1650    // Select a source
1651    if (QueueNext() == false && _error->PendingError() == false)
1652       _error->Error(_("I wasn't able to locate file for the %s package. "
1653                     "This might mean you need to manually fix this package."),
1654                     Version.ParentPkg().Name());
1655 }
1656                                                                         /*}}}*/
1657 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1658 // ---------------------------------------------------------------------
1659 /* This queues the next available file version for download. It checks if
1660    the archive is already available in the cache and stashs the MD5 for
1661    checking later. */
1662 bool pkgAcqArchive::QueueNext()
1663 {
1664    string const ForceHash = _config->Find("Acquire::ForceHash");
1665    for (; Vf.end() == false; Vf++)
1666    {
1667       // Ignore not source sources
1668       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1669          continue;
1670
1671       // Try to cross match against the source list
1672       pkgIndexFile *Index;
1673       if (Sources->FindIndex(Vf.File(),Index) == false)
1674             continue;
1675
1676       // only try to get a trusted package from another source if that source
1677       // is also trusted
1678       if(Trusted && !Index->IsTrusted())
1679          continue;
1680
1681       // Grab the text package record
1682       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1683       if (_error->PendingError() == true)
1684          return false;
1685
1686       string PkgFile = Parse.FileName();
1687       if (ForceHash.empty() == false)
1688       {
1689          if(stringcasecmp(ForceHash, "sha256") == 0)
1690             ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
1691          else if (stringcasecmp(ForceHash, "sha1") == 0)
1692             ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
1693          else
1694             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1695       }
1696       else
1697       {
1698          string Hash;
1699          if ((Hash = Parse.SHA256Hash()).empty() == false)
1700             ExpectedHash = HashString("SHA256", Hash);
1701          else if ((Hash = Parse.SHA1Hash()).empty() == false)
1702             ExpectedHash = HashString("SHA1", Hash);
1703          else
1704             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1705       }
1706       if (PkgFile.empty() == true)
1707          return _error->Error(_("The package index files are corrupted. No Filename: "
1708                               "field for package %s."),
1709                               Version.ParentPkg().Name());
1710
1711       Desc.URI = Index->ArchiveURI(PkgFile);
1712       Desc.Description = Index->ArchiveInfo(Version);
1713       Desc.Owner = this;
1714       Desc.ShortDesc = Version.ParentPkg().Name();
1715
1716       // See if we already have the file. (Legacy filenames)
1717       FileSize = Version->Size;
1718       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
1719       struct stat Buf;
1720       if (stat(FinalFile.c_str(),&Buf) == 0)
1721       {
1722          // Make sure the size matches
1723          if ((unsigned)Buf.st_size == Version->Size)
1724          {
1725             Complete = true;
1726             Local = true;
1727             Status = StatDone;
1728             StoreFilename = DestFile = FinalFile;
1729             return true;
1730          }
1731
1732          /* Hmm, we have a file and its size does not match, this means it is
1733             an old style mismatched arch */
1734          unlink(FinalFile.c_str());
1735       }
1736
1737       // Check it again using the new style output filenames
1738       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
1739       if (stat(FinalFile.c_str(),&Buf) == 0)
1740       {
1741          // Make sure the size matches
1742          if ((unsigned)Buf.st_size == Version->Size)
1743          {
1744             Complete = true;
1745             Local = true;
1746             Status = StatDone;
1747             StoreFilename = DestFile = FinalFile;
1748             return true;
1749          }
1750
1751          /* Hmm, we have a file and its size does not match, this shouldnt
1752             happen.. */
1753          unlink(FinalFile.c_str());
1754       }
1755
1756       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
1757
1758       // Check the destination file
1759       if (stat(DestFile.c_str(),&Buf) == 0)
1760       {
1761          // Hmm, the partial file is too big, erase it
1762          if ((unsigned)Buf.st_size > Version->Size)
1763             unlink(DestFile.c_str());
1764          else
1765             PartialSize = Buf.st_size;
1766       }
1767
1768       // Create the item
1769       Local = false;
1770       Desc.URI = Index->ArchiveURI(PkgFile);
1771       Desc.Description = Index->ArchiveInfo(Version);
1772       Desc.Owner = this;
1773       Desc.ShortDesc = Version.ParentPkg().Name();
1774       QueueURI(Desc);
1775
1776       Vf++;
1777       return true;
1778    }
1779    return false;
1780 }
1781                                                                         /*}}}*/
1782 // AcqArchive::Done - Finished fetching                                 /*{{{*/
1783 // ---------------------------------------------------------------------
1784 /* */
1785 void pkgAcqArchive::Done(string Message,unsigned long Size,string CalcHash,
1786                          pkgAcquire::MethodConfig *Cfg)
1787 {
1788    Item::Done(Message,Size,CalcHash,Cfg);
1789
1790    // Check the size
1791    if (Size != Version->Size)
1792    {
1793       Status = StatError;
1794       ErrorText = _("Size mismatch");
1795       return;
1796    }
1797
1798    // Check the hash
1799    if(ExpectedHash.toStr() != CalcHash)
1800    {
1801       Status = StatError;
1802       ErrorText = _("Hash Sum mismatch");
1803       if(FileExists(DestFile))
1804          Rename(DestFile,DestFile + ".FAILED");
1805       return;
1806    }
1807
1808    // Grab the output filename
1809    string FileName = LookupTag(Message,"Filename");
1810    if (FileName.empty() == true)
1811    {
1812       Status = StatError;
1813       ErrorText = "Method gave a blank filename";
1814       return;
1815    }
1816
1817    Complete = true;
1818
1819    // Reference filename
1820    if (FileName != DestFile)
1821    {
1822       StoreFilename = DestFile = FileName;
1823       Local = true;
1824       return;
1825    }
1826
1827    // Done, move it into position
1828    string FinalFile = _config->FindDir("Dir::Cache::Archives");
1829    FinalFile += flNotDir(StoreFilename);
1830    Rename(DestFile,FinalFile);
1831
1832    StoreFilename = DestFile = FinalFile;
1833    Complete = true;
1834 }
1835                                                                         /*}}}*/
1836 // AcqArchive::Failed - Failure handler                                 /*{{{*/
1837 // ---------------------------------------------------------------------
1838 /* Here we try other sources */
1839 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1840 {
1841    ErrorText = LookupTag(Message,"Message");
1842
1843    /* We don't really want to retry on failed media swaps, this prevents
1844       that. An interesting observation is that permanent failures are not
1845       recorded. */
1846    if (Cnf->Removable == true &&
1847        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1848    {
1849       // Vf = Version.FileList();
1850       while (Vf.end() == false) Vf++;
1851       StoreFilename = string();
1852       Item::Failed(Message,Cnf);
1853       return;
1854    }
1855
1856    if (QueueNext() == false)
1857    {
1858       // This is the retry counter
1859       if (Retries != 0 &&
1860           Cnf->LocalOnly == false &&
1861           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1862       {
1863          Retries--;
1864          Vf = Version.FileList();
1865          if (QueueNext() == true)
1866             return;
1867       }
1868
1869       StoreFilename = string();
1870       Item::Failed(Message,Cnf);
1871    }
1872 }
1873                                                                         /*}}}*/
1874 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
1875 // ---------------------------------------------------------------------
1876 bool pkgAcqArchive::IsTrusted()
1877 {
1878    return Trusted;
1879 }
1880                                                                         /*}}}*/
1881 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
1882 // ---------------------------------------------------------------------
1883 /* */
1884 void pkgAcqArchive::Finished()
1885 {
1886    if (Status == pkgAcquire::Item::StatDone &&
1887        Complete == true)
1888       return;
1889    StoreFilename = string();
1890 }
1891                                                                         /*}}}*/
1892 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
1893 // ---------------------------------------------------------------------
1894 /* The file is added to the queue */
1895 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
1896                        unsigned long Size,string Dsc,string ShortDesc,
1897                        const string &DestDir, const string &DestFilename,
1898                        bool IsIndexFile) :
1899                        Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile)
1900 {
1901    Retries = _config->FindI("Acquire::Retries",0);
1902
1903    if(!DestFilename.empty())
1904       DestFile = DestFilename;
1905    else if(!DestDir.empty())
1906       DestFile = DestDir + "/" + flNotDir(URI);
1907    else
1908       DestFile = flNotDir(URI);
1909
1910    // Create the item
1911    Desc.URI = URI;
1912    Desc.Description = Dsc;
1913    Desc.Owner = this;
1914
1915    // Set the short description to the archive component
1916    Desc.ShortDesc = ShortDesc;
1917
1918    // Get the transfer sizes
1919    FileSize = Size;
1920    struct stat Buf;
1921    if (stat(DestFile.c_str(),&Buf) == 0)
1922    {
1923       // Hmm, the partial file is too big, erase it
1924       if ((unsigned)Buf.st_size > Size)
1925          unlink(DestFile.c_str());
1926       else
1927          PartialSize = Buf.st_size;
1928    }
1929
1930    QueueURI(Desc);
1931 }
1932                                                                         /*}}}*/
1933 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
1934 // ---------------------------------------------------------------------
1935 /* */
1936 void pkgAcqFile::Done(string Message,unsigned long Size,string CalcHash,
1937                       pkgAcquire::MethodConfig *Cnf)
1938 {
1939    Item::Done(Message,Size,CalcHash,Cnf);
1940
1941    // Check the hash
1942    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
1943    {
1944       Status = StatError;
1945       ErrorText = _("Hash Sum mismatch");
1946       Rename(DestFile,DestFile + ".FAILED");
1947       return;
1948    }
1949
1950    string FileName = LookupTag(Message,"Filename");
1951    if (FileName.empty() == true)
1952    {
1953       Status = StatError;
1954       ErrorText = "Method gave a blank filename";
1955       return;
1956    }
1957
1958    Complete = true;
1959
1960    // The files timestamp matches
1961    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1962       return;
1963
1964    // We have to copy it into place
1965    if (FileName != DestFile)
1966    {
1967       Local = true;
1968       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
1969           Cnf->Removable == true)
1970       {
1971          Desc.URI = "copy:" + FileName;
1972          QueueURI(Desc);
1973          return;
1974       }
1975
1976       // Erase the file if it is a symlink so we can overwrite it
1977       struct stat St;
1978       if (lstat(DestFile.c_str(),&St) == 0)
1979       {
1980          if (S_ISLNK(St.st_mode) != 0)
1981             unlink(DestFile.c_str());
1982       }
1983
1984       // Symlink the file
1985       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
1986       {
1987          ErrorText = "Link to " + DestFile + " failure ";
1988          Status = StatError;
1989          Complete = false;
1990       }
1991    }
1992 }
1993                                                                         /*}}}*/
1994 // AcqFile::Failed - Failure handler                                    /*{{{*/
1995 // ---------------------------------------------------------------------
1996 /* Here we try other sources */
1997 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1998 {
1999    ErrorText = LookupTag(Message,"Message");
2000
2001    // This is the retry counter
2002    if (Retries != 0 &&
2003        Cnf->LocalOnly == false &&
2004        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2005    {
2006       Retries--;
2007       QueueURI(Desc);
2008       return;
2009    }
2010
2011    Item::Failed(Message,Cnf);
2012 }
2013                                                                         /*}}}*/
2014 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
2015 // ---------------------------------------------------------------------
2016 /* The only header we use is the last-modified header. */
2017 string pkgAcqFile::Custom600Headers()
2018 {
2019    if (IsIndexFile)
2020       return "\nIndex-File: true";
2021    return "";
2022 }
2023                                                                         /*}}}*/
2024 bool IndexTarget::IsOptional() const {
2025    if (strncmp(ShortDesc.c_str(), "Translation", 11) != 0)
2026       return false;
2027    return true;
2028 }
2029 bool IndexTarget::IsSubIndex() const {
2030    if (ShortDesc != "TranslationIndex")
2031       return false;
2032    return true;
2033 }