apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <apt-pkg/acquire-item.h>
  17 #include <apt-pkg/configuration.h>
  18 #include <apt-pkg/sourcelist.h>
  19 #include <apt-pkg/vendorlist.h>
  20 #include <apt-pkg/error.h>
  21 #include <apt-pkg/strutl.h>
  22 #include <apt-pkg/fileutl.h>
  23 #include <apt-pkg/md5.h>
  24 #include <apt-pkg/sha1.h>
  25 #include <apt-pkg/tagfile.h>
  26
  27 #include <apti18n.h>
  28
  29 #include <sys/stat.h>
  30 #include <unistd.h>
  31 #include <errno.h>
  32 #include <string>
  33 #include <sstream>
  34 #include <stdio.h>
  35                                                                         /*}}}*/
  36
  37 using namespace std;
  38
  39 // Acquire::Item::Item - Constructor                                    /*{{{*/
  40 // ---------------------------------------------------------------------
  41 /* */
  42 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  43                        PartialSize(0), Mode(0), ID(0), Complete(false),
  44                        Local(false), QueueCounter(0)
  45 {
  46    Owner->Add(this);
  47    Status = StatIdle;
  48 }
  49                                                                         /*}}}*/
  50 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  51 // ---------------------------------------------------------------------
  52 /* */
  53 pkgAcquire::Item::~Item()
  54 {
  55    Owner->Remove(this);
  56 }
  57                                                                         /*}}}*/
  58 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  59 // ---------------------------------------------------------------------
  60 /* We return to an idle state if there are still other queues that could
  61    fetch this object */
  62 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  63 {
  64    Status = StatIdle;
  65    ErrorText = LookupTag(Message,"Message");
  66    UsedMirror =  LookupTag(Message,"UsedMirror");
  67    if (QueueCounter <= 1)
  68    {
  69       /* This indicates that the file is not available right now but might
  70          be sometime later. If we do a retry cycle then this should be
  71          retried [CDROMs] */
  72       if (Cnf->LocalOnly == true &&
  73           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  74       {
  75          Status = StatIdle;
  76          Dequeue();
  77          return;
  78       }
  79
  80       Status = StatError;
  81       Dequeue();
  82    }
  83
  84    // report mirror failure back to LP if we actually use a mirror
  85    string FailReason = LookupTag(Message, "FailReason");
  86    if(FailReason.size() != 0)
  87       ReportMirrorFailure(FailReason);
  88    else
  89       ReportMirrorFailure(ErrorText);
  90 }
  91                                                                         /*}}}*/
  92 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
  93 // ---------------------------------------------------------------------
  94 /* Stash status and the file size. Note that setting Complete means
  95    sub-phases of the acquire process such as decompresion are operating */
  96 void pkgAcquire::Item::Start(string /*Message*/,unsigned long Size)
  97 {
  98    Status = StatFetching;
  99    if (FileSize == 0 && Complete == false)
 100       FileSize = Size;
 101 }
 102                                                                         /*}}}*/
 103 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
 104 // ---------------------------------------------------------------------
 105 /* */
 106 void pkgAcquire::Item::Done(string Message,unsigned long Size,string Hash,
 107                             pkgAcquire::MethodConfig *Cnf)
 108 {
 109    // We just downloaded something..
 110    string FileName = LookupTag(Message,"Filename");
 111    UsedMirror =  LookupTag(Message,"UsedMirror");
 112    if (Complete == false && !Local && FileName == DestFile)
 113    {
 114       if (Owner->Log != 0)
 115          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 116    }
 117
 118    if (FileSize == 0)
 119       FileSize= Size;
 120    Status = StatDone;
 121    ErrorText = string();
 122    Owner->Dequeue(this);
 123 }
 124                                                                         /*}}}*/
 125 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 126 // ---------------------------------------------------------------------
 127 /* This helper function is used by alot of item methods as thier final
 128    step */
 129 void pkgAcquire::Item::Rename(string From,string To)
 130 {
 131    if (rename(From.c_str(),To.c_str()) != 0)
 132    {
 133       char S[300];
 134       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 135               From.c_str(),To.c_str());
 136       Status = StatError;
 137       ErrorText = S;
 138    }
 139 }
 140                                                                         /*}}}*/
 141
 142 void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
 143 {
 144    // we only act if a mirror was used at all
 145    if(UsedMirror.empty())
 146       return;
 147 #if 0
 148    std::cerr << "\nReportMirrorFailure: "
 149              << UsedMirror
 150              << " Uri: " << DescURI()
 151              << " FailCode: "
 152              << FailCode << std::endl;
 153 #endif
 154    const char *Args[40];
 155    unsigned int i = 0;
 156    string report = _config->Find("Methods::Mirror::ProblemReporting",
 157                                  "/usr/lib/apt/apt-report-mirror-failure");
 158    if(!FileExists(report))
 159       return;
 160    Args[i++] = report.c_str();
 161    Args[i++] = UsedMirror.c_str();
 162    Args[i++] = DescURI().c_str();
 163    Args[i++] = FailCode.c_str();
 164    Args[i++] = NULL;
 165    pid_t pid = ExecFork();
 166    if(pid < 0)
 167    {
 168       _error->Error("ReportMirrorFailure Fork failed");
 169       return;
 170    }
 171    else if(pid == 0)
 172    {
 173       execvp(Args[0], (char**)Args);
 174       std::cerr << "Could not exec " << Args[0] << std::endl;
 175       _exit(100);
 176    }
 177    if(!ExecWait(pid, "report-mirror-failure"))
 178    {
 179       _error->Warning("Couldn't report problem to '%s'",
 180                       _config->Find("Methods::Mirror::ProblemReporting").c_str());
 181    }
 182 }
 183
 184
 185
 186 // AcqDiffIndex::AcqDiffIndex - Constructor
 187 // ---------------------------------------------------------------------
 188 /* Get the DiffIndex file first and see if there are patches availabe
 189  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 190  * patches. If anything goes wrong in that process, it will fall back to
 191  * the original packages file
 192  */
 193 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 194                                  string URI,string URIDesc,string ShortDesc,
 195                                  HashString ExpectedHash)
 196    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 197      Description(URIDesc)
 198 {
 199
 200    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 201
 202    Desc.Description = URIDesc + "/DiffIndex";
 203    Desc.Owner = this;
 204    Desc.ShortDesc = ShortDesc;
 205    Desc.URI = URI + ".diff/Index";
 206
 207    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 208    DestFile += URItoFileName(URI) + string(".DiffIndex");
 209
 210    if(Debug)
 211       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 212
 213    // look for the current package file
 214    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 215    CurrentPackagesFile += URItoFileName(RealURI);
 216
 217    // FIXME: this file:/ check is a hack to prevent fetching
 218    //        from local sources. this is really silly, and
 219    //        should be fixed cleanly as soon as possible
 220    if(!FileExists(CurrentPackagesFile) ||
 221       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 222    {
 223       // we don't have a pkg file or we don't want to queue
 224       if(Debug)
 225          std::clog << "No index file, local or canceld by user" << std::endl;
 226       Failed("", NULL);
 227       return;
 228    }
 229
 230    if(Debug)
 231       std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): "
 232                 << CurrentPackagesFile << std::endl;
 233
 234    QueueURI(Desc);
 235
 236 }
 237
 238 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 239 // ---------------------------------------------------------------------
 240 /* The only header we use is the last-modified header. */
 241 string pkgAcqDiffIndex::Custom600Headers()
 242 {
 243    string Final = _config->FindDir("Dir::State::lists");
 244    Final += URItoFileName(RealURI) + string(".IndexDiff");
 245
 246    if(Debug)
 247       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 248
 249    struct stat Buf;
 250    if (stat(Final.c_str(),&Buf) != 0)
 251       return "\nIndex-File: true";
 252
 253    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 254 }
 255
 256
 257 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)
 258 {
 259    if(Debug)
 260       std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile
 261                 << std::endl;
 262
 263    pkgTagSection Tags;
 264    string ServerSha1;
 265    vector<DiffInfo> available_patches;
 266
 267    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 268    pkgTagFile TF(&Fd);
 269    if (_error->PendingError() == true)
 270       return false;
 271
 272    if(TF.Step(Tags) == true)
 273    {
 274       string local_sha1;
 275       bool found = false;
 276       DiffInfo d;
 277       string size;
 278
 279       string tmp = Tags.FindS("SHA1-Current");
 280       std::stringstream ss(tmp);
 281       ss >> ServerSha1;
 282
 283       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 284       SHA1Summation SHA1;
 285       SHA1.AddFD(fd.Fd(), fd.Size());
 286       local_sha1 = string(SHA1.Result());
 287
 288       if(local_sha1 == ServerSha1)
 289       {
 290          // we have the same sha1 as the server
 291          if(Debug)
 292             std::clog << "Package file is up-to-date" << std::endl;
 293          // set found to true, this will queue a pkgAcqIndexDiffs with
 294          // a empty availabe_patches
 295          found = true;
 296       }
 297       else
 298       {
 299          if(Debug)
 300             std::clog << "SHA1-Current: " << ServerSha1 << std::endl;
 301
 302          // check the historie and see what patches we need
 303          string history = Tags.FindS("SHA1-History");
 304          std::stringstream hist(history);
 305          while(hist >> d.sha1 >> size >> d.file)
 306          {
 307             d.size = atoi(size.c_str());
 308             // read until the first match is found
 309             if(d.sha1 == local_sha1)
 310                found=true;
 311             // from that point on, we probably need all diffs
 312             if(found)
 313             {
 314                if(Debug)
 315                   std::clog << "Need to get diff: " << d.file << std::endl;
 316                available_patches.push_back(d);
 317             }
 318          }
 319       }
 320
 321       // we have something, queue the next diff
 322       if(found)
 323       {
 324          // queue the diffs
 325          string::size_type last_space = Description.rfind(" ");
 326          if(last_space != string::npos)
 327             Description.erase(last_space, Description.size()-last_space);
 328          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 329                               ExpectedHash, available_patches);
 330          Complete = false;
 331          Status = StatDone;
 332          Dequeue();
 333          return true;
 334       }
 335    }
 336
 337    // Nothing found, report and return false
 338    // Failing here is ok, if we return false later, the full
 339    // IndexFile is queued
 340    if(Debug)
 341       std::clog << "Can't find a patch in the index file" << std::endl;
 342    return false;
 343 }
 344
 345 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 346 {
 347    if(Debug)
 348       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << std::endl
 349                 << "Falling back to normal index file aquire" << std::endl;
 350
 351    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 352                    ExpectedHash);
 353
 354    Complete = false;
 355    Status = StatDone;
 356    Dequeue();
 357 }
 358
 359 void pkgAcqDiffIndex::Done(string Message,unsigned long Size,string Md5Hash,
 360                            pkgAcquire::MethodConfig *Cnf)
 361 {
 362    if(Debug)
 363       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 364
 365    Item::Done(Message,Size,Md5Hash,Cnf);
 366
 367    string FinalFile;
 368    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 369
 370    // sucess in downloading the index
 371    // rename the index
 372    FinalFile += string(".IndexDiff");
 373    if(Debug)
 374       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 375                 << std::endl;
 376    Rename(DestFile,FinalFile);
 377    chmod(FinalFile.c_str(),0644);
 378    DestFile = FinalFile;
 379
 380    if(!ParseDiffIndex(DestFile))
 381       return Failed("", NULL);
 382
 383    Complete = true;
 384    Status = StatDone;
 385    Dequeue();
 386    return;
 387 }
 388
 389
 390
 391 // AcqIndexDiffs::AcqIndexDiffs - Constructor
 392 // ---------------------------------------------------------------------
 393 /* The package diff is added to the queue. one object is constructed
 394  * for each diff and the index
 395  */
 396 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 397                                    string URI,string URIDesc,string ShortDesc,
 398                                    HashString ExpectedMD5,
 399                                    vector<DiffInfo> diffs)
 400    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 401      available_patches(diffs)
 402 {
 403
 404    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 405    DestFile += URItoFileName(URI);
 406
 407    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 408
 409    Description = URIDesc;
 410    Desc.Owner = this;
 411    Desc.ShortDesc = ShortDesc;
 412
 413    if(available_patches.size() == 0)
 414    {
 415       // we are done (yeah!)
 416       Finish(true);
 417    }
 418    else
 419    {
 420       // get the next diff
 421       State = StateFetchDiff;
 422       QueueNextDiff();
 423    }
 424 }
 425
 426
 427 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 428 {
 429    if(Debug)
 430       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl
 431                 << "Falling back to normal index file aquire" << std::endl;
 432    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 433                    ExpectedHash);
 434    Finish();
 435 }
 436
 437
 438 // helper that cleans the item out of the fetcher queue
 439 void pkgAcqIndexDiffs::Finish(bool allDone)
 440 {
 441    // we restore the original name, this is required, otherwise
 442    // the file will be cleaned
 443    if(allDone)
 444    {
 445       DestFile = _config->FindDir("Dir::State::lists");
 446       DestFile += URItoFileName(RealURI);
 447
 448       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 449       {
 450          Status = StatAuthError;
 451          ErrorText = _("MD5Sum mismatch");
 452          Rename(DestFile,DestFile + ".FAILED");
 453          Dequeue();
 454          return;
 455       }
 456
 457       // this is for the "real" finish
 458       Complete = true;
 459       Status = StatDone;
 460       Dequeue();
 461       if(Debug)
 462          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 463       return;
 464    }
 465
 466    if(Debug)
 467       std::clog << "Finishing: " << Desc.URI << std::endl;
 468    Complete = false;
 469    Status = StatDone;
 470    Dequeue();
 471    return;
 472 }
 473
 474
 475
 476 bool pkgAcqIndexDiffs::QueueNextDiff()
 477 {
 478
 479    // calc sha1 of the just patched file
 480    string FinalFile = _config->FindDir("Dir::State::lists");
 481    FinalFile += URItoFileName(RealURI);
 482
 483    FileFd fd(FinalFile, FileFd::ReadOnly);
 484    SHA1Summation SHA1;
 485    SHA1.AddFD(fd.Fd(), fd.Size());
 486    string local_sha1 = string(SHA1.Result());
 487    if(Debug)
 488       std::clog << "QueueNextDiff: "
 489                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 490
 491    // remove all patches until the next matching patch is found
 492    // this requires the Index file to be ordered
 493    for(vector<DiffInfo>::iterator I=available_patches.begin();
 494        available_patches.size() > 0 &&
 495           I != available_patches.end() &&
 496           (*I).sha1 != local_sha1;
 497        I++)
 498    {
 499       available_patches.erase(I);
 500    }
 501
 502    // error checking and falling back if no patch was found
 503    if(available_patches.size() == 0)
 504    {
 505       Failed("", NULL);
 506       return false;
 507    }
 508
 509    // queue the right diff
 510    Desc.URI = string(RealURI) + ".diff/" + available_patches[0].file + ".gz";
 511    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 512    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 513    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 514
 515    if(Debug)
 516       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 517
 518    QueueURI(Desc);
 519
 520    return true;
 521 }
 522
 523
 524
 525 void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash,
 526                             pkgAcquire::MethodConfig *Cnf)
 527 {
 528    if(Debug)
 529       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 530
 531    Item::Done(Message,Size,Md5Hash,Cnf);
 532
 533    string FinalFile;
 534    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 535
 536    // sucess in downloading a diff, enter ApplyDiff state
 537    if(State == StateFetchDiff)
 538    {
 539
 540       if(Debug)
 541          std::clog << "Sending to gzip method: " << FinalFile << std::endl;
 542
 543       string FileName = LookupTag(Message,"Filename");
 544       State = StateUnzipDiff;
 545       Local = true;
 546       Desc.URI = "gzip:" + FileName;
 547       DestFile += ".decomp";
 548       QueueURI(Desc);
 549       Mode = "gzip";
 550       return;
 551    }
 552
 553    // sucess in downloading a diff, enter ApplyDiff state
 554    if(State == StateUnzipDiff)
 555    {
 556
 557       // rred excepts the patch as $FinalFile.ed
 558       Rename(DestFile,FinalFile+".ed");
 559
 560       if(Debug)
 561          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 562
 563       State = StateApplyDiff;
 564       Local = true;
 565       Desc.URI = "rred:" + FinalFile;
 566       QueueURI(Desc);
 567       Mode = "rred";
 568       return;
 569    }
 570
 571
 572    // success in download/apply a diff, queue next (if needed)
 573    if(State == StateApplyDiff)
 574    {
 575       // remove the just applied patch
 576       available_patches.erase(available_patches.begin());
 577
 578       // move into place
 579       if(Debug)
 580       {
 581          std::clog << "Moving patched file in place: " << std::endl
 582                    << DestFile << " -> " << FinalFile << std::endl;
 583       }
 584       Rename(DestFile,FinalFile);
 585       chmod(FinalFile.c_str(),0644);
 586
 587       // see if there is more to download
 588       if(available_patches.size() > 0) {
 589          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 590                               ExpectedHash, available_patches);
 591          return Finish();
 592       } else
 593          return Finish(true);
 594    }
 595 }
 596
 597
 598 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 599 // ---------------------------------------------------------------------
 600 /* The package file is added to the queue and a second class is
 601    instantiated to fetch the revision file */
 602 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 603                          string URI,string URIDesc,string ShortDesc,
 604                          HashString ExpectedHash, string comprExt)
 605    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 606 {
 607    Decompression = false;
 608    Erase = false;
 609
 610    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 611    DestFile += URItoFileName(URI);
 612
 613    if(comprExt.empty())
 614    {
 615       // autoselect the compression method
 616       if(FileExists("/bin/bzip2"))
 617          CompressionExtension = ".bz2";
 618       else
 619          CompressionExtension = ".gz";
 620    } else {
 621       CompressionExtension = comprExt;
 622    }
 623    Desc.URI = URI + CompressionExtension;
 624
 625    Desc.Description = URIDesc;
 626    Desc.Owner = this;
 627    Desc.ShortDesc = ShortDesc;
 628
 629    QueueURI(Desc);
 630 }
 631                                                                         /*}}}*/
 632 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 633 // ---------------------------------------------------------------------
 634 /* The only header we use is the last-modified header. */
 635 string pkgAcqIndex::Custom600Headers()
 636 {
 637    string Final = _config->FindDir("Dir::State::lists");
 638    Final += URItoFileName(RealURI);
 639
 640    struct stat Buf;
 641    if (stat(Final.c_str(),&Buf) != 0)
 642       return "\nIndex-File: true";
 643    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 644 }
 645                                                                         /*}}}*/
 646
 647 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 648 {
 649    // no .bz2 found, retry with .gz
 650    if(Desc.URI.substr(Desc.URI.size()-3) == "bz2") {
 651       Desc.URI = Desc.URI.substr(0,Desc.URI.size()-3) + "gz";
 652
 653       // retry with a gzip one
 654       new pkgAcqIndex(Owner, RealURI, Desc.Description,Desc.ShortDesc,
 655                       ExpectedHash, string(".gz"));
 656       Status = StatDone;
 657       Complete = false;
 658       Dequeue();
 659       return;
 660    }
 661
 662    // on decompression failure, remove bad versions in partial/
 663    if(Decompression && Erase) {
 664       string s = _config->FindDir("Dir::State::lists") + "partial/";
 665       s += URItoFileName(RealURI);
 666       unlink(s.c_str());
 667    }
 668
 669    Item::Failed(Message,Cnf);
 670 }
 671
 672
 673 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
 674 // ---------------------------------------------------------------------
 675 /* This goes through a number of states.. On the initial fetch the
 676    method could possibly return an alternate filename which points
 677    to the uncompressed version of the file. If this is so the file
 678    is copied into the partial directory. In all other cases the file
 679    is decompressed with a gzip uri. */
 680 void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash,
 681                        pkgAcquire::MethodConfig *Cfg)
 682 {
 683    Item::Done(Message,Size,Hash,Cfg);
 684
 685    if (Decompression == true)
 686    {
 687       if (_config->FindB("Debug::pkgAcquire::Auth", false))
 688       {
 689          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
 690          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
 691       }
 692
 693       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
 694       {
 695          Status = StatAuthError;
 696          ErrorText = _("Hash Sum mismatch");
 697          Rename(DestFile,DestFile + ".FAILED");
 698          ReportMirrorFailure("HashChecksumFailure");
 699          return;
 700       }
 701       // Done, move it into position
 702       string FinalFile = _config->FindDir("Dir::State::lists");
 703       FinalFile += URItoFileName(RealURI);
 704       Rename(DestFile,FinalFile);
 705       chmod(FinalFile.c_str(),0644);
 706
 707       /* We restore the original name to DestFile so that the clean operation
 708          will work OK */
 709       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 710       DestFile += URItoFileName(RealURI);
 711
 712       // Remove the compressed version.
 713       if (Erase == true)
 714          unlink(DestFile.c_str());
 715       return;
 716    }
 717
 718    Erase = false;
 719    Complete = true;
 720
 721    // Handle the unzipd case
 722    string FileName = LookupTag(Message,"Alt-Filename");
 723    if (FileName.empty() == false)
 724    {
 725       // The files timestamp matches
 726       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
 727          return;
 728       Decompression = true;
 729       Local = true;
 730       DestFile += ".decomp";
 731       Desc.URI = "copy:" + FileName;
 732       QueueURI(Desc);
 733       Mode = "copy";
 734       return;
 735    }
 736
 737    FileName = LookupTag(Message,"Filename");
 738    if (FileName.empty() == true)
 739    {
 740       Status = StatError;
 741       ErrorText = "Method gave a blank filename";
 742    }
 743
 744    // The files timestamp matches
 745    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 746    {
 747       unlink(FileName.c_str());
 748       return;
 749    }
 750
 751    if (FileName == DestFile)
 752       Erase = true;
 753    else
 754       Local = true;
 755
 756    string compExt = Desc.URI.substr(Desc.URI.size()-3);
 757    char *decompProg;
 758    if(compExt == "bz2")
 759       decompProg = "bzip2";
 760    else if(compExt == ".gz")
 761       decompProg = "gzip";
 762    else {
 763       _error->Error("Unsupported extension: %s", compExt.c_str());
 764       return;
 765    }
 766
 767    Decompression = true;
 768    DestFile += ".decomp";
 769    Desc.URI = string(decompProg) + ":" + FileName;
 770    QueueURI(Desc);
 771    Mode = decompProg;
 772 }
 773
 774 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
 775 // ---------------------------------------------------------------------
 776 /* The Translation file is added to the queue */
 777 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
 778                             string URI,string URIDesc,string ShortDesc)
 779   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
 780 {
 781 }
 782
 783                                                                         /*}}}*/
 784 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
 785 // ---------------------------------------------------------------------
 786 /* */
 787 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 788 {
 789    if (Cnf->LocalOnly == true ||
 790        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 791    {
 792       // Ignore this
 793       Status = StatDone;
 794       Complete = false;
 795       Dequeue();
 796       return;
 797    }
 798
 799    Item::Failed(Message,Cnf);
 800 }
 801                                                                         /*}}}*/
 802
 803 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,
 804                              string URI,string URIDesc,string ShortDesc,
 805                              string MetaIndexURI, string MetaIndexURIDesc,
 806                              string MetaIndexShortDesc,
 807                              const vector<IndexTarget*>* IndexTargets,
 808                              indexRecords* MetaIndexParser) :
 809    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
 810    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
 811    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
 812 {
 813    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 814    DestFile += URItoFileName(URI);
 815
 816    // remove any partial downloaded sig-file in partial/.
 817    // it may confuse proxies and is too small to warrant a
 818    // partial download anyway
 819    unlink(DestFile.c_str());
 820
 821    // Create the item
 822    Desc.Description = URIDesc;
 823    Desc.Owner = this;
 824    Desc.ShortDesc = ShortDesc;
 825    Desc.URI = URI;
 826
 827
 828    string Final = _config->FindDir("Dir::State::lists");
 829    Final += URItoFileName(RealURI);
 830    struct stat Buf;
 831    if (stat(Final.c_str(),&Buf) == 0)
 832    {
 833       // File was already in place.  It needs to be re-verified
 834       // because Release might have changed, so Move it into partial
 835       Rename(Final,DestFile);
 836    }
 837
 838    QueueURI(Desc);
 839 }
 840                                                                         /*}}}*/
 841 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
 842 // ---------------------------------------------------------------------
 843 /* The only header we use is the last-modified header. */
 844 string pkgAcqMetaSig::Custom600Headers()
 845 {
 846    struct stat Buf;
 847    if (stat(DestFile.c_str(),&Buf) != 0)
 848       return "\nIndex-File: true";
 849
 850    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 851 }
 852
 853 void pkgAcqMetaSig::Done(string Message,unsigned long Size,string MD5,
 854                          pkgAcquire::MethodConfig *Cfg)
 855 {
 856    Item::Done(Message,Size,MD5,Cfg);
 857
 858    string FileName = LookupTag(Message,"Filename");
 859    if (FileName.empty() == true)
 860    {
 861       Status = StatError;
 862       ErrorText = "Method gave a blank filename";
 863       return;
 864    }
 865
 866    if (FileName != DestFile)
 867    {
 868       // We have to copy it into place
 869       Local = true;
 870       Desc.URI = "copy:" + FileName;
 871       QueueURI(Desc);
 872       return;
 873    }
 874
 875    Complete = true;
 876
 877    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
 878    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
 879                        DestFile, IndexTargets, MetaIndexParser);
 880
 881 }
 882                                                                         /*}}}*/
 883 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 884 {
 885    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 886
 887    // if we get a network error we fail gracefully
 888    if(Status == StatTransientNetworkError)
 889    {
 890       Item::Failed(Message,Cnf);
 891       // move the sigfile back on transient network failures
 892       if(FileExists(DestFile))
 893          Rename(DestFile,Final);
 894
 895       // set the status back to , Item::Failed likes to reset it
 896       Status = pkgAcquire::Item::StatTransientNetworkError;
 897       return;
 898    }
 899
 900    // Delete any existing sigfile when the acquire failed
 901    unlink(Final.c_str());
 902
 903    // queue a pkgAcqMetaIndex with no sigfile
 904    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
 905                        "", IndexTargets, MetaIndexParser);
 906
 907    if (Cnf->LocalOnly == true ||
 908        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 909    {
 910       // Ignore this
 911       Status = StatDone;
 912       Complete = false;
 913       Dequeue();
 914       return;
 915    }
 916
 917    Item::Failed(Message,Cnf);
 918 }
 919
 920 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,
 921                                  string URI,string URIDesc,string ShortDesc,
 922                                  string SigFile,
 923                                  const vector<struct IndexTarget*>* IndexTargets,
 924                                  indexRecords* MetaIndexParser) :
 925    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
 926    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
 927 {
 928    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 929    DestFile += URItoFileName(URI);
 930
 931    // Create the item
 932    Desc.Description = URIDesc;
 933    Desc.Owner = this;
 934    Desc.ShortDesc = ShortDesc;
 935    Desc.URI = URI;
 936
 937    QueueURI(Desc);
 938 }
 939
 940                                                                         /*}}}*/
 941 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
 942 // ---------------------------------------------------------------------
 943 /* The only header we use is the last-modified header. */
 944 string pkgAcqMetaIndex::Custom600Headers()
 945 {
 946    string Final = _config->FindDir("Dir::State::lists");
 947    Final += URItoFileName(RealURI);
 948
 949    struct stat Buf;
 950    if (stat(Final.c_str(),&Buf) != 0)
 951       return "\nIndex-File: true";
 952
 953    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 954 }
 955
 956 void pkgAcqMetaIndex::Done(string Message,unsigned long Size,string Hash,
 957                            pkgAcquire::MethodConfig *Cfg)
 958 {
 959    Item::Done(Message,Size,Hash,Cfg);
 960
 961    // MetaIndexes are done in two passes: one to download the
 962    // metaindex with an appropriate method, and a second to verify it
 963    // with the gpgv method
 964
 965    if (AuthPass == true)
 966    {
 967       AuthDone(Message);
 968    }
 969    else
 970    {
 971       RetrievalDone(Message);
 972       if (!Complete)
 973          // Still more retrieving to do
 974          return;
 975
 976       if (SigFile == "")
 977       {
 978          // There was no signature file, so we are finished.  Download
 979          // the indexes without verification.
 980          QueueIndexes(false);
 981       }
 982       else
 983       {
 984          // There was a signature file, so pass it to gpgv for
 985          // verification
 986
 987          if (_config->FindB("Debug::pkgAcquire::Auth", false))
 988             std::cerr << "Metaindex acquired, queueing gpg verification ("
 989                       << SigFile << "," << DestFile << ")\n";
 990          AuthPass = true;
 991          Desc.URI = "gpgv:" + SigFile;
 992          QueueURI(Desc);
 993          Mode = "gpgv";
 994       }
 995    }
 996 }
 997
 998 void pkgAcqMetaIndex::RetrievalDone(string Message)
 999 {
1000    // We have just finished downloading a Release file (it is not
1001    // verified yet)
1002
1003    string FileName = LookupTag(Message,"Filename");
1004    if (FileName.empty() == true)
1005    {
1006       Status = StatError;
1007       ErrorText = "Method gave a blank filename";
1008       return;
1009    }
1010
1011    if (FileName != DestFile)
1012    {
1013       Local = true;
1014       Desc.URI = "copy:" + FileName;
1015       QueueURI(Desc);
1016       return;
1017    }
1018
1019    // see if the download was a IMSHit
1020    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1021    Complete = true;
1022
1023    string FinalFile = _config->FindDir("Dir::State::lists");
1024    FinalFile += URItoFileName(RealURI);
1025
1026    // If we get a IMS hit we can remove the empty file in partial
1027    // othersie we move the file in place
1028    if (IMSHit)
1029       unlink(DestFile.c_str());
1030    else
1031       Rename(DestFile,FinalFile);
1032
1033    chmod(FinalFile.c_str(),0644);
1034    DestFile = FinalFile;
1035 }
1036
1037 void pkgAcqMetaIndex::AuthDone(string Message)
1038 {
1039    // At this point, the gpgv method has succeeded, so there is a
1040    // valid signature from a key in the trusted keyring.  We
1041    // perform additional verification of its contents, and use them
1042    // to verify the indexes we are about to download
1043
1044    if (!MetaIndexParser->Load(DestFile))
1045    {
1046       Status = StatAuthError;
1047       ErrorText = MetaIndexParser->ErrorText;
1048       return;
1049    }
1050
1051    if (!VerifyVendor(Message))
1052    {
1053       return;
1054    }
1055
1056    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1057       std::cerr << "Signature verification succeeded: "
1058                 << DestFile << std::endl;
1059
1060    // Download further indexes with verification
1061    QueueIndexes(true);
1062
1063    // Done, move signature file into position
1064
1065    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1066       URItoFileName(RealURI) + ".gpg";
1067    Rename(SigFile,VerifiedSigFile);
1068    chmod(VerifiedSigFile.c_str(),0644);
1069 }
1070
1071 void pkgAcqMetaIndex::QueueIndexes(bool verify)
1072 {
1073    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1074         Target != IndexTargets->end();
1075         Target++)
1076    {
1077       HashString ExpectedIndexHash;
1078       if (verify)
1079       {
1080          const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1081          if (!Record)
1082          {
1083             Status = StatAuthError;
1084             ErrorText = "Unable to find expected entry  "
1085                + (*Target)->MetaKey + " in Meta-index file (malformed Release file?)";
1086             return;
1087          }
1088          ExpectedIndexHash = Record->Hash;
1089          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1090          {
1091             std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1092             std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1093          }
1094          if (ExpectedIndexHash.empty())
1095          {
1096             Status = StatAuthError;
1097             ErrorText = "Unable to find hash sum for "
1098                + (*Target)->MetaKey + " in Meta-index file";
1099             return;
1100          }
1101       }
1102
1103       // Queue Packages file (either diff or full packages files, depending
1104       // on the users option)
1105       if(_config->FindB("Acquire::PDiffs",false) == true)
1106          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1107                              (*Target)->ShortDesc, ExpectedIndexHash);
1108       else
1109          new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description,
1110                             (*Target)->ShortDesc, ExpectedIndexHash);
1111    }
1112 }
1113
1114 bool pkgAcqMetaIndex::VerifyVendor(string Message)
1115 {
1116 //    // Maybe this should be made available from above so we don't have
1117 //    // to read and parse it every time?
1118 //    pkgVendorList List;
1119 //    List.ReadMainList();
1120
1121 //    const Vendor* Vndr = NULL;
1122 //    for (std::vector<string>::const_iterator I = GPGVOutput.begin(); I != GPGVOutput.end(); I++)
1123 //    {
1124 //       string::size_type pos = (*I).find("VALIDSIG ");
1125 //       if (_config->FindB("Debug::Vendor", false))
1126 //          std::cerr << "Looking for VALIDSIG in \"" << (*I) << "\": pos " << pos
1127 //                    << std::endl;
1128 //       if (pos != std::string::npos)
1129 //       {
1130 //          string Fingerprint = (*I).substr(pos+sizeof("VALIDSIG"));
1131 //          if (_config->FindB("Debug::Vendor", false))
1132 //             std::cerr << "Looking for \"" << Fingerprint << "\" in vendor..." <<
1133 //                std::endl;
1134 //          Vndr = List.FindVendor(Fingerprint) != "";
1135 //          if (Vndr != NULL);
1136 //          break;
1137 //       }
1138 //    }
1139    string::size_type pos;
1140
1141    // check for missing sigs (that where not fatal because otherwise we had
1142    // bombed earlier)
1143    string missingkeys;
1144    string msg = _("There is no public key available for the "
1145                   "following key IDs:\n");
1146    pos = Message.find("NO_PUBKEY ");
1147    if (pos != std::string::npos)
1148    {
1149       string::size_type start = pos+strlen("NO_PUBKEY ");
1150       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1151       missingkeys += (Fingerprint);
1152    }
1153    if(!missingkeys.empty())
1154       _error->Warning("%s", string(msg+missingkeys).c_str());
1155
1156    string Transformed = MetaIndexParser->GetExpectedDist();
1157
1158    if (Transformed == "../project/experimental")
1159    {
1160       Transformed = "experimental";
1161    }
1162
1163    pos = Transformed.rfind('/');
1164    if (pos != string::npos)
1165    {
1166       Transformed = Transformed.substr(0, pos);
1167    }
1168
1169    if (Transformed == ".")
1170    {
1171       Transformed = "";
1172    }
1173
1174    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1175    {
1176       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1177       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1178       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1179    }
1180
1181    if (MetaIndexParser->CheckDist(Transformed) == false)
1182    {
1183       // This might become fatal one day
1184 //       Status = StatAuthError;
1185 //       ErrorText = "Conflicting distribution; expected "
1186 //          + MetaIndexParser->GetExpectedDist() + " but got "
1187 //          + MetaIndexParser->GetDist();
1188 //       return false;
1189       if (!Transformed.empty())
1190       {
1191          _error->Warning("Conflicting distribution: %s (expected %s but got %s)",
1192                          Desc.Description.c_str(),
1193                          Transformed.c_str(),
1194                          MetaIndexParser->GetDist().c_str());
1195       }
1196    }
1197
1198    return true;
1199 }
1200                                                                 /*}}}*/
1201 // pkgAcqMetaIndex::Failed - no Release file present or no signature
1202 //      file present                                            /*{{{*/
1203 // ---------------------------------------------------------------------
1204 /* */
1205 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1206 {
1207    if (AuthPass == true)
1208    {
1209       // if we fail the authentication but got the file via a IMS-Hit
1210       // this means that the file wasn't downloaded and that it might be
1211       // just stale (server problem, proxy etc). we delete what we have
1212       // queue it again without i-m-s
1213       // alternatively we could just unlink the file and let the user try again
1214       if (IMSHit)
1215       {
1216          Complete = false;
1217          Local = false;
1218          AuthPass = false;
1219          unlink(DestFile.c_str());
1220
1221          DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1222          DestFile += URItoFileName(RealURI);
1223          Desc.URI = RealURI;
1224          QueueURI(Desc);
1225          return;
1226       }
1227
1228       // gpgv method failed
1229       ReportMirrorFailure("GPGFailure");
1230       _error->Warning("GPG error: %s: %s",
1231                       Desc.Description.c_str(),
1232                       LookupTag(Message,"Message").c_str());
1233
1234    }
1235
1236    // No Release file was present, or verification failed, so fall
1237    // back to queueing Packages files without verification
1238    QueueIndexes(false);
1239 }
1240
1241                                                                         /*}}}*/
1242
1243 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1244 // ---------------------------------------------------------------------
1245 /* This just sets up the initial fetch environment and queues the first
1246    possibilitiy */
1247 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1248                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1249                              string &StoreFilename) :
1250                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1251                StoreFilename(StoreFilename), Vf(Version.FileList()),
1252                Trusted(false)
1253 {
1254    Retries = _config->FindI("Acquire::Retries",0);
1255
1256    if (Version.Arch() == 0)
1257    {
1258       _error->Error(_("I wasn't able to locate a file for the %s package. "
1259                       "This might mean you need to manually fix this package. "
1260                       "(due to missing arch)"),
1261                     Version.ParentPkg().Name());
1262       return;
1263    }
1264
1265    /* We need to find a filename to determine the extension. We make the
1266       assumption here that all the available sources for this version share
1267       the same extension.. */
1268    // Skip not source sources, they do not have file fields.
1269    for (; Vf.end() == false; Vf++)
1270    {
1271       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1272          continue;
1273       break;
1274    }
1275
1276    // Does not really matter here.. we are going to fail out below
1277    if (Vf.end() != true)
1278    {
1279       // If this fails to get a file name we will bomb out below.
1280       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1281       if (_error->PendingError() == true)
1282          return;
1283
1284       // Generate the final file name as: package_version_arch.foo
1285       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1286                       QuoteString(Version.VerStr(),"_:") + '_' +
1287                       QuoteString(Version.Arch(),"_:.") +
1288                       "." + flExtension(Parse.FileName());
1289    }
1290
1291    // check if we have one trusted source for the package. if so, switch
1292    // to "TrustedOnly" mode
1293    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; i++)
1294    {
1295       pkgIndexFile *Index;
1296       if (Sources->FindIndex(i.File(),Index) == false)
1297          continue;
1298       if (_config->FindB("Debug::pkgAcquire::Auth", false))
1299       {
1300          std::cerr << "Checking index: " << Index->Describe()
1301                    << "(Trusted=" << Index->IsTrusted() << ")\n";
1302       }
1303       if (Index->IsTrusted()) {
1304          Trusted = true;
1305          break;
1306       }
1307    }
1308
1309    // "allow-unauthenticated" restores apts old fetching behaviour
1310    // that means that e.g. unauthenticated file:// uris are higher
1311    // priority than authenticated http:// uris
1312    if (_config->FindB("APT::Get::AllowUnauthenticated",false) == true)
1313       Trusted = false;
1314
1315    // Select a source
1316    if (QueueNext() == false && _error->PendingError() == false)
1317       _error->Error(_("I wasn't able to locate file for the %s package. "
1318                     "This might mean you need to manually fix this package."),
1319                     Version.ParentPkg().Name());
1320 }
1321                                                                         /*}}}*/
1322 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1323 // ---------------------------------------------------------------------
1324 /* This queues the next available file version for download. It checks if
1325    the archive is already available in the cache and stashs the MD5 for
1326    checking later. */
1327 bool pkgAcqArchive::QueueNext()
1328 {
1329    for (; Vf.end() == false; Vf++)
1330    {
1331       // Ignore not source sources
1332       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1333          continue;
1334
1335       // Try to cross match against the source list
1336       pkgIndexFile *Index;
1337       if (Sources->FindIndex(Vf.File(),Index) == false)
1338             continue;
1339
1340       // only try to get a trusted package from another source if that source
1341       // is also trusted
1342       if(Trusted && !Index->IsTrusted())
1343          continue;
1344
1345       // Grab the text package record
1346       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1347       if (_error->PendingError() == true)
1348          return false;
1349
1350       string PkgFile = Parse.FileName();
1351       if(Parse.SHA256Hash() != "")
1352          ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
1353       else if (Parse.SHA1Hash() != "")
1354          ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
1355       else
1356          ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1357       if (PkgFile.empty() == true)
1358          return _error->Error(_("The package index files are corrupted. No Filename: "
1359                               "field for package %s."),
1360                               Version.ParentPkg().Name());
1361
1362       Desc.URI = Index->ArchiveURI(PkgFile);
1363       Desc.Description = Index->ArchiveInfo(Version);
1364       Desc.Owner = this;
1365       Desc.ShortDesc = Version.ParentPkg().Name();
1366
1367       // See if we already have the file. (Legacy filenames)
1368       FileSize = Version->Size;
1369       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
1370       struct stat Buf;
1371       if (stat(FinalFile.c_str(),&Buf) == 0)
1372       {
1373          // Make sure the size matches
1374          if ((unsigned)Buf.st_size == Version->Size)
1375          {
1376             Complete = true;
1377             Local = true;
1378             Status = StatDone;
1379             StoreFilename = DestFile = FinalFile;
1380             return true;
1381          }
1382
1383          /* Hmm, we have a file and its size does not match, this means it is
1384             an old style mismatched arch */
1385          unlink(FinalFile.c_str());
1386       }
1387
1388       // Check it again using the new style output filenames
1389       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
1390       if (stat(FinalFile.c_str(),&Buf) == 0)
1391       {
1392          // Make sure the size matches
1393          if ((unsigned)Buf.st_size == Version->Size)
1394          {
1395             Complete = true;
1396             Local = true;
1397             Status = StatDone;
1398             StoreFilename = DestFile = FinalFile;
1399             return true;
1400          }
1401
1402          /* Hmm, we have a file and its size does not match, this shouldnt
1403             happen.. */
1404          unlink(FinalFile.c_str());
1405       }
1406
1407       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
1408
1409       // Check the destination file
1410       if (stat(DestFile.c_str(),&Buf) == 0)
1411       {
1412          // Hmm, the partial file is too big, erase it
1413          if ((unsigned)Buf.st_size > Version->Size)
1414             unlink(DestFile.c_str());
1415          else
1416             PartialSize = Buf.st_size;
1417       }
1418
1419       // Create the item
1420       Local = false;
1421       Desc.URI = Index->ArchiveURI(PkgFile);
1422       Desc.Description = Index->ArchiveInfo(Version);
1423       Desc.Owner = this;
1424       Desc.ShortDesc = Version.ParentPkg().Name();
1425       QueueURI(Desc);
1426
1427       Vf++;
1428       return true;
1429    }
1430    return false;
1431 }
1432                                                                         /*}}}*/
1433 // AcqArchive::Done - Finished fetching                                 /*{{{*/
1434 // ---------------------------------------------------------------------
1435 /* */
1436 void pkgAcqArchive::Done(string Message,unsigned long Size,string CalcHash,
1437                          pkgAcquire::MethodConfig *Cfg)
1438 {
1439    Item::Done(Message,Size,CalcHash,Cfg);
1440
1441    // Check the size
1442    if (Size != Version->Size)
1443    {
1444       Status = StatError;
1445       ErrorText = _("Size mismatch");
1446       return;
1447    }
1448
1449    // Check the hash
1450    if(ExpectedHash.toStr() != CalcHash)
1451    {
1452       Status = StatError;
1453       ErrorText = _("Hash Sum mismatch");
1454       if(FileExists(DestFile))
1455          Rename(DestFile,DestFile + ".FAILED");
1456       return;
1457    }
1458
1459    // Grab the output filename
1460    string FileName = LookupTag(Message,"Filename");
1461    if (FileName.empty() == true)
1462    {
1463       Status = StatError;
1464       ErrorText = "Method gave a blank filename";
1465       return;
1466    }
1467
1468    Complete = true;
1469
1470    // Reference filename
1471    if (FileName != DestFile)
1472    {
1473       StoreFilename = DestFile = FileName;
1474       Local = true;
1475       return;
1476    }
1477
1478    // Done, move it into position
1479    string FinalFile = _config->FindDir("Dir::Cache::Archives");
1480    FinalFile += flNotDir(StoreFilename);
1481    Rename(DestFile,FinalFile);
1482
1483    StoreFilename = DestFile = FinalFile;
1484    Complete = true;
1485 }
1486                                                                         /*}}}*/
1487 // AcqArchive::Failed - Failure handler                                 /*{{{*/
1488 // ---------------------------------------------------------------------
1489 /* Here we try other sources */
1490 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1491 {
1492    ErrorText = LookupTag(Message,"Message");
1493
1494    /* We don't really want to retry on failed media swaps, this prevents
1495       that. An interesting observation is that permanent failures are not
1496       recorded. */
1497    if (Cnf->Removable == true &&
1498        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1499    {
1500       // Vf = Version.FileList();
1501       while (Vf.end() == false) Vf++;
1502       StoreFilename = string();
1503       Item::Failed(Message,Cnf);
1504       return;
1505    }
1506
1507    if (QueueNext() == false)
1508    {
1509       // This is the retry counter
1510       if (Retries != 0 &&
1511           Cnf->LocalOnly == false &&
1512           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1513       {
1514          Retries--;
1515          Vf = Version.FileList();
1516          if (QueueNext() == true)
1517             return;
1518       }
1519
1520       StoreFilename = string();
1521       Item::Failed(Message,Cnf);
1522    }
1523 }
1524                                                                         /*}}}*/
1525 // AcqArchive::IsTrusted - Determine whether this archive comes from a
1526 // trusted source                                                       /*{{{*/
1527 // ---------------------------------------------------------------------
1528 bool pkgAcqArchive::IsTrusted()
1529 {
1530    return Trusted;
1531 }
1532
1533 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
1534 // ---------------------------------------------------------------------
1535 /* */
1536 void pkgAcqArchive::Finished()
1537 {
1538    if (Status == pkgAcquire::Item::StatDone &&
1539        Complete == true)
1540       return;
1541    StoreFilename = string();
1542 }
1543                                                                         /*}}}*/
1544
1545 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
1546 // ---------------------------------------------------------------------
1547 /* The file is added to the queue */
1548 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
1549                        unsigned long Size,string Dsc,string ShortDesc,
1550                        const string &DestDir, const string &DestFilename) :
1551                        Item(Owner), ExpectedHash(Hash)
1552 {
1553    Retries = _config->FindI("Acquire::Retries",0);
1554
1555    if(!DestFilename.empty())
1556       DestFile = DestFilename;
1557    else if(!DestDir.empty())
1558       DestFile = DestDir + "/" + flNotDir(URI);
1559    else
1560       DestFile = flNotDir(URI);
1561
1562    // Create the item
1563    Desc.URI = URI;
1564    Desc.Description = Dsc;
1565    Desc.Owner = this;
1566
1567    // Set the short description to the archive component
1568    Desc.ShortDesc = ShortDesc;
1569
1570    // Get the transfer sizes
1571    FileSize = Size;
1572    struct stat Buf;
1573    if (stat(DestFile.c_str(),&Buf) == 0)
1574    {
1575       // Hmm, the partial file is too big, erase it
1576       if ((unsigned)Buf.st_size > Size)
1577          unlink(DestFile.c_str());
1578       else
1579          PartialSize = Buf.st_size;
1580    }
1581
1582    QueueURI(Desc);
1583 }
1584                                                                         /*}}}*/
1585 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
1586 // ---------------------------------------------------------------------
1587 /* */
1588 void pkgAcqFile::Done(string Message,unsigned long Size,string CalcHash,
1589                       pkgAcquire::MethodConfig *Cnf)
1590 {
1591    Item::Done(Message,Size,CalcHash,Cnf);
1592
1593    // Check the hash
1594    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
1595    {
1596       Status = StatError;
1597       ErrorText = "Hash Sum mismatch";
1598       Rename(DestFile,DestFile + ".FAILED");
1599       return;
1600    }
1601
1602    string FileName = LookupTag(Message,"Filename");
1603    if (FileName.empty() == true)
1604    {
1605       Status = StatError;
1606       ErrorText = "Method gave a blank filename";
1607       return;
1608    }
1609
1610    Complete = true;
1611
1612    // The files timestamp matches
1613    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1614       return;
1615
1616    // We have to copy it into place
1617    if (FileName != DestFile)
1618    {
1619       Local = true;
1620       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
1621           Cnf->Removable == true)
1622       {
1623          Desc.URI = "copy:" + FileName;
1624          QueueURI(Desc);
1625          return;
1626       }
1627
1628       // Erase the file if it is a symlink so we can overwrite it
1629       struct stat St;
1630       if (lstat(DestFile.c_str(),&St) == 0)
1631       {
1632          if (S_ISLNK(St.st_mode) != 0)
1633             unlink(DestFile.c_str());
1634       }
1635
1636       // Symlink the file
1637       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
1638       {
1639          ErrorText = "Link to " + DestFile + " failure ";
1640          Status = StatError;
1641          Complete = false;
1642       }
1643    }
1644 }
1645                                                                         /*}}}*/
1646 // AcqFile::Failed - Failure handler                                    /*{{{*/
1647 // ---------------------------------------------------------------------
1648 /* Here we try other sources */
1649 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1650 {
1651    ErrorText = LookupTag(Message,"Message");
1652
1653    // This is the retry counter
1654    if (Retries != 0 &&
1655        Cnf->LocalOnly == false &&
1656        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1657    {
1658       Retries--;
1659       QueueURI(Desc);
1660       return;
1661    }
1662
1663    Item::Failed(Message,Cnf);
1664 }
1665                                                                         /*}}}*/