apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <apt-pkg/acquire-item.h>
  17 #include <apt-pkg/configuration.h>
  18 #include <apt-pkg/sourcelist.h>
  19 #include <apt-pkg/vendorlist.h>
  20 #include <apt-pkg/error.h>
  21 #include <apt-pkg/strutl.h>
  22 #include <apt-pkg/fileutl.h>
  23 #include <apt-pkg/md5.h>
  24 #include <apt-pkg/sha1.h>
  25 #include <apt-pkg/tagfile.h>
  26
  27 #include <apti18n.h>
  28
  29 #include <sys/stat.h>
  30 #include <unistd.h>
  31 #include <errno.h>
  32 #include <string>
  33 #include <sstream>
  34 #include <stdio.h>
  35                                                                         /*}}}*/
  36
  37 using namespace std;
  38
  39 // Acquire::Item::Item - Constructor                                    /*{{{*/
  40 // ---------------------------------------------------------------------
  41 /* */
  42 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  43                        PartialSize(0), Mode(0), ID(0), Complete(false),
  44                        Local(false), QueueCounter(0)
  45 {
  46    Owner->Add(this);
  47    Status = StatIdle;
  48 }
  49                                                                         /*}}}*/
  50 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  51 // ---------------------------------------------------------------------
  52 /* */
  53 pkgAcquire::Item::~Item()
  54 {
  55    Owner->Remove(this);
  56 }
  57                                                                         /*}}}*/
  58 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  59 // ---------------------------------------------------------------------
  60 /* We return to an idle state if there are still other queues that could
  61    fetch this object */
  62 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  63 {
  64    Status = StatIdle;
  65    ErrorText = LookupTag(Message,"Message");
  66    UsedMirror =  LookupTag(Message,"UsedMirror");
  67    if (QueueCounter <= 1)
  68    {
  69       /* This indicates that the file is not available right now but might
  70          be sometime later. If we do a retry cycle then this should be
  71          retried [CDROMs] */
  72       if (Cnf->LocalOnly == true &&
  73           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  74       {
  75          Status = StatIdle;
  76          Dequeue();
  77          return;
  78       }
  79
  80       Status = StatError;
  81       Dequeue();
  82    }
  83
  84    // report mirror failure back to LP if we actually use a mirror
  85    string FailReason = LookupTag(Message, "FailReason");
  86    if(FailReason.size() != 0)
  87       ReportMirrorFailure(FailReason);
  88    else
  89       ReportMirrorFailure(ErrorText);
  90 }
  91                                                                         /*}}}*/
  92 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
  93 // ---------------------------------------------------------------------
  94 /* Stash status and the file size. Note that setting Complete means
  95    sub-phases of the acquire process such as decompresion are operating */
  96 void pkgAcquire::Item::Start(string /*Message*/,unsigned long Size)
  97 {
  98    Status = StatFetching;
  99    if (FileSize == 0 && Complete == false)
 100       FileSize = Size;
 101 }
 102                                                                         /*}}}*/
 103 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
 104 // ---------------------------------------------------------------------
 105 /* */
 106 void pkgAcquire::Item::Done(string Message,unsigned long Size,string Hash,
 107                             pkgAcquire::MethodConfig *Cnf)
 108 {
 109    // We just downloaded something..
 110    string FileName = LookupTag(Message,"Filename");
 111    UsedMirror =  LookupTag(Message,"UsedMirror");
 112    if (Complete == false && !Local && FileName == DestFile)
 113    {
 114       if (Owner->Log != 0)
 115          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 116    }
 117
 118    if (FileSize == 0)
 119       FileSize= Size;
 120    Status = StatDone;
 121    ErrorText = string();
 122    Owner->Dequeue(this);
 123 }
 124                                                                         /*}}}*/
 125 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 126 // ---------------------------------------------------------------------
 127 /* This helper function is used by alot of item methods as thier final
 128    step */
 129 void pkgAcquire::Item::Rename(string From,string To)
 130 {
 131    if (rename(From.c_str(),To.c_str()) != 0)
 132    {
 133       char S[300];
 134       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 135               From.c_str(),To.c_str());
 136       Status = StatError;
 137       ErrorText = S;
 138    }
 139 }
 140                                                                         /*}}}*/
 141
 142 void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
 143 {
 144    // we only act if a mirror was used at all
 145    if(UsedMirror.empty())
 146       return;
 147 #if 0
 148    std::cerr << "\nReportMirrorFailure: "
 149              << UsedMirror
 150              << " Uri: " << DescURI()
 151              << " FailCode: "
 152              << FailCode << std::endl;
 153 #endif
 154    const char *Args[40];
 155    unsigned int i = 0;
 156    string report = _config->Find("Methods::Mirror::ProblemReporting",
 157                                  "/usr/lib/apt/apt-report-mirror-failure");
 158    if(!FileExists(report))
 159       return;
 160    Args[i++] = report.c_str();
 161    Args[i++] = UsedMirror.c_str();
 162    Args[i++] = DescURI().c_str();
 163    Args[i++] = FailCode.c_str();
 164    Args[i++] = NULL;
 165    pid_t pid = ExecFork();
 166    if(pid < 0)
 167    {
 168       _error->Error("ReportMirrorFailure Fork failed");
 169       return;
 170    }
 171    else if(pid == 0)
 172    {
 173       execvp(Args[0], (char**)Args);
 174       std::cerr << "Could not exec " << Args[0] << std::endl;
 175       _exit(100);
 176    }
 177    if(!ExecWait(pid, "report-mirror-failure"))
 178    {
 179       _error->Warning("Couldn't report problem to '%s'",
 180                       _config->Find("Methods::Mirror::ProblemReporting").c_str());
 181    }
 182 }
 183
 184
 185
 186 // AcqDiffIndex::AcqDiffIndex - Constructor
 187 // ---------------------------------------------------------------------
 188 /* Get the DiffIndex file first and see if there are patches availabe
 189  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 190  * patches. If anything goes wrong in that process, it will fall back to
 191  * the original packages file
 192  */
 193 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 194                                  string URI,string URIDesc,string ShortDesc,
 195                                  HashString ExpectedHash)
 196    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 197      Description(URIDesc)
 198 {
 199
 200    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 201
 202    Desc.Description = URIDesc + "/DiffIndex";
 203    Desc.Owner = this;
 204    Desc.ShortDesc = ShortDesc;
 205    Desc.URI = URI + ".diff/Index";
 206
 207    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 208    DestFile += URItoFileName(URI) + string(".DiffIndex");
 209
 210    if(Debug)
 211       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 212
 213    // look for the current package file
 214    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 215    CurrentPackagesFile += URItoFileName(RealURI);
 216
 217    // FIXME: this file:/ check is a hack to prevent fetching
 218    //        from local sources. this is really silly, and
 219    //        should be fixed cleanly as soon as possible
 220    if(!FileExists(CurrentPackagesFile) ||
 221       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 222    {
 223       // we don't have a pkg file or we don't want to queue
 224       if(Debug)
 225          std::clog << "No index file, local or canceld by user" << std::endl;
 226       Failed("", NULL);
 227       return;
 228    }
 229
 230    if(Debug)
 231       std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): "
 232                 << CurrentPackagesFile << std::endl;
 233
 234    QueueURI(Desc);
 235
 236 }
 237
 238 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 239 // ---------------------------------------------------------------------
 240 /* The only header we use is the last-modified header. */
 241 string pkgAcqDiffIndex::Custom600Headers()
 242 {
 243    string Final = _config->FindDir("Dir::State::lists");
 244    Final += URItoFileName(RealURI) + string(".IndexDiff");
 245
 246    if(Debug)
 247       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 248
 249    struct stat Buf;
 250    if (stat(Final.c_str(),&Buf) != 0)
 251       return "\nIndex-File: true";
 252
 253    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 254 }
 255
 256
 257 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)
 258 {
 259    if(Debug)
 260       std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile
 261                 << std::endl;
 262
 263    pkgTagSection Tags;
 264    string ServerSha1;
 265    vector<DiffInfo> available_patches;
 266
 267    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 268    pkgTagFile TF(&Fd);
 269    if (_error->PendingError() == true)
 270       return false;
 271
 272    if(TF.Step(Tags) == true)
 273    {
 274       string local_sha1;
 275       bool found = false;
 276       DiffInfo d;
 277       string size;
 278
 279       string tmp = Tags.FindS("SHA1-Current");
 280       std::stringstream ss(tmp);
 281       ss >> ServerSha1;
 282
 283       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 284       SHA1Summation SHA1;
 285       SHA1.AddFD(fd.Fd(), fd.Size());
 286       local_sha1 = string(SHA1.Result());
 287
 288       if(local_sha1 == ServerSha1)
 289       {
 290          // we have the same sha1 as the server
 291          if(Debug)
 292             std::clog << "Package file is up-to-date" << std::endl;
 293          // set found to true, this will queue a pkgAcqIndexDiffs with
 294          // a empty availabe_patches
 295          found = true;
 296       }
 297       else
 298       {
 299          if(Debug)
 300             std::clog << "SHA1-Current: " << ServerSha1 << std::endl;
 301
 302          // check the historie and see what patches we need
 303          string history = Tags.FindS("SHA1-History");
 304          std::stringstream hist(history);
 305          while(hist >> d.sha1 >> size >> d.file)
 306          {
 307             d.size = atoi(size.c_str());
 308             // read until the first match is found
 309             if(d.sha1 == local_sha1)
 310                found=true;
 311             // from that point on, we probably need all diffs
 312             if(found)
 313             {
 314                if(Debug)
 315                   std::clog << "Need to get diff: " << d.file << std::endl;
 316                available_patches.push_back(d);
 317             }
 318          }
 319       }
 320
 321       // we have something, queue the next diff
 322       if(found)
 323       {
 324          // queue the diffs
 325          string::size_type last_space = Description.rfind(" ");
 326          if(last_space != string::npos)
 327             Description.erase(last_space, Description.size()-last_space);
 328          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 329                               ExpectedHash, available_patches);
 330          Complete = false;
 331          Status = StatDone;
 332          Dequeue();
 333          return true;
 334       }
 335    }
 336
 337    // Nothing found, report and return false
 338    // Failing here is ok, if we return false later, the full
 339    // IndexFile is queued
 340    if(Debug)
 341       std::clog << "Can't find a patch in the index file" << std::endl;
 342    return false;
 343 }
 344
 345 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 346 {
 347    if(Debug)
 348       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << std::endl
 349                 << "Falling back to normal index file aquire" << std::endl;
 350
 351    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 352                    ExpectedHash);
 353
 354    Complete = false;
 355    Status = StatDone;
 356    Dequeue();
 357 }
 358
 359 void pkgAcqDiffIndex::Done(string Message,unsigned long Size,string Md5Hash,
 360                            pkgAcquire::MethodConfig *Cnf)
 361 {
 362    if(Debug)
 363       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 364
 365    Item::Done(Message,Size,Md5Hash,Cnf);
 366
 367    string FinalFile;
 368    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 369
 370    // sucess in downloading the index
 371    // rename the index
 372    FinalFile += string(".IndexDiff");
 373    if(Debug)
 374       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 375                 << std::endl;
 376    Rename(DestFile,FinalFile);
 377    chmod(FinalFile.c_str(),0644);
 378    DestFile = FinalFile;
 379
 380    if(!ParseDiffIndex(DestFile))
 381       return Failed("", NULL);
 382
 383    Complete = true;
 384    Status = StatDone;
 385    Dequeue();
 386    return;
 387 }
 388
 389
 390
 391 // AcqIndexDiffs::AcqIndexDiffs - Constructor
 392 // ---------------------------------------------------------------------
 393 /* The package diff is added to the queue. one object is constructed
 394  * for each diff and the index
 395  */
 396 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 397                                    string URI,string URIDesc,string ShortDesc,
 398                                    HashString ExpectedHash,
 399                                    vector<DiffInfo> diffs)
 400    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 401      available_patches(diffs)
 402 {
 403
 404    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 405    DestFile += URItoFileName(URI);
 406
 407    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 408
 409    Description = URIDesc;
 410    Desc.Owner = this;
 411    Desc.ShortDesc = ShortDesc;
 412
 413    if(available_patches.size() == 0)
 414    {
 415       // we are done (yeah!)
 416       Finish(true);
 417    }
 418    else
 419    {
 420       // get the next diff
 421       State = StateFetchDiff;
 422       QueueNextDiff();
 423    }
 424 }
 425
 426
 427 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 428 {
 429    if(Debug)
 430       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl
 431                 << "Falling back to normal index file aquire" << std::endl;
 432    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 433                    ExpectedHash);
 434    Finish();
 435 }
 436
 437
 438 // helper that cleans the item out of the fetcher queue
 439 void pkgAcqIndexDiffs::Finish(bool allDone)
 440 {
 441    // we restore the original name, this is required, otherwise
 442    // the file will be cleaned
 443    if(allDone)
 444    {
 445       DestFile = _config->FindDir("Dir::State::lists");
 446       DestFile += URItoFileName(RealURI);
 447
 448       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 449       {
 450          Status = StatAuthError;
 451          ErrorText = _("MD5Sum mismatch");
 452          Rename(DestFile,DestFile + ".FAILED");
 453          Dequeue();
 454          return;
 455       }
 456
 457       // this is for the "real" finish
 458       Complete = true;
 459       Status = StatDone;
 460       Dequeue();
 461       if(Debug)
 462          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 463       return;
 464    }
 465
 466    if(Debug)
 467       std::clog << "Finishing: " << Desc.URI << std::endl;
 468    Complete = false;
 469    Status = StatDone;
 470    Dequeue();
 471    return;
 472 }
 473
 474
 475
 476 bool pkgAcqIndexDiffs::QueueNextDiff()
 477 {
 478
 479    // calc sha1 of the just patched file
 480    string FinalFile = _config->FindDir("Dir::State::lists");
 481    FinalFile += URItoFileName(RealURI);
 482
 483    FileFd fd(FinalFile, FileFd::ReadOnly);
 484    SHA1Summation SHA1;
 485    SHA1.AddFD(fd.Fd(), fd.Size());
 486    string local_sha1 = string(SHA1.Result());
 487    if(Debug)
 488       std::clog << "QueueNextDiff: "
 489                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 490
 491    // remove all patches until the next matching patch is found
 492    // this requires the Index file to be ordered
 493    for(vector<DiffInfo>::iterator I=available_patches.begin();
 494        available_patches.size() > 0 &&
 495           I != available_patches.end() &&
 496           (*I).sha1 != local_sha1;
 497        I++)
 498    {
 499       available_patches.erase(I);
 500    }
 501
 502    // error checking and falling back if no patch was found
 503    if(available_patches.size() == 0)
 504    {
 505       Failed("", NULL);
 506       return false;
 507    }
 508
 509    // queue the right diff
 510    Desc.URI = string(RealURI) + ".diff/" + available_patches[0].file + ".gz";
 511    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 512    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 513    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 514
 515    if(Debug)
 516       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 517
 518    QueueURI(Desc);
 519
 520    return true;
 521 }
 522
 523
 524
 525 void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash,
 526                             pkgAcquire::MethodConfig *Cnf)
 527 {
 528    if(Debug)
 529       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 530
 531    Item::Done(Message,Size,Md5Hash,Cnf);
 532
 533    string FinalFile;
 534    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 535
 536    // sucess in downloading a diff, enter ApplyDiff state
 537    if(State == StateFetchDiff)
 538    {
 539
 540       if(Debug)
 541          std::clog << "Sending to gzip method: " << FinalFile << std::endl;
 542
 543       string FileName = LookupTag(Message,"Filename");
 544       State = StateUnzipDiff;
 545       Local = true;
 546       Desc.URI = "gzip:" + FileName;
 547       DestFile += ".decomp";
 548       QueueURI(Desc);
 549       Mode = "gzip";
 550       return;
 551    }
 552
 553    // sucess in downloading a diff, enter ApplyDiff state
 554    if(State == StateUnzipDiff)
 555    {
 556
 557       // rred excepts the patch as $FinalFile.ed
 558       Rename(DestFile,FinalFile+".ed");
 559
 560       if(Debug)
 561          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 562
 563       State = StateApplyDiff;
 564       Local = true;
 565       Desc.URI = "rred:" + FinalFile;
 566       QueueURI(Desc);
 567       Mode = "rred";
 568       return;
 569    }
 570
 571
 572    // success in download/apply a diff, queue next (if needed)
 573    if(State == StateApplyDiff)
 574    {
 575       // remove the just applied patch
 576       available_patches.erase(available_patches.begin());
 577
 578       // move into place
 579       if(Debug)
 580       {
 581          std::clog << "Moving patched file in place: " << std::endl
 582                    << DestFile << " -> " << FinalFile << std::endl;
 583       }
 584       Rename(DestFile,FinalFile);
 585       chmod(FinalFile.c_str(),0644);
 586
 587       // see if there is more to download
 588       if(available_patches.size() > 0) {
 589          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 590                               ExpectedHash, available_patches);
 591          return Finish();
 592       } else
 593          return Finish(true);
 594    }
 595 }
 596
 597
 598 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 599 // ---------------------------------------------------------------------
 600 /* The package file is added to the queue and a second class is
 601    instantiated to fetch the revision file */
 602 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 603                          string URI,string URIDesc,string ShortDesc,
 604                          HashString ExpectedHash, string comprExt)
 605    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 606 {
 607    Decompression = false;
 608    Erase = false;
 609
 610    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 611    DestFile += URItoFileName(URI);
 612
 613    if(comprExt.empty())
 614    {
 615       // autoselect the compression method
 616       if(FileExists("/bin/bzip2"))
 617          CompressionExtension = ".bz2";
 618       else
 619          CompressionExtension = ".gz";
 620    } else {
 621       CompressionExtension = comprExt;
 622    }
 623    Desc.URI = URI + CompressionExtension;
 624
 625    Desc.Description = URIDesc;
 626    Desc.Owner = this;
 627    Desc.ShortDesc = ShortDesc;
 628
 629    QueueURI(Desc);
 630 }
 631                                                                         /*}}}*/
 632 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 633 // ---------------------------------------------------------------------
 634 /* The only header we use is the last-modified header. */
 635 string pkgAcqIndex::Custom600Headers()
 636 {
 637    string Final = _config->FindDir("Dir::State::lists");
 638    Final += URItoFileName(RealURI);
 639
 640    struct stat Buf;
 641    if (stat(Final.c_str(),&Buf) != 0)
 642       return "\nIndex-File: true";
 643    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 644 }
 645                                                                         /*}}}*/
 646
 647 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 648 {
 649    // no .bz2 found, retry with .gz
 650    if(Desc.URI.substr(Desc.URI.size()-3) == "bz2") {
 651       Desc.URI = Desc.URI.substr(0,Desc.URI.size()-3) + "gz";
 652
 653       // retry with a gzip one
 654       new pkgAcqIndex(Owner, RealURI, Desc.Description,Desc.ShortDesc,
 655                       ExpectedHash, string(".gz"));
 656       Status = StatDone;
 657       Complete = false;
 658       Dequeue();
 659       return;
 660    }
 661
 662    // on decompression failure, remove bad versions in partial/
 663    if(Decompression && Erase) {
 664       string s = _config->FindDir("Dir::State::lists") + "partial/";
 665       s += URItoFileName(RealURI);
 666       unlink(s.c_str());
 667    }
 668
 669    Item::Failed(Message,Cnf);
 670 }
 671
 672
 673 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
 674 // ---------------------------------------------------------------------
 675 /* This goes through a number of states.. On the initial fetch the
 676    method could possibly return an alternate filename which points
 677    to the uncompressed version of the file. If this is so the file
 678    is copied into the partial directory. In all other cases the file
 679    is decompressed with a gzip uri. */
 680 void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash,
 681                        pkgAcquire::MethodConfig *Cfg)
 682 {
 683    Item::Done(Message,Size,Hash,Cfg);
 684
 685    if (Decompression == true)
 686    {
 687       if (_config->FindB("Debug::pkgAcquire::Auth", false))
 688       {
 689          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
 690          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
 691       }
 692
 693       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
 694       {
 695          Status = StatAuthError;
 696          ErrorText = _("Hash Sum mismatch");
 697          Rename(DestFile,DestFile + ".FAILED");
 698          ReportMirrorFailure("HashChecksumFailure");
 699          return;
 700       }
 701       // Done, move it into position
 702       string FinalFile = _config->FindDir("Dir::State::lists");
 703       FinalFile += URItoFileName(RealURI);
 704       Rename(DestFile,FinalFile);
 705       chmod(FinalFile.c_str(),0644);
 706
 707       /* We restore the original name to DestFile so that the clean operation
 708          will work OK */
 709       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 710       DestFile += URItoFileName(RealURI);
 711
 712       // Remove the compressed version.
 713       if (Erase == true)
 714          unlink(DestFile.c_str());
 715       return;
 716    }
 717
 718    Erase = false;
 719    Complete = true;
 720
 721    // Handle the unzipd case
 722    string FileName = LookupTag(Message,"Alt-Filename");
 723    if (FileName.empty() == false)
 724    {
 725       // The files timestamp matches
 726       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
 727          return;
 728       Decompression = true;
 729       Local = true;
 730       DestFile += ".decomp";
 731       Desc.URI = "copy:" + FileName;
 732       QueueURI(Desc);
 733       Mode = "copy";
 734       return;
 735    }
 736
 737    FileName = LookupTag(Message,"Filename");
 738    if (FileName.empty() == true)
 739    {
 740       Status = StatError;
 741       ErrorText = "Method gave a blank filename";
 742    }
 743
 744    // The files timestamp matches
 745    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 746       return;
 747
 748    if (FileName == DestFile)
 749       Erase = true;
 750    else
 751       Local = true;
 752
 753    string compExt = Desc.URI.substr(Desc.URI.size()-3);
 754    const char *decompProg;
 755    if(compExt == "bz2")
 756       decompProg = "bzip2";
 757    else if(compExt == ".gz")
 758       decompProg = "gzip";
 759    else {
 760       _error->Error("Unsupported extension: %s", compExt.c_str());
 761       return;
 762    }
 763
 764    Decompression = true;
 765    DestFile += ".decomp";
 766    Desc.URI = string(decompProg) + ":" + FileName;
 767    QueueURI(Desc);
 768    Mode = decompProg;
 769 }
 770
 771 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
 772 // ---------------------------------------------------------------------
 773 /* The Translation file is added to the queue */
 774 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
 775                             string URI,string URIDesc,string ShortDesc)
 776   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
 777 {
 778 }
 779
 780                                                                         /*}}}*/
 781 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
 782 // ---------------------------------------------------------------------
 783 /* */
 784 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 785 {
 786    if (Cnf->LocalOnly == true ||
 787        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 788    {
 789       // Ignore this
 790       Status = StatDone;
 791       Complete = false;
 792       Dequeue();
 793       return;
 794    }
 795
 796    Item::Failed(Message,Cnf);
 797 }
 798                                                                         /*}}}*/
 799
 800 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,
 801                              string URI,string URIDesc,string ShortDesc,
 802                              string MetaIndexURI, string MetaIndexURIDesc,
 803                              string MetaIndexShortDesc,
 804                              const vector<IndexTarget*>* IndexTargets,
 805                              indexRecords* MetaIndexParser) :
 806    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
 807    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
 808    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
 809 {
 810    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 811    DestFile += URItoFileName(URI);
 812
 813    // remove any partial downloaded sig-file in partial/.
 814    // it may confuse proxies and is too small to warrant a
 815    // partial download anyway
 816    unlink(DestFile.c_str());
 817
 818    // Create the item
 819    Desc.Description = URIDesc;
 820    Desc.Owner = this;
 821    Desc.ShortDesc = ShortDesc;
 822    Desc.URI = URI;
 823
 824    string Final = _config->FindDir("Dir::State::lists");
 825    Final += URItoFileName(RealURI);
 826    struct stat Buf;
 827    if (stat(Final.c_str(),&Buf) == 0)
 828    {
 829       // File was already in place.  It needs to be re-downloaded/verified
 830       // because Release might have changed, we do give it a differnt
 831       // name than DestFile because otherwise the http method will
 832       // send If-Range requests and there are too many broken servers
 833       // out there that do not understand them
 834       LastGoodSig = DestFile+".reverify";
 835       Rename(Final,LastGoodSig);
 836    }
 837
 838    QueueURI(Desc);
 839 }
 840                                                                         /*}}}*/
 841 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
 842 // ---------------------------------------------------------------------
 843 /* The only header we use is the last-modified header. */
 844 string pkgAcqMetaSig::Custom600Headers()
 845 {
 846    struct stat Buf;
 847    if (stat(LastGoodSig.c_str(),&Buf) != 0)
 848       return "\nIndex-File: true";
 849
 850    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 851 }
 852
 853 void pkgAcqMetaSig::Done(string Message,unsigned long Size,string MD5,
 854                          pkgAcquire::MethodConfig *Cfg)
 855 {
 856    Item::Done(Message,Size,MD5,Cfg);
 857
 858    string FileName = LookupTag(Message,"Filename");
 859    if (FileName.empty() == true)
 860    {
 861       Status = StatError;
 862       ErrorText = "Method gave a blank filename";
 863       return;
 864    }
 865
 866    if (FileName != DestFile)
 867    {
 868       // We have to copy it into place
 869       Local = true;
 870       Desc.URI = "copy:" + FileName;
 871       QueueURI(Desc);
 872       return;
 873    }
 874
 875    Complete = true;
 876
 877    // put the last known good file back on i-m-s hit (it will
 878    // be re-verified again)
 879    // Else do nothing, we have the new file in DestFile then
 880    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 881       Rename(LastGoodSig, DestFile);
 882
 883    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
 884    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc,
 885                        MetaIndexShortDesc,  DestFile, IndexTargets,
 886                        MetaIndexParser);
 887
 888 }
 889                                                                         /*}}}*/
 890 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 891 {
 892    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 893
 894    // if we get a network error we fail gracefully
 895    if(Status == StatTransientNetworkError)
 896    {
 897       Item::Failed(Message,Cnf);
 898       // move the sigfile back on transient network failures
 899       if(FileExists(DestFile))
 900          Rename(LastGoodSig,Final);
 901
 902       // set the status back to , Item::Failed likes to reset it
 903       Status = pkgAcquire::Item::StatTransientNetworkError;
 904       return;
 905    }
 906
 907    // Delete any existing sigfile when the acquire failed
 908    unlink(Final.c_str());
 909
 910    // queue a pkgAcqMetaIndex with no sigfile
 911    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
 912                        "", IndexTargets, MetaIndexParser);
 913
 914    if (Cnf->LocalOnly == true ||
 915        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 916    {
 917       // Ignore this
 918       Status = StatDone;
 919       Complete = false;
 920       Dequeue();
 921       return;
 922    }
 923
 924    Item::Failed(Message,Cnf);
 925 }
 926
 927 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,
 928                                  string URI,string URIDesc,string ShortDesc,
 929                                  string SigFile,
 930                                  const vector<struct IndexTarget*>* IndexTargets,
 931                                  indexRecords* MetaIndexParser) :
 932    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
 933    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
 934 {
 935    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 936    DestFile += URItoFileName(URI);
 937
 938    // Create the item
 939    Desc.Description = URIDesc;
 940    Desc.Owner = this;
 941    Desc.ShortDesc = ShortDesc;
 942    Desc.URI = URI;
 943
 944    QueueURI(Desc);
 945 }
 946
 947                                                                         /*}}}*/
 948 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
 949 // ---------------------------------------------------------------------
 950 /* The only header we use is the last-modified header. */
 951 string pkgAcqMetaIndex::Custom600Headers()
 952 {
 953    string Final = _config->FindDir("Dir::State::lists");
 954    Final += URItoFileName(RealURI);
 955
 956    struct stat Buf;
 957    if (stat(Final.c_str(),&Buf) != 0)
 958       return "\nIndex-File: true";
 959
 960    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 961 }
 962
 963 void pkgAcqMetaIndex::Done(string Message,unsigned long Size,string Hash,
 964                            pkgAcquire::MethodConfig *Cfg)
 965 {
 966    Item::Done(Message,Size,Hash,Cfg);
 967
 968    // MetaIndexes are done in two passes: one to download the
 969    // metaindex with an appropriate method, and a second to verify it
 970    // with the gpgv method
 971
 972    if (AuthPass == true)
 973    {
 974       AuthDone(Message);
 975
 976       // all cool, move Release file into place
 977       Complete = true;
 978
 979       string FinalFile = _config->FindDir("Dir::State::lists");
 980       FinalFile += URItoFileName(RealURI);
 981       Rename(DestFile,FinalFile);
 982       chmod(FinalFile.c_str(),0644);
 983       DestFile = FinalFile;
 984    }
 985    else
 986    {
 987       RetrievalDone(Message);
 988       if (!Complete)
 989          // Still more retrieving to do
 990          return;
 991
 992       if (SigFile == "")
 993       {
 994          // There was no signature file, so we are finished.  Download
 995          // the indexes without verification.
 996          QueueIndexes(false);
 997       }
 998       else
 999       {
1000          // There was a signature file, so pass it to gpgv for
1001          // verification
1002
1003          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1004             std::cerr << "Metaindex acquired, queueing gpg verification ("
1005                       << SigFile << "," << DestFile << ")\n";
1006          AuthPass = true;
1007          Desc.URI = "gpgv:" + SigFile;
1008          QueueURI(Desc);
1009          Mode = "gpgv";
1010       }
1011    }
1012 }
1013
1014 void pkgAcqMetaIndex::RetrievalDone(string Message)
1015 {
1016    // We have just finished downloading a Release file (it is not
1017    // verified yet)
1018
1019    string FileName = LookupTag(Message,"Filename");
1020    if (FileName.empty() == true)
1021    {
1022       Status = StatError;
1023       ErrorText = "Method gave a blank filename";
1024       return;
1025    }
1026
1027    if (FileName != DestFile)
1028    {
1029       Local = true;
1030       Desc.URI = "copy:" + FileName;
1031       QueueURI(Desc);
1032       return;
1033    }
1034
1035    // make sure to verify against the right file on I-M-S hit
1036    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1037    if(IMSHit)
1038    {
1039       string FinalFile = _config->FindDir("Dir::State::lists");
1040       FinalFile += URItoFileName(RealURI);
1041       DestFile = FinalFile;
1042    }
1043    Complete = true;
1044 }
1045
1046 void pkgAcqMetaIndex::AuthDone(string Message)
1047 {
1048    // At this point, the gpgv method has succeeded, so there is a
1049    // valid signature from a key in the trusted keyring.  We
1050    // perform additional verification of its contents, and use them
1051    // to verify the indexes we are about to download
1052
1053    if (!MetaIndexParser->Load(DestFile))
1054    {
1055       Status = StatAuthError;
1056       ErrorText = MetaIndexParser->ErrorText;
1057       return;
1058    }
1059
1060    if (!VerifyVendor(Message))
1061    {
1062       return;
1063    }
1064
1065    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1066       std::cerr << "Signature verification succeeded: "
1067                 << DestFile << std::endl;
1068
1069    // Download further indexes with verification
1070    QueueIndexes(true);
1071
1072    // Done, move signature file into position
1073    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1074       URItoFileName(RealURI) + ".gpg";
1075    Rename(SigFile,VerifiedSigFile);
1076    chmod(VerifiedSigFile.c_str(),0644);
1077 }
1078
1079 void pkgAcqMetaIndex::QueueIndexes(bool verify)
1080 {
1081    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1082         Target != IndexTargets->end();
1083         Target++)
1084    {
1085       HashString ExpectedIndexHash;
1086       if (verify)
1087       {
1088          const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1089          if (!Record)
1090          {
1091             Status = StatAuthError;
1092             ErrorText = "Unable to find expected entry  "
1093                + (*Target)->MetaKey + " in Meta-index file (malformed Release file?)";
1094             return;
1095          }
1096          ExpectedIndexHash = Record->Hash;
1097          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1098          {
1099             std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1100             std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1101          }
1102          if (ExpectedIndexHash.empty())
1103          {
1104             Status = StatAuthError;
1105             ErrorText = "Unable to find hash sum for "
1106                + (*Target)->MetaKey + " in Meta-index file";
1107             return;
1108          }
1109       }
1110
1111       // Queue Packages file (either diff or full packages files, depending
1112       // on the users option)
1113       if(_config->FindB("Acquire::PDiffs",false) == true)
1114          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1115                              (*Target)->ShortDesc, ExpectedIndexHash);
1116       else
1117          new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description,
1118                             (*Target)->ShortDesc, ExpectedIndexHash);
1119    }
1120 }
1121
1122 bool pkgAcqMetaIndex::VerifyVendor(string Message)
1123 {
1124 //    // Maybe this should be made available from above so we don't have
1125 //    // to read and parse it every time?
1126 //    pkgVendorList List;
1127 //    List.ReadMainList();
1128
1129 //    const Vendor* Vndr = NULL;
1130 //    for (std::vector<string>::const_iterator I = GPGVOutput.begin(); I != GPGVOutput.end(); I++)
1131 //    {
1132 //       string::size_type pos = (*I).find("VALIDSIG ");
1133 //       if (_config->FindB("Debug::Vendor", false))
1134 //          std::cerr << "Looking for VALIDSIG in \"" << (*I) << "\": pos " << pos
1135 //                    << std::endl;
1136 //       if (pos != std::string::npos)
1137 //       {
1138 //          string Fingerprint = (*I).substr(pos+sizeof("VALIDSIG"));
1139 //          if (_config->FindB("Debug::Vendor", false))
1140 //             std::cerr << "Looking for \"" << Fingerprint << "\" in vendor..." <<
1141 //                std::endl;
1142 //          Vndr = List.FindVendor(Fingerprint) != "";
1143 //          if (Vndr != NULL);
1144 //          break;
1145 //       }
1146 //    }
1147    string::size_type pos;
1148
1149    // check for missing sigs (that where not fatal because otherwise we had
1150    // bombed earlier)
1151    string missingkeys;
1152    string msg = _("There is no public key available for the "
1153                   "following key IDs:\n");
1154    pos = Message.find("NO_PUBKEY ");
1155    if (pos != std::string::npos)
1156    {
1157       string::size_type start = pos+strlen("NO_PUBKEY ");
1158       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1159       missingkeys += (Fingerprint);
1160    }
1161    if(!missingkeys.empty())
1162       _error->Warning("%s", string(msg+missingkeys).c_str());
1163
1164    string Transformed = MetaIndexParser->GetExpectedDist();
1165
1166    if (Transformed == "../project/experimental")
1167    {
1168       Transformed = "experimental";
1169    }
1170
1171    pos = Transformed.rfind('/');
1172    if (pos != string::npos)
1173    {
1174       Transformed = Transformed.substr(0, pos);
1175    }
1176
1177    if (Transformed == ".")
1178    {
1179       Transformed = "";
1180    }
1181
1182    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1183    {
1184       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1185       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1186       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1187    }
1188
1189    if (MetaIndexParser->CheckDist(Transformed) == false)
1190    {
1191       // This might become fatal one day
1192 //       Status = StatAuthError;
1193 //       ErrorText = "Conflicting distribution; expected "
1194 //          + MetaIndexParser->GetExpectedDist() + " but got "
1195 //          + MetaIndexParser->GetDist();
1196 //       return false;
1197       if (!Transformed.empty())
1198       {
1199          _error->Warning("Conflicting distribution: %s (expected %s but got %s)",
1200                          Desc.Description.c_str(),
1201                          Transformed.c_str(),
1202                          MetaIndexParser->GetDist().c_str());
1203       }
1204    }
1205
1206    return true;
1207 }
1208                                                                 /*}}}*/
1209 // pkgAcqMetaIndex::Failed - no Release file present or no signature
1210 //      file present                                            /*{{{*/
1211 // ---------------------------------------------------------------------
1212 /* */
1213 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1214 {
1215    if (AuthPass == true)
1216    {
1217       // gpgv method failed, if we have a good signature
1218       string LastGoodSigFile = _config->FindDir("Dir::State::lists") +
1219          "partial/" + URItoFileName(RealURI) + ".gpg.reverify";
1220       if(FileExists(LastGoodSigFile))
1221       {
1222          string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1223             URItoFileName(RealURI) + ".gpg";
1224          Rename(LastGoodSigFile,VerifiedSigFile);
1225          Status = StatTransientNetworkError;
1226          _error->Warning(_("A error occurred during the signature "
1227                            "verification. The repository is not updated "
1228                            "and the previous index files will be used."
1229                            "GPG error: %s: %s\n"),
1230                          Desc.Description.c_str(),
1231                          LookupTag(Message,"Message").c_str());
1232          RunScripts("APT::Update::Auth-Failure");
1233          return;
1234       } else {
1235          _error->Warning(_("GPG error: %s: %s"),
1236                          Desc.Description.c_str(),
1237                          LookupTag(Message,"Message").c_str());
1238       }
1239       // gpgv method failed
1240       ReportMirrorFailure("GPGFailure");
1241    }
1242
1243    // No Release file was present, or verification failed, so fall
1244    // back to queueing Packages files without verification
1245    QueueIndexes(false);
1246 }
1247
1248                                                                         /*}}}*/
1249
1250 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1251 // ---------------------------------------------------------------------
1252 /* This just sets up the initial fetch environment and queues the first
1253    possibilitiy */
1254 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1255                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1256                              string &StoreFilename) :
1257                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1258                StoreFilename(StoreFilename), Vf(Version.FileList()),
1259                Trusted(false)
1260 {
1261    Retries = _config->FindI("Acquire::Retries",0);
1262
1263    if (Version.Arch() == 0)
1264    {
1265       _error->Error(_("I wasn't able to locate a file for the %s package. "
1266                       "This might mean you need to manually fix this package. "
1267                       "(due to missing arch)"),
1268                     Version.ParentPkg().Name());
1269       return;
1270    }
1271
1272    /* We need to find a filename to determine the extension. We make the
1273       assumption here that all the available sources for this version share
1274       the same extension.. */
1275    // Skip not source sources, they do not have file fields.
1276    for (; Vf.end() == false; Vf++)
1277    {
1278       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1279          continue;
1280       break;
1281    }
1282
1283    // Does not really matter here.. we are going to fail out below
1284    if (Vf.end() != true)
1285    {
1286       // If this fails to get a file name we will bomb out below.
1287       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1288       if (_error->PendingError() == true)
1289          return;
1290
1291       // Generate the final file name as: package_version_arch.foo
1292       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1293                       QuoteString(Version.VerStr(),"_:") + '_' +
1294                       QuoteString(Version.Arch(),"_:.") +
1295                       "." + flExtension(Parse.FileName());
1296    }
1297
1298    // check if we have one trusted source for the package. if so, switch
1299    // to "TrustedOnly" mode
1300    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; i++)
1301    {
1302       pkgIndexFile *Index;
1303       if (Sources->FindIndex(i.File(),Index) == false)
1304          continue;
1305       if (_config->FindB("Debug::pkgAcquire::Auth", false))
1306       {
1307          std::cerr << "Checking index: " << Index->Describe()
1308                    << "(Trusted=" << Index->IsTrusted() << ")\n";
1309       }
1310       if (Index->IsTrusted()) {
1311          Trusted = true;
1312          break;
1313       }
1314    }
1315
1316    // "allow-unauthenticated" restores apts old fetching behaviour
1317    // that means that e.g. unauthenticated file:// uris are higher
1318    // priority than authenticated http:// uris
1319    if (_config->FindB("APT::Get::AllowUnauthenticated",false) == true)
1320       Trusted = false;
1321
1322    // Select a source
1323    if (QueueNext() == false && _error->PendingError() == false)
1324       _error->Error(_("I wasn't able to locate file for the %s package. "
1325                     "This might mean you need to manually fix this package."),
1326                     Version.ParentPkg().Name());
1327 }
1328                                                                         /*}}}*/
1329 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1330 // ---------------------------------------------------------------------
1331 /* This queues the next available file version for download. It checks if
1332    the archive is already available in the cache and stashs the MD5 for
1333    checking later. */
1334 bool pkgAcqArchive::QueueNext()
1335 {
1336    for (; Vf.end() == false; Vf++)
1337    {
1338       // Ignore not source sources
1339       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1340          continue;
1341
1342       // Try to cross match against the source list
1343       pkgIndexFile *Index;
1344       if (Sources->FindIndex(Vf.File(),Index) == false)
1345             continue;
1346
1347       // only try to get a trusted package from another source if that source
1348       // is also trusted
1349       if(Trusted && !Index->IsTrusted())
1350          continue;
1351
1352       // Grab the text package record
1353       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1354       if (_error->PendingError() == true)
1355          return false;
1356
1357       string PkgFile = Parse.FileName();
1358       if(Parse.SHA256Hash() != "")
1359          ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
1360       else if (Parse.SHA1Hash() != "")
1361          ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
1362       else
1363          ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1364       if (PkgFile.empty() == true)
1365          return _error->Error(_("The package index files are corrupted. No Filename: "
1366                               "field for package %s."),
1367                               Version.ParentPkg().Name());
1368
1369       Desc.URI = Index->ArchiveURI(PkgFile);
1370       Desc.Description = Index->ArchiveInfo(Version);
1371       Desc.Owner = this;
1372       Desc.ShortDesc = Version.ParentPkg().Name();
1373
1374       // See if we already have the file. (Legacy filenames)
1375       FileSize = Version->Size;
1376       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
1377       struct stat Buf;
1378       if (stat(FinalFile.c_str(),&Buf) == 0)
1379       {
1380          // Make sure the size matches
1381          if ((unsigned)Buf.st_size == Version->Size)
1382          {
1383             Complete = true;
1384             Local = true;
1385             Status = StatDone;
1386             StoreFilename = DestFile = FinalFile;
1387             return true;
1388          }
1389
1390          /* Hmm, we have a file and its size does not match, this means it is
1391             an old style mismatched arch */
1392          unlink(FinalFile.c_str());
1393       }
1394
1395       // Check it again using the new style output filenames
1396       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
1397       if (stat(FinalFile.c_str(),&Buf) == 0)
1398       {
1399          // Make sure the size matches
1400          if ((unsigned)Buf.st_size == Version->Size)
1401          {
1402             Complete = true;
1403             Local = true;
1404             Status = StatDone;
1405             StoreFilename = DestFile = FinalFile;
1406             return true;
1407          }
1408
1409          /* Hmm, we have a file and its size does not match, this shouldnt
1410             happen.. */
1411          unlink(FinalFile.c_str());
1412       }
1413
1414       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
1415
1416       // Check the destination file
1417       if (stat(DestFile.c_str(),&Buf) == 0)
1418       {
1419          // Hmm, the partial file is too big, erase it
1420          if ((unsigned)Buf.st_size > Version->Size)
1421             unlink(DestFile.c_str());
1422          else
1423             PartialSize = Buf.st_size;
1424       }
1425
1426       // Create the item
1427       Local = false;
1428       Desc.URI = Index->ArchiveURI(PkgFile);
1429       Desc.Description = Index->ArchiveInfo(Version);
1430       Desc.Owner = this;
1431       Desc.ShortDesc = Version.ParentPkg().Name();
1432       QueueURI(Desc);
1433
1434       Vf++;
1435       return true;
1436    }
1437    return false;
1438 }
1439                                                                         /*}}}*/
1440 // AcqArchive::Done - Finished fetching                                 /*{{{*/
1441 // ---------------------------------------------------------------------
1442 /* */
1443 void pkgAcqArchive::Done(string Message,unsigned long Size,string CalcHash,
1444                          pkgAcquire::MethodConfig *Cfg)
1445 {
1446    Item::Done(Message,Size,CalcHash,Cfg);
1447
1448    // Check the size
1449    if (Size != Version->Size)
1450    {
1451       Status = StatError;
1452       ErrorText = _("Size mismatch");
1453       return;
1454    }
1455
1456    // Check the hash
1457    if(ExpectedHash.toStr() != CalcHash)
1458    {
1459       Status = StatError;
1460       ErrorText = _("Hash Sum mismatch");
1461       if(FileExists(DestFile))
1462          Rename(DestFile,DestFile + ".FAILED");
1463       return;
1464    }
1465
1466    // Grab the output filename
1467    string FileName = LookupTag(Message,"Filename");
1468    if (FileName.empty() == true)
1469    {
1470       Status = StatError;
1471       ErrorText = "Method gave a blank filename";
1472       return;
1473    }
1474
1475    Complete = true;
1476
1477    // Reference filename
1478    if (FileName != DestFile)
1479    {
1480       StoreFilename = DestFile = FileName;
1481       Local = true;
1482       return;
1483    }
1484
1485    // Done, move it into position
1486    string FinalFile = _config->FindDir("Dir::Cache::Archives");
1487    FinalFile += flNotDir(StoreFilename);
1488    Rename(DestFile,FinalFile);
1489
1490    StoreFilename = DestFile = FinalFile;
1491    Complete = true;
1492 }
1493                                                                         /*}}}*/
1494 // AcqArchive::Failed - Failure handler                                 /*{{{*/
1495 // ---------------------------------------------------------------------
1496 /* Here we try other sources */
1497 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1498 {
1499    ErrorText = LookupTag(Message,"Message");
1500
1501    /* We don't really want to retry on failed media swaps, this prevents
1502       that. An interesting observation is that permanent failures are not
1503       recorded. */
1504    if (Cnf->Removable == true &&
1505        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1506    {
1507       // Vf = Version.FileList();
1508       while (Vf.end() == false) Vf++;
1509       StoreFilename = string();
1510       Item::Failed(Message,Cnf);
1511       return;
1512    }
1513
1514    if (QueueNext() == false)
1515    {
1516       // This is the retry counter
1517       if (Retries != 0 &&
1518           Cnf->LocalOnly == false &&
1519           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1520       {
1521          Retries--;
1522          Vf = Version.FileList();
1523          if (QueueNext() == true)
1524             return;
1525       }
1526
1527       StoreFilename = string();
1528       Item::Failed(Message,Cnf);
1529    }
1530 }
1531                                                                         /*}}}*/
1532 // AcqArchive::IsTrusted - Determine whether this archive comes from a
1533 // trusted source                                                       /*{{{*/
1534 // ---------------------------------------------------------------------
1535 bool pkgAcqArchive::IsTrusted()
1536 {
1537    return Trusted;
1538 }
1539
1540 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
1541 // ---------------------------------------------------------------------
1542 /* */
1543 void pkgAcqArchive::Finished()
1544 {
1545    if (Status == pkgAcquire::Item::StatDone &&
1546        Complete == true)
1547       return;
1548    StoreFilename = string();
1549 }
1550                                                                         /*}}}*/
1551
1552 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
1553 // ---------------------------------------------------------------------
1554 /* The file is added to the queue */
1555 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
1556                        unsigned long Size,string Dsc,string ShortDesc,
1557                        const string &DestDir, const string &DestFilename) :
1558                        Item(Owner), ExpectedHash(Hash)
1559 {
1560    Retries = _config->FindI("Acquire::Retries",0);
1561
1562    if(!DestFilename.empty())
1563       DestFile = DestFilename;
1564    else if(!DestDir.empty())
1565       DestFile = DestDir + "/" + flNotDir(URI);
1566    else
1567       DestFile = flNotDir(URI);
1568
1569    // Create the item
1570    Desc.URI = URI;
1571    Desc.Description = Dsc;
1572    Desc.Owner = this;
1573
1574    // Set the short description to the archive component
1575    Desc.ShortDesc = ShortDesc;
1576
1577    // Get the transfer sizes
1578    FileSize = Size;
1579    struct stat Buf;
1580    if (stat(DestFile.c_str(),&Buf) == 0)
1581    {
1582       // Hmm, the partial file is too big, erase it
1583       if ((unsigned)Buf.st_size > Size)
1584          unlink(DestFile.c_str());
1585       else
1586          PartialSize = Buf.st_size;
1587    }
1588
1589    QueueURI(Desc);
1590 }
1591                                                                         /*}}}*/
1592 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
1593 // ---------------------------------------------------------------------
1594 /* */
1595 void pkgAcqFile::Done(string Message,unsigned long Size,string CalcHash,
1596                       pkgAcquire::MethodConfig *Cnf)
1597 {
1598    Item::Done(Message,Size,CalcHash,Cnf);
1599
1600    // Check the hash
1601    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
1602    {
1603       Status = StatError;
1604       ErrorText = "Hash Sum mismatch";
1605       Rename(DestFile,DestFile + ".FAILED");
1606       return;
1607    }
1608
1609    string FileName = LookupTag(Message,"Filename");
1610    if (FileName.empty() == true)
1611    {
1612       Status = StatError;
1613       ErrorText = "Method gave a blank filename";
1614       return;
1615    }
1616
1617    Complete = true;
1618
1619    // The files timestamp matches
1620    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1621       return;
1622
1623    // We have to copy it into place
1624    if (FileName != DestFile)
1625    {
1626       Local = true;
1627       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
1628           Cnf->Removable == true)
1629       {
1630          Desc.URI = "copy:" + FileName;
1631          QueueURI(Desc);
1632          return;
1633       }
1634
1635       // Erase the file if it is a symlink so we can overwrite it
1636       struct stat St;
1637       if (lstat(DestFile.c_str(),&St) == 0)
1638       {
1639          if (S_ISLNK(St.st_mode) != 0)
1640             unlink(DestFile.c_str());
1641       }
1642
1643       // Symlink the file
1644       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
1645       {
1646          ErrorText = "Link to " + DestFile + " failure ";
1647          Status = StatError;
1648          Complete = false;
1649       }
1650    }
1651 }
1652                                                                         /*}}}*/
1653 // AcqFile::Failed - Failure handler                                    /*{{{*/
1654 // ---------------------------------------------------------------------
1655 /* Here we try other sources */
1656 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1657 {
1658    ErrorText = LookupTag(Message,"Message");
1659
1660    // This is the retry counter
1661    if (Retries != 0 &&
1662        Cnf->LocalOnly == false &&
1663        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1664    {
1665       Retries--;
1666       QueueURI(Desc);
1667       return;
1668    }
1669
1670    Item::Failed(Message,Cnf);
1671 }
1672                                                                         /*}}}*/