apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <apt-pkg/acquire-item.h>
  17 #include <apt-pkg/configuration.h>
  18 #include <apt-pkg/sourcelist.h>
  19 #include <apt-pkg/vendorlist.h>
  20 #include <apt-pkg/error.h>
  21 #include <apt-pkg/strutl.h>
  22 #include <apt-pkg/fileutl.h>
  23 #include <apt-pkg/md5.h>
  24 #include <apt-pkg/sha1.h>
  25 #include <apt-pkg/tagfile.h>
  26
  27 #include <apti18n.h>
  28
  29 #include <sys/stat.h>
  30 #include <unistd.h>
  31 #include <errno.h>
  32 #include <string>
  33 #include <sstream>
  34 #include <stdio.h>
  35                                                                         /*}}}*/
  36
  37 using namespace std;
  38
  39 // Acquire::Item::Item - Constructor                                    /*{{{*/
  40 // ---------------------------------------------------------------------
  41 /* */
  42 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  43                        PartialSize(0), Mode(0), ID(0), Complete(false),
  44                        Local(false), QueueCounter(0)
  45 {
  46    Owner->Add(this);
  47    Status = StatIdle;
  48 }
  49                                                                         /*}}}*/
  50 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  51 // ---------------------------------------------------------------------
  52 /* */
  53 pkgAcquire::Item::~Item()
  54 {
  55    Owner->Remove(this);
  56 }
  57                                                                         /*}}}*/
  58 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  59 // ---------------------------------------------------------------------
  60 /* We return to an idle state if there are still other queues that could
  61    fetch this object */
  62 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  63 {
  64    Status = StatIdle;
  65    ErrorText = LookupTag(Message,"Message");
  66    UsedMirror =  LookupTag(Message,"UsedMirror");
  67    if (QueueCounter <= 1)
  68    {
  69       /* This indicates that the file is not available right now but might
  70          be sometime later. If we do a retry cycle then this should be
  71          retried [CDROMs] */
  72       if (Cnf->LocalOnly == true &&
  73           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  74       {
  75          Status = StatIdle;
  76          Dequeue();
  77          return;
  78       }
  79
  80       Status = StatError;
  81       Dequeue();
  82    }
  83
  84    // report mirror failure back to LP if we actually use a mirror
  85    string FailReason = LookupTag(Message, "FailReason");
  86    if(FailReason.size() != 0)
  87       ReportMirrorFailure(FailReason);
  88    else
  89       ReportMirrorFailure(ErrorText);
  90 }
  91                                                                         /*}}}*/
  92 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
  93 // ---------------------------------------------------------------------
  94 /* Stash status and the file size. Note that setting Complete means
  95    sub-phases of the acquire process such as decompresion are operating */
  96 void pkgAcquire::Item::Start(string /*Message*/,unsigned long Size)
  97 {
  98    Status = StatFetching;
  99    if (FileSize == 0 && Complete == false)
 100       FileSize = Size;
 101 }
 102                                                                         /*}}}*/
 103 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
 104 // ---------------------------------------------------------------------
 105 /* */
 106 void pkgAcquire::Item::Done(string Message,unsigned long Size,string Hash,
 107                             pkgAcquire::MethodConfig *Cnf)
 108 {
 109    // We just downloaded something..
 110    string FileName = LookupTag(Message,"Filename");
 111    UsedMirror =  LookupTag(Message,"UsedMirror");
 112    if (Complete == false && !Local && FileName == DestFile)
 113    {
 114       if (Owner->Log != 0)
 115          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 116    }
 117
 118    if (FileSize == 0)
 119       FileSize= Size;
 120    Status = StatDone;
 121    ErrorText = string();
 122    Owner->Dequeue(this);
 123 }
 124                                                                         /*}}}*/
 125 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 126 // ---------------------------------------------------------------------
 127 /* This helper function is used by alot of item methods as thier final
 128    step */
 129 void pkgAcquire::Item::Rename(string From,string To)
 130 {
 131    if (rename(From.c_str(),To.c_str()) != 0)
 132    {
 133       char S[300];
 134       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 135               From.c_str(),To.c_str());
 136       Status = StatError;
 137       ErrorText = S;
 138    }
 139 }
 140                                                                         /*}}}*/
 141
 142 void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
 143 {
 144    // we only act if a mirror was used at all
 145    if(UsedMirror.empty())
 146       return;
 147 #if 0
 148    std::cerr << "\nReportMirrorFailure: "
 149              << UsedMirror
 150              << " Uri: " << DescURI()
 151              << " FailCode: "
 152              << FailCode << std::endl;
 153 #endif
 154    const char *Args[40];
 155    unsigned int i = 0;
 156    string report = _config->Find("Methods::Mirror::ProblemReporting",
 157                                  "/usr/lib/apt/apt-report-mirror-failure");
 158    if(!FileExists(report))
 159       return;
 160    Args[i++] = report.c_str();
 161    Args[i++] = UsedMirror.c_str();
 162    Args[i++] = DescURI().c_str();
 163    Args[i++] = FailCode.c_str();
 164    Args[i++] = NULL;
 165    pid_t pid = ExecFork();
 166    if(pid < 0)
 167    {
 168       _error->Error("ReportMirrorFailure Fork failed");
 169       return;
 170    }
 171    else if(pid == 0)
 172    {
 173       execvp(Args[0], (char**)Args);
 174       std::cerr << "Could not exec " << Args[0] << std::endl;
 175       _exit(100);
 176    }
 177    if(!ExecWait(pid, "report-mirror-failure"))
 178    {
 179       _error->Warning("Couldn't report problem to '%s'",
 180                       _config->Find("Methods::Mirror::ProblemReporting").c_str());
 181    }
 182 }
 183
 184
 185
 186 // AcqDiffIndex::AcqDiffIndex - Constructor
 187 // ---------------------------------------------------------------------
 188 /* Get the DiffIndex file first and see if there are patches availabe
 189  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 190  * patches. If anything goes wrong in that process, it will fall back to
 191  * the original packages file
 192  */
 193 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 194                                  string URI,string URIDesc,string ShortDesc,
 195                                  HashString ExpectedHash)
 196    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 197      Description(URIDesc)
 198 {
 199
 200    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 201
 202    Desc.Description = URIDesc + "/DiffIndex";
 203    Desc.Owner = this;
 204    Desc.ShortDesc = ShortDesc;
 205    Desc.URI = URI + ".diff/Index";
 206
 207    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 208    DestFile += URItoFileName(URI) + string(".DiffIndex");
 209
 210    if(Debug)
 211       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 212
 213    // look for the current package file
 214    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 215    CurrentPackagesFile += URItoFileName(RealURI);
 216
 217    // FIXME: this file:/ check is a hack to prevent fetching
 218    //        from local sources. this is really silly, and
 219    //        should be fixed cleanly as soon as possible
 220    if(!FileExists(CurrentPackagesFile) ||
 221       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 222    {
 223       // we don't have a pkg file or we don't want to queue
 224       if(Debug)
 225          std::clog << "No index file, local or canceld by user" << std::endl;
 226       Failed("", NULL);
 227       return;
 228    }
 229
 230    if(Debug)
 231       std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): "
 232                 << CurrentPackagesFile << std::endl;
 233
 234    QueueURI(Desc);
 235
 236 }
 237
 238 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 239 // ---------------------------------------------------------------------
 240 /* The only header we use is the last-modified header. */
 241 string pkgAcqDiffIndex::Custom600Headers()
 242 {
 243    string Final = _config->FindDir("Dir::State::lists");
 244    Final += URItoFileName(RealURI) + string(".IndexDiff");
 245
 246    if(Debug)
 247       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 248
 249    struct stat Buf;
 250    if (stat(Final.c_str(),&Buf) != 0)
 251       return "\nIndex-File: true";
 252
 253    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 254 }
 255
 256
 257 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)
 258 {
 259    if(Debug)
 260       std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile
 261                 << std::endl;
 262
 263    pkgTagSection Tags;
 264    string ServerSha1;
 265    vector<DiffInfo> available_patches;
 266
 267    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 268    pkgTagFile TF(&Fd);
 269    if (_error->PendingError() == true)
 270       return false;
 271
 272    if(TF.Step(Tags) == true)
 273    {
 274       string local_sha1;
 275       bool found = false;
 276       DiffInfo d;
 277       string size;
 278
 279       string tmp = Tags.FindS("SHA1-Current");
 280       std::stringstream ss(tmp);
 281       ss >> ServerSha1;
 282
 283       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 284       SHA1Summation SHA1;
 285       SHA1.AddFD(fd.Fd(), fd.Size());
 286       local_sha1 = string(SHA1.Result());
 287
 288       if(local_sha1 == ServerSha1)
 289       {
 290          // we have the same sha1 as the server
 291          if(Debug)
 292             std::clog << "Package file is up-to-date" << std::endl;
 293          // set found to true, this will queue a pkgAcqIndexDiffs with
 294          // a empty availabe_patches
 295          found = true;
 296       }
 297       else
 298       {
 299          if(Debug)
 300             std::clog << "SHA1-Current: " << ServerSha1 << std::endl;
 301
 302          // check the historie and see what patches we need
 303          string history = Tags.FindS("SHA1-History");
 304          std::stringstream hist(history);
 305          while(hist >> d.sha1 >> size >> d.file)
 306          {
 307             d.size = atoi(size.c_str());
 308             // read until the first match is found
 309             if(d.sha1 == local_sha1)
 310                found=true;
 311             // from that point on, we probably need all diffs
 312             if(found)
 313             {
 314                if(Debug)
 315                   std::clog << "Need to get diff: " << d.file << std::endl;
 316                available_patches.push_back(d);
 317             }
 318          }
 319       }
 320
 321       // we have something, queue the next diff
 322       if(found)
 323       {
 324          // queue the diffs
 325          string::size_type last_space = Description.rfind(" ");
 326          if(last_space != string::npos)
 327             Description.erase(last_space, Description.size()-last_space);
 328          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 329                               ExpectedHash, available_patches);
 330          Complete = false;
 331          Status = StatDone;
 332          Dequeue();
 333          return true;
 334       }
 335    }
 336
 337    // Nothing found, report and return false
 338    // Failing here is ok, if we return false later, the full
 339    // IndexFile is queued
 340    if(Debug)
 341       std::clog << "Can't find a patch in the index file" << std::endl;
 342    return false;
 343 }
 344
 345 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 346 {
 347    if(Debug)
 348       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << std::endl
 349                 << "Falling back to normal index file aquire" << std::endl;
 350
 351    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 352                    ExpectedHash);
 353
 354    Complete = false;
 355    Status = StatDone;
 356    Dequeue();
 357 }
 358
 359 void pkgAcqDiffIndex::Done(string Message,unsigned long Size,string Md5Hash,
 360                            pkgAcquire::MethodConfig *Cnf)
 361 {
 362    if(Debug)
 363       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 364
 365    Item::Done(Message,Size,Md5Hash,Cnf);
 366
 367    string FinalFile;
 368    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 369
 370    // sucess in downloading the index
 371    // rename the index
 372    FinalFile += string(".IndexDiff");
 373    if(Debug)
 374       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 375                 << std::endl;
 376    Rename(DestFile,FinalFile);
 377    chmod(FinalFile.c_str(),0644);
 378    DestFile = FinalFile;
 379
 380    if(!ParseDiffIndex(DestFile))
 381       return Failed("", NULL);
 382
 383    Complete = true;
 384    Status = StatDone;
 385    Dequeue();
 386    return;
 387 }
 388
 389
 390
 391 // AcqIndexDiffs::AcqIndexDiffs - Constructor
 392 // ---------------------------------------------------------------------
 393 /* The package diff is added to the queue. one object is constructed
 394  * for each diff and the index
 395  */
 396 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 397                                    string URI,string URIDesc,string ShortDesc,
 398                                    HashString ExpectedHash,
 399                                    vector<DiffInfo> diffs)
 400    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 401      available_patches(diffs)
 402 {
 403
 404    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 405    DestFile += URItoFileName(URI);
 406
 407    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 408
 409    Description = URIDesc;
 410    Desc.Owner = this;
 411    Desc.ShortDesc = ShortDesc;
 412
 413    if(available_patches.size() == 0)
 414    {
 415       // we are done (yeah!)
 416       Finish(true);
 417    }
 418    else
 419    {
 420       // get the next diff
 421       State = StateFetchDiff;
 422       QueueNextDiff();
 423    }
 424 }
 425
 426
 427 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 428 {
 429    if(Debug)
 430       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl
 431                 << "Falling back to normal index file aquire" << std::endl;
 432    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 433                    ExpectedHash);
 434    Finish();
 435 }
 436
 437
 438 // helper that cleans the item out of the fetcher queue
 439 void pkgAcqIndexDiffs::Finish(bool allDone)
 440 {
 441    // we restore the original name, this is required, otherwise
 442    // the file will be cleaned
 443    if(allDone)
 444    {
 445       DestFile = _config->FindDir("Dir::State::lists");
 446       DestFile += URItoFileName(RealURI);
 447
 448       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 449       {
 450          Status = StatAuthError;
 451          ErrorText = _("MD5Sum mismatch");
 452          Rename(DestFile,DestFile + ".FAILED");
 453          Dequeue();
 454          return;
 455       }
 456
 457       // this is for the "real" finish
 458       Complete = true;
 459       Status = StatDone;
 460       Dequeue();
 461       if(Debug)
 462          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 463       return;
 464    }
 465
 466    if(Debug)
 467       std::clog << "Finishing: " << Desc.URI << std::endl;
 468    Complete = false;
 469    Status = StatDone;
 470    Dequeue();
 471    return;
 472 }
 473
 474
 475
 476 bool pkgAcqIndexDiffs::QueueNextDiff()
 477 {
 478
 479    // calc sha1 of the just patched file
 480    string FinalFile = _config->FindDir("Dir::State::lists");
 481    FinalFile += URItoFileName(RealURI);
 482
 483    FileFd fd(FinalFile, FileFd::ReadOnly);
 484    SHA1Summation SHA1;
 485    SHA1.AddFD(fd.Fd(), fd.Size());
 486    string local_sha1 = string(SHA1.Result());
 487    if(Debug)
 488       std::clog << "QueueNextDiff: "
 489                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 490
 491    // remove all patches until the next matching patch is found
 492    // this requires the Index file to be ordered
 493    for(vector<DiffInfo>::iterator I=available_patches.begin();
 494        available_patches.size() > 0 &&
 495           I != available_patches.end() &&
 496           (*I).sha1 != local_sha1;
 497        I++)
 498    {
 499       available_patches.erase(I);
 500    }
 501
 502    // error checking and falling back if no patch was found
 503    if(available_patches.size() == 0)
 504    {
 505       Failed("", NULL);
 506       return false;
 507    }
 508
 509    // queue the right diff
 510    Desc.URI = string(RealURI) + ".diff/" + available_patches[0].file + ".gz";
 511    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 512    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 513    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 514
 515    if(Debug)
 516       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 517
 518    QueueURI(Desc);
 519
 520    return true;
 521 }
 522
 523
 524
 525 void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash,
 526                             pkgAcquire::MethodConfig *Cnf)
 527 {
 528    if(Debug)
 529       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 530
 531    Item::Done(Message,Size,Md5Hash,Cnf);
 532
 533    string FinalFile;
 534    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 535
 536    // sucess in downloading a diff, enter ApplyDiff state
 537    if(State == StateFetchDiff)
 538    {
 539
 540       if(Debug)
 541          std::clog << "Sending to gzip method: " << FinalFile << std::endl;
 542
 543       string FileName = LookupTag(Message,"Filename");
 544       State = StateUnzipDiff;
 545       Local = true;
 546       Desc.URI = "gzip:" + FileName;
 547       DestFile += ".decomp";
 548       QueueURI(Desc);
 549       Mode = "gzip";
 550       return;
 551    }
 552
 553    // sucess in downloading a diff, enter ApplyDiff state
 554    if(State == StateUnzipDiff)
 555    {
 556
 557       // rred excepts the patch as $FinalFile.ed
 558       Rename(DestFile,FinalFile+".ed");
 559
 560       if(Debug)
 561          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 562
 563       State = StateApplyDiff;
 564       Local = true;
 565       Desc.URI = "rred:" + FinalFile;
 566       QueueURI(Desc);
 567       Mode = "rred";
 568       return;
 569    }
 570
 571
 572    // success in download/apply a diff, queue next (if needed)
 573    if(State == StateApplyDiff)
 574    {
 575       // remove the just applied patch
 576       available_patches.erase(available_patches.begin());
 577
 578       // move into place
 579       if(Debug)
 580       {
 581          std::clog << "Moving patched file in place: " << std::endl
 582                    << DestFile << " -> " << FinalFile << std::endl;
 583       }
 584       Rename(DestFile,FinalFile);
 585       chmod(FinalFile.c_str(),0644);
 586
 587       // see if there is more to download
 588       if(available_patches.size() > 0) {
 589          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 590                               ExpectedHash, available_patches);
 591          return Finish();
 592       } else
 593          return Finish(true);
 594    }
 595 }
 596
 597
 598 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 599 // ---------------------------------------------------------------------
 600 /* The package file is added to the queue and a second class is
 601    instantiated to fetch the revision file */
 602 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 603                          string URI,string URIDesc,string ShortDesc,
 604                          HashString ExpectedHash, string comprExt)
 605    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 606 {
 607    Decompression = false;
 608    Erase = false;
 609
 610    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 611    DestFile += URItoFileName(URI);
 612
 613    if(comprExt.empty())
 614    {
 615       // autoselect the compression method
 616       if(FileExists("/bin/bzip2"))
 617          CompressionExtension = ".bz2";
 618       else
 619          CompressionExtension = ".gz";
 620    } else {
 621       CompressionExtension = comprExt;
 622    }
 623    Desc.URI = URI + CompressionExtension;
 624
 625    Desc.Description = URIDesc;
 626    Desc.Owner = this;
 627    Desc.ShortDesc = ShortDesc;
 628
 629    QueueURI(Desc);
 630 }
 631                                                                         /*}}}*/
 632 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 633 // ---------------------------------------------------------------------
 634 /* The only header we use is the last-modified header. */
 635 string pkgAcqIndex::Custom600Headers()
 636 {
 637    string Final = _config->FindDir("Dir::State::lists");
 638    Final += URItoFileName(RealURI);
 639
 640    struct stat Buf;
 641    if (stat(Final.c_str(),&Buf) != 0)
 642       return "\nIndex-File: true";
 643    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 644 }
 645                                                                         /*}}}*/
 646
 647 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 648 {
 649    // no .bz2 found, retry with .gz
 650    if(Desc.URI.substr(Desc.URI.size()-3) == "bz2") {
 651       Desc.URI = Desc.URI.substr(0,Desc.URI.size()-3) + "gz";
 652
 653       // retry with a gzip one
 654       new pkgAcqIndex(Owner, RealURI, Desc.Description,Desc.ShortDesc,
 655                       ExpectedHash, string(".gz"));
 656       Status = StatDone;
 657       Complete = false;
 658       Dequeue();
 659       return;
 660    }
 661
 662    // on decompression failure, remove bad versions in partial/
 663    if(Decompression && Erase) {
 664       string s = _config->FindDir("Dir::State::lists") + "partial/";
 665       s += URItoFileName(RealURI);
 666       unlink(s.c_str());
 667    }
 668
 669    Item::Failed(Message,Cnf);
 670 }
 671
 672
 673 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
 674 // ---------------------------------------------------------------------
 675 /* This goes through a number of states.. On the initial fetch the
 676    method could possibly return an alternate filename which points
 677    to the uncompressed version of the file. If this is so the file
 678    is copied into the partial directory. In all other cases the file
 679    is decompressed with a gzip uri. */
 680 void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash,
 681                        pkgAcquire::MethodConfig *Cfg)
 682 {
 683    Item::Done(Message,Size,Hash,Cfg);
 684
 685    if (Decompression == true)
 686    {
 687       if (_config->FindB("Debug::pkgAcquire::Auth", false))
 688       {
 689          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
 690          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
 691       }
 692
 693       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
 694       {
 695          Status = StatAuthError;
 696          ErrorText = _("Hash Sum mismatch");
 697          Rename(DestFile,DestFile + ".FAILED");
 698          ReportMirrorFailure("HashChecksumFailure");
 699          return;
 700       }
 701       // Done, move it into position
 702       string FinalFile = _config->FindDir("Dir::State::lists");
 703       FinalFile += URItoFileName(RealURI);
 704       Rename(DestFile,FinalFile);
 705       chmod(FinalFile.c_str(),0644);
 706
 707       /* We restore the original name to DestFile so that the clean operation
 708          will work OK */
 709       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 710       DestFile += URItoFileName(RealURI);
 711
 712       // Remove the compressed version.
 713       if (Erase == true)
 714          unlink(DestFile.c_str());
 715       return;
 716    }
 717
 718    Erase = false;
 719    Complete = true;
 720
 721    // Handle the unzipd case
 722    string FileName = LookupTag(Message,"Alt-Filename");
 723    if (FileName.empty() == false)
 724    {
 725       // The files timestamp matches
 726       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
 727          return;
 728       Decompression = true;
 729       Local = true;
 730       DestFile += ".decomp";
 731       Desc.URI = "copy:" + FileName;
 732       QueueURI(Desc);
 733       Mode = "copy";
 734       return;
 735    }
 736
 737    FileName = LookupTag(Message,"Filename");
 738    if (FileName.empty() == true)
 739    {
 740       Status = StatError;
 741       ErrorText = "Method gave a blank filename";
 742    }
 743
 744    // The files timestamp matches
 745    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 746       return;
 747
 748    if (FileName == DestFile)
 749       Erase = true;
 750    else
 751       Local = true;
 752
 753    string compExt = Desc.URI.substr(Desc.URI.size()-3);
 754    const char *decompProg;
 755    if(compExt == "bz2")
 756       decompProg = "bzip2";
 757    else if(compExt == ".gz")
 758       decompProg = "gzip";
 759    else {
 760       _error->Error("Unsupported extension: %s", compExt.c_str());
 761       return;
 762    }
 763
 764    Decompression = true;
 765    DestFile += ".decomp";
 766    Desc.URI = string(decompProg) + ":" + FileName;
 767    QueueURI(Desc);
 768    Mode = decompProg;
 769 }
 770
 771 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
 772 // ---------------------------------------------------------------------
 773 /* The Translation file is added to the queue */
 774 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
 775                             string URI,string URIDesc,string ShortDesc)
 776   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
 777 {
 778 }
 779
 780                                                                         /*}}}*/
 781 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
 782 // ---------------------------------------------------------------------
 783 /* */
 784 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 785 {
 786    if (Cnf->LocalOnly == true ||
 787        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 788    {
 789       // Ignore this
 790       Status = StatDone;
 791       Complete = false;
 792       Dequeue();
 793       return;
 794    }
 795
 796    Item::Failed(Message,Cnf);
 797 }
 798                                                                         /*}}}*/
 799
 800 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,
 801                              string URI,string URIDesc,string ShortDesc,
 802                              string MetaIndexURI, string MetaIndexURIDesc,
 803                              string MetaIndexShortDesc,
 804                              const vector<IndexTarget*>* IndexTargets,
 805                              indexRecords* MetaIndexParser) :
 806    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
 807    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
 808    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
 809 {
 810    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 811    DestFile += URItoFileName(URI);
 812
 813    // remove any partial downloaded sig-file in partial/.
 814    // it may confuse proxies and is too small to warrant a
 815    // partial download anyway
 816    unlink(DestFile.c_str());
 817
 818    // Create the item
 819    Desc.Description = URIDesc;
 820    Desc.Owner = this;
 821    Desc.ShortDesc = ShortDesc;
 822    Desc.URI = URI;
 823
 824    string Final = _config->FindDir("Dir::State::lists");
 825    Final += URItoFileName(RealURI);
 826    struct stat Buf;
 827    if (stat(Final.c_str(),&Buf) == 0)
 828    {
 829       // File was already in place.  It needs to be re-downloaded/verified
 830       // because Release might have changed, we do give it a differnt
 831       // name than DestFile because otherwise the http method will
 832       // send If-Range requests and there are too many broken servers
 833       // out there that do not understand them
 834       LastGoodSig = DestFile+".reverify";
 835       Rename(Final,LastGoodSig);
 836    }
 837
 838    QueueURI(Desc);
 839 }
 840                                                                         /*}}}*/
 841 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
 842 // ---------------------------------------------------------------------
 843 /* The only header we use is the last-modified header. */
 844 string pkgAcqMetaSig::Custom600Headers()
 845 {
 846    struct stat Buf;
 847    if (stat(LastGoodSig.c_str(),&Buf) != 0)
 848       return "\nIndex-File: true";
 849
 850    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 851 }
 852
 853 void pkgAcqMetaSig::Done(string Message,unsigned long Size,string MD5,
 854                          pkgAcquire::MethodConfig *Cfg)
 855 {
 856    Item::Done(Message,Size,MD5,Cfg);
 857
 858    string FileName = LookupTag(Message,"Filename");
 859    if (FileName.empty() == true)
 860    {
 861       Status = StatError;
 862       ErrorText = "Method gave a blank filename";
 863       return;
 864    }
 865
 866    if (FileName != DestFile)
 867    {
 868       // We have to copy it into place
 869       Local = true;
 870       Desc.URI = "copy:" + FileName;
 871       QueueURI(Desc);
 872       return;
 873    }
 874
 875    Complete = true;
 876
 877    // put the last known good file back on i-m-s hit (it will
 878    // be re-verified again)
 879    // Else do nothing, we have the new file in DestFile then
 880    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 881       Rename(LastGoodSig, DestFile);
 882
 883    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
 884    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
 885                        DestFile, IndexTargets, MetaIndexParser);
 886
 887 }
 888                                                                         /*}}}*/
 889 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 890 {
 891    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 892
 893    // if we get a network error we fail gracefully
 894    if(Status == StatTransientNetworkError)
 895    {
 896       Item::Failed(Message,Cnf);
 897       // move the sigfile back on transient network failures
 898       if(FileExists(DestFile))
 899          Rename(LastGoodSig,Final);
 900
 901       // set the status back to , Item::Failed likes to reset it
 902       Status = pkgAcquire::Item::StatTransientNetworkError;
 903       return;
 904    }
 905
 906    // Delete any existing sigfile when the acquire failed
 907    unlink(Final.c_str());
 908
 909    // queue a pkgAcqMetaIndex with no sigfile
 910    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
 911                        "", IndexTargets, MetaIndexParser);
 912
 913    if (Cnf->LocalOnly == true ||
 914        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 915    {
 916       // Ignore this
 917       Status = StatDone;
 918       Complete = false;
 919       Dequeue();
 920       return;
 921    }
 922
 923    Item::Failed(Message,Cnf);
 924 }
 925
 926 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,
 927                                  string URI,string URIDesc,string ShortDesc,
 928                                  string SigFile,
 929                                  const vector<struct IndexTarget*>* IndexTargets,
 930                                  indexRecords* MetaIndexParser) :
 931    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
 932    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
 933 {
 934    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 935    DestFile += URItoFileName(URI);
 936
 937    // Create the item
 938    Desc.Description = URIDesc;
 939    Desc.Owner = this;
 940    Desc.ShortDesc = ShortDesc;
 941    Desc.URI = URI;
 942
 943    QueueURI(Desc);
 944 }
 945
 946                                                                         /*}}}*/
 947 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
 948 // ---------------------------------------------------------------------
 949 /* The only header we use is the last-modified header. */
 950 string pkgAcqMetaIndex::Custom600Headers()
 951 {
 952    string Final = _config->FindDir("Dir::State::lists");
 953    Final += URItoFileName(RealURI);
 954
 955    struct stat Buf;
 956    if (stat(Final.c_str(),&Buf) != 0)
 957       return "\nIndex-File: true";
 958
 959    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 960 }
 961
 962 void pkgAcqMetaIndex::Done(string Message,unsigned long Size,string Hash,
 963                            pkgAcquire::MethodConfig *Cfg)
 964 {
 965    Item::Done(Message,Size,Hash,Cfg);
 966
 967    // MetaIndexes are done in two passes: one to download the
 968    // metaindex with an appropriate method, and a second to verify it
 969    // with the gpgv method
 970
 971    if (AuthPass == true)
 972    {
 973       AuthDone(Message);
 974    }
 975    else
 976    {
 977       RetrievalDone(Message);
 978       if (!Complete)
 979          // Still more retrieving to do
 980          return;
 981
 982       if (SigFile == "")
 983       {
 984          // There was no signature file, so we are finished.  Download
 985          // the indexes without verification.
 986          QueueIndexes(false);
 987       }
 988       else
 989       {
 990          // There was a signature file, so pass it to gpgv for
 991          // verification
 992
 993          if (_config->FindB("Debug::pkgAcquire::Auth", false))
 994             std::cerr << "Metaindex acquired, queueing gpg verification ("
 995                       << SigFile << "," << DestFile << ")\n";
 996          AuthPass = true;
 997          Desc.URI = "gpgv:" + SigFile;
 998          QueueURI(Desc);
 999          Mode = "gpgv";
1000       }
1001    }
1002 }
1003
1004 void pkgAcqMetaIndex::RetrievalDone(string Message)
1005 {
1006    // We have just finished downloading a Release file (it is not
1007    // verified yet)
1008
1009    string FileName = LookupTag(Message,"Filename");
1010    if (FileName.empty() == true)
1011    {
1012       Status = StatError;
1013       ErrorText = "Method gave a blank filename";
1014       return;
1015    }
1016
1017    if (FileName != DestFile)
1018    {
1019       Local = true;
1020       Desc.URI = "copy:" + FileName;
1021       QueueURI(Desc);
1022       return;
1023    }
1024
1025    // see if the download was a IMSHit
1026    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1027    Complete = true;
1028
1029    string FinalFile = _config->FindDir("Dir::State::lists");
1030    FinalFile += URItoFileName(RealURI);
1031
1032    // If we get a IMS hit we can remove the empty file in partial
1033    // othersie we move the file in place
1034    if (IMSHit)
1035       unlink(DestFile.c_str());
1036    else
1037       Rename(DestFile,FinalFile);
1038
1039    chmod(FinalFile.c_str(),0644);
1040    DestFile = FinalFile;
1041 }
1042
1043 void pkgAcqMetaIndex::AuthDone(string Message)
1044 {
1045    // At this point, the gpgv method has succeeded, so there is a
1046    // valid signature from a key in the trusted keyring.  We
1047    // perform additional verification of its contents, and use them
1048    // to verify the indexes we are about to download
1049
1050    if (!MetaIndexParser->Load(DestFile))
1051    {
1052       Status = StatAuthError;
1053       ErrorText = MetaIndexParser->ErrorText;
1054       return;
1055    }
1056
1057    if (!VerifyVendor(Message))
1058    {
1059       return;
1060    }
1061
1062    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1063       std::cerr << "Signature verification succeeded: "
1064                 << DestFile << std::endl;
1065
1066    // Download further indexes with verification
1067    QueueIndexes(true);
1068
1069    // Done, move signature file into position
1070
1071    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1072       URItoFileName(RealURI) + ".gpg";
1073    Rename(SigFile,VerifiedSigFile);
1074    chmod(VerifiedSigFile.c_str(),0644);
1075 }
1076
1077 void pkgAcqMetaIndex::QueueIndexes(bool verify)
1078 {
1079    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1080         Target != IndexTargets->end();
1081         Target++)
1082    {
1083       HashString ExpectedIndexHash;
1084       if (verify)
1085       {
1086          const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1087          if (!Record)
1088          {
1089             Status = StatAuthError;
1090             ErrorText = "Unable to find expected entry  "
1091                + (*Target)->MetaKey + " in Meta-index file (malformed Release file?)";
1092             return;
1093          }
1094          ExpectedIndexHash = Record->Hash;
1095          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1096          {
1097             std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1098             std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1099          }
1100          if (ExpectedIndexHash.empty())
1101          {
1102             Status = StatAuthError;
1103             ErrorText = "Unable to find hash sum for "
1104                + (*Target)->MetaKey + " in Meta-index file";
1105             return;
1106          }
1107       }
1108
1109       // Queue Packages file (either diff or full packages files, depending
1110       // on the users option)
1111       if(_config->FindB("Acquire::PDiffs",false) == true)
1112          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1113                              (*Target)->ShortDesc, ExpectedIndexHash);
1114       else
1115          new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description,
1116                             (*Target)->ShortDesc, ExpectedIndexHash);
1117    }
1118 }
1119
1120 bool pkgAcqMetaIndex::VerifyVendor(string Message)
1121 {
1122 //    // Maybe this should be made available from above so we don't have
1123 //    // to read and parse it every time?
1124 //    pkgVendorList List;
1125 //    List.ReadMainList();
1126
1127 //    const Vendor* Vndr = NULL;
1128 //    for (std::vector<string>::const_iterator I = GPGVOutput.begin(); I != GPGVOutput.end(); I++)
1129 //    {
1130 //       string::size_type pos = (*I).find("VALIDSIG ");
1131 //       if (_config->FindB("Debug::Vendor", false))
1132 //          std::cerr << "Looking for VALIDSIG in \"" << (*I) << "\": pos " << pos
1133 //                    << std::endl;
1134 //       if (pos != std::string::npos)
1135 //       {
1136 //          string Fingerprint = (*I).substr(pos+sizeof("VALIDSIG"));
1137 //          if (_config->FindB("Debug::Vendor", false))
1138 //             std::cerr << "Looking for \"" << Fingerprint << "\" in vendor..." <<
1139 //                std::endl;
1140 //          Vndr = List.FindVendor(Fingerprint) != "";
1141 //          if (Vndr != NULL);
1142 //          break;
1143 //       }
1144 //    }
1145    string::size_type pos;
1146
1147    // check for missing sigs (that where not fatal because otherwise we had
1148    // bombed earlier)
1149    string missingkeys;
1150    string msg = _("There is no public key available for the "
1151                   "following key IDs:\n");
1152    pos = Message.find("NO_PUBKEY ");
1153    if (pos != std::string::npos)
1154    {
1155       string::size_type start = pos+strlen("NO_PUBKEY ");
1156       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1157       missingkeys += (Fingerprint);
1158    }
1159    if(!missingkeys.empty())
1160       _error->Warning("%s", string(msg+missingkeys).c_str());
1161
1162    string Transformed = MetaIndexParser->GetExpectedDist();
1163
1164    if (Transformed == "../project/experimental")
1165    {
1166       Transformed = "experimental";
1167    }
1168
1169    pos = Transformed.rfind('/');
1170    if (pos != string::npos)
1171    {
1172       Transformed = Transformed.substr(0, pos);
1173    }
1174
1175    if (Transformed == ".")
1176    {
1177       Transformed = "";
1178    }
1179
1180    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1181    {
1182       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1183       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1184       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1185    }
1186
1187    if (MetaIndexParser->CheckDist(Transformed) == false)
1188    {
1189       // This might become fatal one day
1190 //       Status = StatAuthError;
1191 //       ErrorText = "Conflicting distribution; expected "
1192 //          + MetaIndexParser->GetExpectedDist() + " but got "
1193 //          + MetaIndexParser->GetDist();
1194 //       return false;
1195       if (!Transformed.empty())
1196       {
1197          _error->Warning("Conflicting distribution: %s (expected %s but got %s)",
1198                          Desc.Description.c_str(),
1199                          Transformed.c_str(),
1200                          MetaIndexParser->GetDist().c_str());
1201       }
1202    }
1203
1204    return true;
1205 }
1206                                                                 /*}}}*/
1207 // pkgAcqMetaIndex::Failed - no Release file present or no signature
1208 //      file present                                            /*{{{*/
1209 // ---------------------------------------------------------------------
1210 /* */
1211 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1212 {
1213    if (AuthPass == true)
1214    {
1215       // if we fail the authentication but got the file via a IMS-Hit
1216       // this means that the file wasn't downloaded and that it might be
1217       // just stale (server problem, proxy etc). we delete what we have
1218       // queue it again without i-m-s
1219       // alternatively we could just unlink the file and let the user try again
1220       if (IMSHit)
1221       {
1222          Complete = false;
1223          Local = false;
1224          AuthPass = false;
1225          unlink(DestFile.c_str());
1226
1227          DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1228          DestFile += URItoFileName(RealURI);
1229          Desc.URI = RealURI;
1230          QueueURI(Desc);
1231          return;
1232       }
1233
1234       // gpgv method failed
1235       ReportMirrorFailure("GPGFailure");
1236       _error->Warning("GPG error: %s: %s",
1237                       Desc.Description.c_str(),
1238                       LookupTag(Message,"Message").c_str());
1239
1240    }
1241
1242    // No Release file was present, or verification failed, so fall
1243    // back to queueing Packages files without verification
1244    QueueIndexes(false);
1245 }
1246
1247                                                                         /*}}}*/
1248
1249 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1250 // ---------------------------------------------------------------------
1251 /* This just sets up the initial fetch environment and queues the first
1252    possibilitiy */
1253 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1254                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1255                              string &StoreFilename) :
1256                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1257                StoreFilename(StoreFilename), Vf(Version.FileList()),
1258                Trusted(false)
1259 {
1260    Retries = _config->FindI("Acquire::Retries",0);
1261
1262    if (Version.Arch() == 0)
1263    {
1264       _error->Error(_("I wasn't able to locate a file for the %s package. "
1265                       "This might mean you need to manually fix this package. "
1266                       "(due to missing arch)"),
1267                     Version.ParentPkg().Name());
1268       return;
1269    }
1270
1271    /* We need to find a filename to determine the extension. We make the
1272       assumption here that all the available sources for this version share
1273       the same extension.. */
1274    // Skip not source sources, they do not have file fields.
1275    for (; Vf.end() == false; Vf++)
1276    {
1277       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1278          continue;
1279       break;
1280    }
1281
1282    // Does not really matter here.. we are going to fail out below
1283    if (Vf.end() != true)
1284    {
1285       // If this fails to get a file name we will bomb out below.
1286       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1287       if (_error->PendingError() == true)
1288          return;
1289
1290       // Generate the final file name as: package_version_arch.foo
1291       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1292                       QuoteString(Version.VerStr(),"_:") + '_' +
1293                       QuoteString(Version.Arch(),"_:.") +
1294                       "." + flExtension(Parse.FileName());
1295    }
1296
1297    // check if we have one trusted source for the package. if so, switch
1298    // to "TrustedOnly" mode
1299    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; i++)
1300    {
1301       pkgIndexFile *Index;
1302       if (Sources->FindIndex(i.File(),Index) == false)
1303          continue;
1304       if (_config->FindB("Debug::pkgAcquire::Auth", false))
1305       {
1306          std::cerr << "Checking index: " << Index->Describe()
1307                    << "(Trusted=" << Index->IsTrusted() << ")\n";
1308       }
1309       if (Index->IsTrusted()) {
1310          Trusted = true;
1311          break;
1312       }
1313    }
1314
1315    // "allow-unauthenticated" restores apts old fetching behaviour
1316    // that means that e.g. unauthenticated file:// uris are higher
1317    // priority than authenticated http:// uris
1318    if (_config->FindB("APT::Get::AllowUnauthenticated",false) == true)
1319       Trusted = false;
1320
1321    // Select a source
1322    if (QueueNext() == false && _error->PendingError() == false)
1323       _error->Error(_("I wasn't able to locate file for the %s package. "
1324                     "This might mean you need to manually fix this package."),
1325                     Version.ParentPkg().Name());
1326 }
1327                                                                         /*}}}*/
1328 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1329 // ---------------------------------------------------------------------
1330 /* This queues the next available file version for download. It checks if
1331    the archive is already available in the cache and stashs the MD5 for
1332    checking later. */
1333 bool pkgAcqArchive::QueueNext()
1334 {
1335    for (; Vf.end() == false; Vf++)
1336    {
1337       // Ignore not source sources
1338       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1339          continue;
1340
1341       // Try to cross match against the source list
1342       pkgIndexFile *Index;
1343       if (Sources->FindIndex(Vf.File(),Index) == false)
1344             continue;
1345
1346       // only try to get a trusted package from another source if that source
1347       // is also trusted
1348       if(Trusted && !Index->IsTrusted())
1349          continue;
1350
1351       // Grab the text package record
1352       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1353       if (_error->PendingError() == true)
1354          return false;
1355
1356       string PkgFile = Parse.FileName();
1357       if(Parse.SHA256Hash() != "")
1358          ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
1359       else if (Parse.SHA1Hash() != "")
1360          ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
1361       else
1362          ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1363       if (PkgFile.empty() == true)
1364          return _error->Error(_("The package index files are corrupted. No Filename: "
1365                               "field for package %s."),
1366                               Version.ParentPkg().Name());
1367
1368       Desc.URI = Index->ArchiveURI(PkgFile);
1369       Desc.Description = Index->ArchiveInfo(Version);
1370       Desc.Owner = this;
1371       Desc.ShortDesc = Version.ParentPkg().Name();
1372
1373       // See if we already have the file. (Legacy filenames)
1374       FileSize = Version->Size;
1375       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
1376       struct stat Buf;
1377       if (stat(FinalFile.c_str(),&Buf) == 0)
1378       {
1379          // Make sure the size matches
1380          if ((unsigned)Buf.st_size == Version->Size)
1381          {
1382             Complete = true;
1383             Local = true;
1384             Status = StatDone;
1385             StoreFilename = DestFile = FinalFile;
1386             return true;
1387          }
1388
1389          /* Hmm, we have a file and its size does not match, this means it is
1390             an old style mismatched arch */
1391          unlink(FinalFile.c_str());
1392       }
1393
1394       // Check it again using the new style output filenames
1395       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
1396       if (stat(FinalFile.c_str(),&Buf) == 0)
1397       {
1398          // Make sure the size matches
1399          if ((unsigned)Buf.st_size == Version->Size)
1400          {
1401             Complete = true;
1402             Local = true;
1403             Status = StatDone;
1404             StoreFilename = DestFile = FinalFile;
1405             return true;
1406          }
1407
1408          /* Hmm, we have a file and its size does not match, this shouldnt
1409             happen.. */
1410          unlink(FinalFile.c_str());
1411       }
1412
1413       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
1414
1415       // Check the destination file
1416       if (stat(DestFile.c_str(),&Buf) == 0)
1417       {
1418          // Hmm, the partial file is too big, erase it
1419          if ((unsigned)Buf.st_size > Version->Size)
1420             unlink(DestFile.c_str());
1421          else
1422             PartialSize = Buf.st_size;
1423       }
1424
1425       // Create the item
1426       Local = false;
1427       Desc.URI = Index->ArchiveURI(PkgFile);
1428       Desc.Description = Index->ArchiveInfo(Version);
1429       Desc.Owner = this;
1430       Desc.ShortDesc = Version.ParentPkg().Name();
1431       QueueURI(Desc);
1432
1433       Vf++;
1434       return true;
1435    }
1436    return false;
1437 }
1438                                                                         /*}}}*/
1439 // AcqArchive::Done - Finished fetching                                 /*{{{*/
1440 // ---------------------------------------------------------------------
1441 /* */
1442 void pkgAcqArchive::Done(string Message,unsigned long Size,string CalcHash,
1443                          pkgAcquire::MethodConfig *Cfg)
1444 {
1445    Item::Done(Message,Size,CalcHash,Cfg);
1446
1447    // Check the size
1448    if (Size != Version->Size)
1449    {
1450       Status = StatError;
1451       ErrorText = _("Size mismatch");
1452       return;
1453    }
1454
1455    // Check the hash
1456    if(ExpectedHash.toStr() != CalcHash)
1457    {
1458       Status = StatError;
1459       ErrorText = _("Hash Sum mismatch");
1460       if(FileExists(DestFile))
1461          Rename(DestFile,DestFile + ".FAILED");
1462       return;
1463    }
1464
1465    // Grab the output filename
1466    string FileName = LookupTag(Message,"Filename");
1467    if (FileName.empty() == true)
1468    {
1469       Status = StatError;
1470       ErrorText = "Method gave a blank filename";
1471       return;
1472    }
1473
1474    Complete = true;
1475
1476    // Reference filename
1477    if (FileName != DestFile)
1478    {
1479       StoreFilename = DestFile = FileName;
1480       Local = true;
1481       return;
1482    }
1483
1484    // Done, move it into position
1485    string FinalFile = _config->FindDir("Dir::Cache::Archives");
1486    FinalFile += flNotDir(StoreFilename);
1487    Rename(DestFile,FinalFile);
1488
1489    StoreFilename = DestFile = FinalFile;
1490    Complete = true;
1491 }
1492                                                                         /*}}}*/
1493 // AcqArchive::Failed - Failure handler                                 /*{{{*/
1494 // ---------------------------------------------------------------------
1495 /* Here we try other sources */
1496 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1497 {
1498    ErrorText = LookupTag(Message,"Message");
1499
1500    /* We don't really want to retry on failed media swaps, this prevents
1501       that. An interesting observation is that permanent failures are not
1502       recorded. */
1503    if (Cnf->Removable == true &&
1504        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1505    {
1506       // Vf = Version.FileList();
1507       while (Vf.end() == false) Vf++;
1508       StoreFilename = string();
1509       Item::Failed(Message,Cnf);
1510       return;
1511    }
1512
1513    if (QueueNext() == false)
1514    {
1515       // This is the retry counter
1516       if (Retries != 0 &&
1517           Cnf->LocalOnly == false &&
1518           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1519       {
1520          Retries--;
1521          Vf = Version.FileList();
1522          if (QueueNext() == true)
1523             return;
1524       }
1525
1526       StoreFilename = string();
1527       Item::Failed(Message,Cnf);
1528    }
1529 }
1530                                                                         /*}}}*/
1531 // AcqArchive::IsTrusted - Determine whether this archive comes from a
1532 // trusted source                                                       /*{{{*/
1533 // ---------------------------------------------------------------------
1534 bool pkgAcqArchive::IsTrusted()
1535 {
1536    return Trusted;
1537 }
1538
1539 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
1540 // ---------------------------------------------------------------------
1541 /* */
1542 void pkgAcqArchive::Finished()
1543 {
1544    if (Status == pkgAcquire::Item::StatDone &&
1545        Complete == true)
1546       return;
1547    StoreFilename = string();
1548 }
1549                                                                         /*}}}*/
1550
1551 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
1552 // ---------------------------------------------------------------------
1553 /* The file is added to the queue */
1554 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
1555                        unsigned long Size,string Dsc,string ShortDesc,
1556                        const string &DestDir, const string &DestFilename) :
1557                        Item(Owner), ExpectedHash(Hash)
1558 {
1559    Retries = _config->FindI("Acquire::Retries",0);
1560
1561    if(!DestFilename.empty())
1562       DestFile = DestFilename;
1563    else if(!DestDir.empty())
1564       DestFile = DestDir + "/" + flNotDir(URI);
1565    else
1566       DestFile = flNotDir(URI);
1567
1568    // Create the item
1569    Desc.URI = URI;
1570    Desc.Description = Dsc;
1571    Desc.Owner = this;
1572
1573    // Set the short description to the archive component
1574    Desc.ShortDesc = ShortDesc;
1575
1576    // Get the transfer sizes
1577    FileSize = Size;
1578    struct stat Buf;
1579    if (stat(DestFile.c_str(),&Buf) == 0)
1580    {
1581       // Hmm, the partial file is too big, erase it
1582       if ((unsigned)Buf.st_size > Size)
1583          unlink(DestFile.c_str());
1584       else
1585          PartialSize = Buf.st_size;
1586    }
1587
1588    QueueURI(Desc);
1589 }
1590                                                                         /*}}}*/
1591 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
1592 // ---------------------------------------------------------------------
1593 /* */
1594 void pkgAcqFile::Done(string Message,unsigned long Size,string CalcHash,
1595                       pkgAcquire::MethodConfig *Cnf)
1596 {
1597    Item::Done(Message,Size,CalcHash,Cnf);
1598
1599    // Check the hash
1600    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
1601    {
1602       Status = StatError;
1603       ErrorText = "Hash Sum mismatch";
1604       Rename(DestFile,DestFile + ".FAILED");
1605       return;
1606    }
1607
1608    string FileName = LookupTag(Message,"Filename");
1609    if (FileName.empty() == true)
1610    {
1611       Status = StatError;
1612       ErrorText = "Method gave a blank filename";
1613       return;
1614    }
1615
1616    Complete = true;
1617
1618    // The files timestamp matches
1619    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1620       return;
1621
1622    // We have to copy it into place
1623    if (FileName != DestFile)
1624    {
1625       Local = true;
1626       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
1627           Cnf->Removable == true)
1628       {
1629          Desc.URI = "copy:" + FileName;
1630          QueueURI(Desc);
1631          return;
1632       }
1633
1634       // Erase the file if it is a symlink so we can overwrite it
1635       struct stat St;
1636       if (lstat(DestFile.c_str(),&St) == 0)
1637       {
1638          if (S_ISLNK(St.st_mode) != 0)
1639             unlink(DestFile.c_str());
1640       }
1641
1642       // Symlink the file
1643       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
1644       {
1645          ErrorText = "Link to " + DestFile + " failure ";
1646          Status = StatError;
1647          Complete = false;
1648       }
1649    }
1650 }
1651                                                                         /*}}}*/
1652 // AcqFile::Failed - Failure handler                                    /*{{{*/
1653 // ---------------------------------------------------------------------
1654 /* Here we try other sources */
1655 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1656 {
1657    ErrorText = LookupTag(Message,"Message");
1658
1659    // This is the retry counter
1660    if (Retries != 0 &&
1661        Cnf->LocalOnly == false &&
1662        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1663    {
1664       Retries--;
1665       QueueURI(Desc);
1666       return;
1667    }
1668
1669    Item::Failed(Message,Cnf);
1670 }
1671                                                                         /*}}}*/