apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <apt-pkg/acquire-item.h>
  17 #include <apt-pkg/configuration.h>
  18 #include <apt-pkg/aptconfiguration.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/vendorlist.h>
  21 #include <apt-pkg/error.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <apt-pkg/fileutl.h>
  24 #include <apt-pkg/md5.h>
  25 #include <apt-pkg/sha1.h>
  26 #include <apt-pkg/tagfile.h>
  27
  28 #include <apti18n.h>
  29
  30 #include <sys/stat.h>
  31 #include <unistd.h>
  32 #include <errno.h>
  33 #include <string>
  34 #include <sstream>
  35 #include <stdio.h>
  36                                                                         /*}}}*/
  37
  38 using namespace std;
  39
  40 // Acquire::Item::Item - Constructor                                    /*{{{*/
  41 // ---------------------------------------------------------------------
  42 /* */
  43 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  44                        PartialSize(0), Mode(0), ID(0), Complete(false),
  45                        Local(false), QueueCounter(0)
  46 {
  47    Owner->Add(this);
  48    Status = StatIdle;
  49 }
  50                                                                         /*}}}*/
  51 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  52 // ---------------------------------------------------------------------
  53 /* */
  54 pkgAcquire::Item::~Item()
  55 {
  56    Owner->Remove(this);
  57 }
  58                                                                         /*}}}*/
  59 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  60 // ---------------------------------------------------------------------
  61 /* We return to an idle state if there are still other queues that could
  62    fetch this object */
  63 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  64 {
  65    Status = StatIdle;
  66    ErrorText = LookupTag(Message,"Message");
  67    if (QueueCounter <= 1)
  68    {
  69       /* This indicates that the file is not available right now but might
  70          be sometime later. If we do a retry cycle then this should be
  71          retried [CDROMs] */
  72       if (Cnf->LocalOnly == true &&
  73           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  74       {
  75          Status = StatIdle;
  76          Dequeue();
  77          return;
  78       }
  79
  80       Status = StatError;
  81       Dequeue();
  82    }
  83 }
  84                                                                         /*}}}*/
  85 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
  86 // ---------------------------------------------------------------------
  87 /* Stash status and the file size. Note that setting Complete means
  88    sub-phases of the acquire process such as decompresion are operating */
  89 void pkgAcquire::Item::Start(string /*Message*/,unsigned long Size)
  90 {
  91    Status = StatFetching;
  92    if (FileSize == 0 && Complete == false)
  93       FileSize = Size;
  94 }
  95                                                                         /*}}}*/
  96 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
  97 // ---------------------------------------------------------------------
  98 /* */
  99 void pkgAcquire::Item::Done(string Message,unsigned long Size,string Hash,
 100                             pkgAcquire::MethodConfig *Cnf)
 101 {
 102    // We just downloaded something..
 103    string FileName = LookupTag(Message,"Filename");
 104    // we only inform the Log class if it was actually not a local thing
 105    if (Complete == false && !Local && FileName == DestFile)
 106    {
 107       if (Owner->Log != 0)
 108          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 109    }
 110
 111    if (FileSize == 0)
 112       FileSize= Size;
 113
 114    Status = StatDone;
 115    ErrorText = string();
 116    Owner->Dequeue(this);
 117 }
 118                                                                         /*}}}*/
 119 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 120 // ---------------------------------------------------------------------
 121 /* This helper function is used by alot of item methods as thier final
 122    step */
 123 void pkgAcquire::Item::Rename(string From,string To)
 124 {
 125    if (rename(From.c_str(),To.c_str()) != 0)
 126    {
 127       char S[300];
 128       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 129               From.c_str(),To.c_str());
 130       Status = StatError;
 131       ErrorText = S;
 132    }
 133 }
 134                                                                         /*}}}*/
 135 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 136 // ---------------------------------------------------------------------
 137 /* Get the DiffIndex file first and see if there are patches availabe
 138  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 139  * patches. If anything goes wrong in that process, it will fall back to
 140  * the original packages file
 141  */
 142 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 143                                  string URI,string URIDesc,string ShortDesc,
 144                                  HashString ExpectedHash)
 145    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 146      Description(URIDesc)
 147 {
 148
 149    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 150
 151    Desc.Description = URIDesc + "/DiffIndex";
 152    Desc.Owner = this;
 153    Desc.ShortDesc = ShortDesc;
 154    Desc.URI = URI + ".diff/Index";
 155
 156    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 157    DestFile += URItoFileName(URI) + string(".DiffIndex");
 158
 159    if(Debug)
 160       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 161
 162    // look for the current package file
 163    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 164    CurrentPackagesFile += URItoFileName(RealURI);
 165
 166    // FIXME: this file:/ check is a hack to prevent fetching
 167    //        from local sources. this is really silly, and
 168    //        should be fixed cleanly as soon as possible
 169    if(!FileExists(CurrentPackagesFile) ||
 170       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 171    {
 172       // we don't have a pkg file or we don't want to queue
 173       if(Debug)
 174          std::clog << "No index file, local or canceld by user" << std::endl;
 175       Failed("", NULL);
 176       return;
 177    }
 178
 179    if(Debug)
 180       std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): "
 181                 << CurrentPackagesFile << std::endl;
 182
 183    QueueURI(Desc);
 184
 185 }
 186                                                                         /*}}}*/
 187 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 188 // ---------------------------------------------------------------------
 189 /* The only header we use is the last-modified header. */
 190 string pkgAcqDiffIndex::Custom600Headers()
 191 {
 192    string Final = _config->FindDir("Dir::State::lists");
 193    Final += URItoFileName(RealURI) + string(".IndexDiff");
 194
 195    if(Debug)
 196       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 197
 198    struct stat Buf;
 199    if (stat(Final.c_str(),&Buf) != 0)
 200       return "\nIndex-File: true";
 201
 202    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 203 }
 204                                                                         /*}}}*/
 205 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 206 {
 207    if(Debug)
 208       std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile
 209                 << std::endl;
 210
 211    pkgTagSection Tags;
 212    string ServerSha1;
 213    vector<DiffInfo> available_patches;
 214
 215    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 216    pkgTagFile TF(&Fd);
 217    if (_error->PendingError() == true)
 218       return false;
 219
 220    if(TF.Step(Tags) == true)
 221    {
 222       string local_sha1;
 223       bool found = false;
 224       DiffInfo d;
 225       string size;
 226
 227       string tmp = Tags.FindS("SHA1-Current");
 228       std::stringstream ss(tmp);
 229       ss >> ServerSha1;
 230
 231       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 232       SHA1Summation SHA1;
 233       SHA1.AddFD(fd.Fd(), fd.Size());
 234       local_sha1 = string(SHA1.Result());
 235
 236       if(local_sha1 == ServerSha1)
 237       {
 238          // we have the same sha1 as the server
 239          if(Debug)
 240             std::clog << "Package file is up-to-date" << std::endl;
 241          // set found to true, this will queue a pkgAcqIndexDiffs with
 242          // a empty availabe_patches
 243          found = true;
 244       }
 245       else
 246       {
 247          if(Debug)
 248             std::clog << "SHA1-Current: " << ServerSha1 << std::endl;
 249
 250          // check the historie and see what patches we need
 251          string history = Tags.FindS("SHA1-History");
 252          std::stringstream hist(history);
 253          while(hist >> d.sha1 >> size >> d.file)
 254          {
 255             d.size = atoi(size.c_str());
 256             // read until the first match is found
 257             if(d.sha1 == local_sha1)
 258                found=true;
 259             // from that point on, we probably need all diffs
 260             if(found)
 261             {
 262                if(Debug)
 263                   std::clog << "Need to get diff: " << d.file << std::endl;
 264                available_patches.push_back(d);
 265             }
 266          }
 267       }
 268
 269       // we have something, queue the next diff
 270       if(found)
 271       {
 272          // queue the diffs
 273          string::size_type last_space = Description.rfind(" ");
 274          if(last_space != string::npos)
 275             Description.erase(last_space, Description.size()-last_space);
 276          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 277                               ExpectedHash, ServerSha1, available_patches);
 278          Complete = false;
 279          Status = StatDone;
 280          Dequeue();
 281          return true;
 282       }
 283    }
 284
 285    // Nothing found, report and return false
 286    // Failing here is ok, if we return false later, the full
 287    // IndexFile is queued
 288    if(Debug)
 289       std::clog << "Can't find a patch in the index file" << std::endl;
 290    return false;
 291 }
 292                                                                         /*}}}*/
 293 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)      /*{{{*/
 294 {
 295    if(Debug)
 296       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << std::endl
 297                 << "Falling back to normal index file aquire" << std::endl;
 298
 299    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 300                    ExpectedHash);
 301
 302    Complete = false;
 303    Status = StatDone;
 304    Dequeue();
 305 }
 306                                                                         /*}}}*/
 307 void pkgAcqDiffIndex::Done(string Message,unsigned long Size,string Md5Hash,    /*{{{*/
 308                            pkgAcquire::MethodConfig *Cnf)
 309 {
 310    if(Debug)
 311       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 312
 313    Item::Done(Message,Size,Md5Hash,Cnf);
 314
 315    string FinalFile;
 316    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 317
 318    // sucess in downloading the index
 319    // rename the index
 320    FinalFile += string(".IndexDiff");
 321    if(Debug)
 322       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 323                 << std::endl;
 324    Rename(DestFile,FinalFile);
 325    chmod(FinalFile.c_str(),0644);
 326    DestFile = FinalFile;
 327
 328    if(!ParseDiffIndex(DestFile))
 329       return Failed("", NULL);
 330
 331    Complete = true;
 332    Status = StatDone;
 333    Dequeue();
 334    return;
 335 }
 336                                                                         /*}}}*/
 337 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 338 // ---------------------------------------------------------------------
 339 /* The package diff is added to the queue. one object is constructed
 340  * for each diff and the index
 341  */
 342 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 343                                    string URI,string URIDesc,string ShortDesc,
 344                                    HashString ExpectedHash,
 345                                    string ServerSha1,
 346                                    vector<DiffInfo> diffs)
 347    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 348      available_patches(diffs), ServerSha1(ServerSha1)
 349 {
 350
 351    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 352    DestFile += URItoFileName(URI);
 353
 354    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 355
 356    Description = URIDesc;
 357    Desc.Owner = this;
 358    Desc.ShortDesc = ShortDesc;
 359
 360    if(available_patches.size() == 0)
 361    {
 362       // we are done (yeah!)
 363       Finish(true);
 364    }
 365    else
 366    {
 367       // get the next diff
 368       State = StateFetchDiff;
 369       QueueNextDiff();
 370    }
 371 }
 372                                                                         /*}}}*/
 373 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)     /*{{{*/
 374 {
 375    if(Debug)
 376       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl
 377                 << "Falling back to normal index file aquire" << std::endl;
 378    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 379                    ExpectedHash);
 380    Finish();
 381 }
 382                                                                         /*}}}*/
 383 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 384 void pkgAcqIndexDiffs::Finish(bool allDone)
 385 {
 386    // we restore the original name, this is required, otherwise
 387    // the file will be cleaned
 388    if(allDone)
 389    {
 390       DestFile = _config->FindDir("Dir::State::lists");
 391       DestFile += URItoFileName(RealURI);
 392
 393       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 394       {
 395          Status = StatAuthError;
 396          ErrorText = _("MD5Sum mismatch");
 397          Rename(DestFile,DestFile + ".FAILED");
 398          Dequeue();
 399          return;
 400       }
 401
 402       // this is for the "real" finish
 403       Complete = true;
 404       Status = StatDone;
 405       Dequeue();
 406       if(Debug)
 407          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 408       return;
 409    }
 410
 411    if(Debug)
 412       std::clog << "Finishing: " << Desc.URI << std::endl;
 413    Complete = false;
 414    Status = StatDone;
 415    Dequeue();
 416    return;
 417 }
 418                                                                         /*}}}*/
 419 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 420 {
 421
 422    // calc sha1 of the just patched file
 423    string FinalFile = _config->FindDir("Dir::State::lists");
 424    FinalFile += URItoFileName(RealURI);
 425
 426    FileFd fd(FinalFile, FileFd::ReadOnly);
 427    SHA1Summation SHA1;
 428    SHA1.AddFD(fd.Fd(), fd.Size());
 429    string local_sha1 = string(SHA1.Result());
 430    if(Debug)
 431       std::clog << "QueueNextDiff: "
 432                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 433
 434    // final file reached before all patches are applied
 435    if(local_sha1 == ServerSha1)
 436    {
 437       Finish(true);
 438       return true;
 439    }
 440
 441    // remove all patches until the next matching patch is found
 442    // this requires the Index file to be ordered
 443    for(vector<DiffInfo>::iterator I=available_patches.begin();
 444        available_patches.size() > 0 &&
 445           I != available_patches.end() &&
 446           (*I).sha1 != local_sha1;
 447        I++)
 448    {
 449       available_patches.erase(I);
 450    }
 451
 452    // error checking and falling back if no patch was found
 453    if(available_patches.size() == 0)
 454    {
 455       Failed("", NULL);
 456       return false;
 457    }
 458
 459    // queue the right diff
 460    Desc.URI = string(RealURI) + ".diff/" + available_patches[0].file + ".gz";
 461    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 462    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 463    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 464
 465    if(Debug)
 466       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 467
 468    QueueURI(Desc);
 469
 470    return true;
 471 }
 472                                                                         /*}}}*/
 473 void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash,   /*{{{*/
 474                             pkgAcquire::MethodConfig *Cnf)
 475 {
 476    if(Debug)
 477       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 478
 479    Item::Done(Message,Size,Md5Hash,Cnf);
 480
 481    string FinalFile;
 482    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 483
 484    // sucess in downloading a diff, enter ApplyDiff state
 485    if(State == StateFetchDiff)
 486    {
 487
 488       if(Debug)
 489          std::clog << "Sending to gzip method: " << FinalFile << std::endl;
 490
 491       string FileName = LookupTag(Message,"Filename");
 492       State = StateUnzipDiff;
 493       Local = true;
 494       Desc.URI = "gzip:" + FileName;
 495       DestFile += ".decomp";
 496       QueueURI(Desc);
 497       Mode = "gzip";
 498       return;
 499    }
 500
 501    // sucess in downloading a diff, enter ApplyDiff state
 502    if(State == StateUnzipDiff)
 503    {
 504
 505       // rred excepts the patch as $FinalFile.ed
 506       Rename(DestFile,FinalFile+".ed");
 507
 508       if(Debug)
 509          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 510
 511       State = StateApplyDiff;
 512       Local = true;
 513       Desc.URI = "rred:" + FinalFile;
 514       QueueURI(Desc);
 515       Mode = "rred";
 516       return;
 517    }
 518
 519
 520    // success in download/apply a diff, queue next (if needed)
 521    if(State == StateApplyDiff)
 522    {
 523       // remove the just applied patch
 524       available_patches.erase(available_patches.begin());
 525
 526       // move into place
 527       if(Debug)
 528       {
 529          std::clog << "Moving patched file in place: " << std::endl
 530                    << DestFile << " -> " << FinalFile << std::endl;
 531       }
 532       Rename(DestFile,FinalFile);
 533       chmod(FinalFile.c_str(),0644);
 534
 535       // see if there is more to download
 536       if(available_patches.size() > 0) {
 537          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 538                               ExpectedHash, ServerSha1, available_patches);
 539          return Finish();
 540       } else
 541          return Finish(true);
 542    }
 543 }
 544                                                                         /*}}}*/
 545 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 546 // ---------------------------------------------------------------------
 547 /* The package file is added to the queue and a second class is
 548    instantiated to fetch the revision file */
 549 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 550                          string URI,string URIDesc,string ShortDesc,
 551                          HashString ExpectedHash, string comprExt)
 552    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 553 {
 554    Decompression = false;
 555    Erase = false;
 556
 557    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 558    DestFile += URItoFileName(URI);
 559
 560    if(comprExt.empty())
 561    {
 562       // autoselect the compression method
 563       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 564       if (types.empty() == true)
 565          comprExt = "plain";
 566       else
 567          comprExt = "." + types[0];
 568    }
 569    CompressionExtension = ((comprExt == "plain" || comprExt == ".") ? "" : comprExt);
 570
 571    Desc.URI = URI + CompressionExtension;
 572
 573    Desc.Description = URIDesc;
 574    Desc.Owner = this;
 575    Desc.ShortDesc = ShortDesc;
 576
 577    QueueURI(Desc);
 578 }
 579                                                                         /*}}}*/
 580 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 581 // ---------------------------------------------------------------------
 582 /* The only header we use is the last-modified header. */
 583 string pkgAcqIndex::Custom600Headers()
 584 {
 585    string Final = _config->FindDir("Dir::State::lists");
 586    Final += URItoFileName(RealURI);
 587
 588    struct stat Buf;
 589    if (stat(Final.c_str(),&Buf) != 0)
 590       return "\nIndex-File: true";
 591
 592    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 593 }
 594                                                                         /*}}}*/
 595 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
 596 {
 597    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 598
 599    for (std::vector<std::string>::const_iterator t = types.begin();
 600         t != types.end(); t++)
 601    {
 602       // jump over all already tried compression types
 603       const unsigned int nameLen = Desc.URI.size() - (*t).size();
 604       if(Desc.URI.substr(nameLen) != *t)
 605          continue;
 606
 607       // we want to try it with the next extension (and make sure to
 608       // not skip over the end)
 609       t++;
 610       if (t == types.end())
 611          break;
 612
 613       // queue new download
 614       Desc.URI = Desc.URI.substr(0, nameLen) + *t;
 615       new pkgAcqIndex(Owner, RealURI, Desc.Description, Desc.ShortDesc,
 616       ExpectedHash, string(".").append(*t));
 617
 618       Status = StatDone;
 619       Complete = false;
 620       Dequeue();
 621       return;
 622    }
 623
 624    // on decompression failure, remove bad versions in partial/
 625    if(Decompression && Erase) {
 626       string s = _config->FindDir("Dir::State::lists") + "partial/";
 627       s += URItoFileName(RealURI);
 628       unlink(s.c_str());
 629    }
 630
 631    Item::Failed(Message,Cnf);
 632 }
 633                                                                         /*}}}*/
 634 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
 635 // ---------------------------------------------------------------------
 636 /* This goes through a number of states.. On the initial fetch the
 637    method could possibly return an alternate filename which points
 638    to the uncompressed version of the file. If this is so the file
 639    is copied into the partial directory. In all other cases the file
 640    is decompressed with a gzip uri. */
 641 void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash,
 642                        pkgAcquire::MethodConfig *Cfg)
 643 {
 644    Item::Done(Message,Size,Hash,Cfg);
 645
 646    if (Decompression == true)
 647    {
 648       if (_config->FindB("Debug::pkgAcquire::Auth", false))
 649       {
 650          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
 651          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
 652       }
 653
 654       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
 655       {
 656          Status = StatAuthError;
 657          ErrorText = _("Hash Sum mismatch");
 658          Rename(DestFile,DestFile + ".FAILED");
 659          return;
 660       }
 661       // Done, move it into position
 662       string FinalFile = _config->FindDir("Dir::State::lists");
 663       FinalFile += URItoFileName(RealURI);
 664       Rename(DestFile,FinalFile);
 665       chmod(FinalFile.c_str(),0644);
 666
 667       /* We restore the original name to DestFile so that the clean operation
 668          will work OK */
 669       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 670       DestFile += URItoFileName(RealURI);
 671
 672       // Remove the compressed version.
 673       if (Erase == true)
 674          unlink(DestFile.c_str());
 675       return;
 676    }
 677
 678    Erase = false;
 679    Complete = true;
 680
 681    // Handle the unzipd case
 682    string FileName = LookupTag(Message,"Alt-Filename");
 683    if (FileName.empty() == false)
 684    {
 685       // The files timestamp matches
 686       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
 687          return;
 688       Decompression = true;
 689       Local = true;
 690       DestFile += ".decomp";
 691       Desc.URI = "copy:" + FileName;
 692       QueueURI(Desc);
 693       Mode = "copy";
 694       return;
 695    }
 696
 697    FileName = LookupTag(Message,"Filename");
 698    if (FileName.empty() == true)
 699    {
 700       Status = StatError;
 701       ErrorText = "Method gave a blank filename";
 702    }
 703
 704    // The files timestamp matches
 705    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 706       return;
 707
 708    if (FileName == DestFile)
 709       Erase = true;
 710    else
 711       Local = true;
 712
 713    string compExt = flExtension(flNotDir(URI(Desc.URI).Path));
 714    string decompProg;
 715
 716    // get the binary name for your used compression type
 717    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
 718    if(decompProg.empty() == false);
 719    // flExtensions returns the full name if no extension is found
 720    // this is why we have this complicated compare operation here
 721    // FIMXE: add a new flJustExtension() that return "" if no
 722    //        extension is found and use that above so that it can
 723    //        be tested against ""
 724    else if(compExt == flNotDir(URI(Desc.URI).Path))
 725       decompProg = "copy";
 726    else {
 727       _error->Error("Unsupported extension: %s", compExt.c_str());
 728       return;
 729    }
 730
 731    Decompression = true;
 732    DestFile += ".decomp";
 733    Desc.URI = decompProg + ":" + FileName;
 734    QueueURI(Desc);
 735    Mode = decompProg.c_str();
 736 }
 737                                                                         /*}}}*/
 738 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
 739 // ---------------------------------------------------------------------
 740 /* The Translation file is added to the queue */
 741 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
 742                             string URI,string URIDesc,string ShortDesc)
 743   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
 744 {
 745 }
 746                                                                         /*}}}*/
 747 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
 748 // ---------------------------------------------------------------------
 749 /* */
 750 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 751 {
 752    if (Cnf->LocalOnly == true ||
 753        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 754    {
 755       // Ignore this
 756       Status = StatDone;
 757       Complete = false;
 758       Dequeue();
 759       return;
 760    }
 761
 762    Item::Failed(Message,Cnf);
 763 }
 764                                                                         /*}}}*/
 765 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
 766                              string URI,string URIDesc,string ShortDesc,
 767                              string MetaIndexURI, string MetaIndexURIDesc,
 768                              string MetaIndexShortDesc,
 769                              const vector<IndexTarget*>* IndexTargets,
 770                              indexRecords* MetaIndexParser) :
 771    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
 772    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
 773    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
 774 {
 775    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 776    DestFile += URItoFileName(URI);
 777
 778    // remove any partial downloaded sig-file in partial/.
 779    // it may confuse proxies and is too small to warrant a
 780    // partial download anyway
 781    unlink(DestFile.c_str());
 782
 783    // Create the item
 784    Desc.Description = URIDesc;
 785    Desc.Owner = this;
 786    Desc.ShortDesc = ShortDesc;
 787    Desc.URI = URI;
 788
 789    string Final = _config->FindDir("Dir::State::lists");
 790    Final += URItoFileName(RealURI);
 791    struct stat Buf;
 792    if (stat(Final.c_str(),&Buf) == 0)
 793    {
 794       // File was already in place.  It needs to be re-downloaded/verified
 795       // because Release might have changed, we do give it a differnt
 796       // name than DestFile because otherwise the http method will
 797       // send If-Range requests and there are too many broken servers
 798       // out there that do not understand them
 799       LastGoodSig = DestFile+".reverify";
 800       Rename(Final,LastGoodSig);
 801    }
 802
 803    QueueURI(Desc);
 804 }
 805                                                                         /*}}}*/
 806 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
 807 // ---------------------------------------------------------------------
 808 /* The only header we use is the last-modified header. */
 809 string pkgAcqMetaSig::Custom600Headers()
 810 {
 811    struct stat Buf;
 812    if (stat(LastGoodSig.c_str(),&Buf) != 0)
 813       return "\nIndex-File: true";
 814
 815    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 816 }
 817
 818 void pkgAcqMetaSig::Done(string Message,unsigned long Size,string MD5,
 819                          pkgAcquire::MethodConfig *Cfg)
 820 {
 821    Item::Done(Message,Size,MD5,Cfg);
 822
 823    string FileName = LookupTag(Message,"Filename");
 824    if (FileName.empty() == true)
 825    {
 826       Status = StatError;
 827       ErrorText = "Method gave a blank filename";
 828       return;
 829    }
 830
 831    if (FileName != DestFile)
 832    {
 833       // We have to copy it into place
 834       Local = true;
 835       Desc.URI = "copy:" + FileName;
 836       QueueURI(Desc);
 837       return;
 838    }
 839
 840    Complete = true;
 841
 842    // put the last known good file back on i-m-s hit (it will
 843    // be re-verified again)
 844    // Else do nothing, we have the new file in DestFile then
 845    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 846       Rename(LastGoodSig, DestFile);
 847
 848    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
 849    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
 850                        DestFile, IndexTargets, MetaIndexParser);
 851
 852 }
 853                                                                         /*}}}*/
 854 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
 855 {
 856    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 857
 858    // if we get a network error we fail gracefully
 859    if(Status == StatTransientNetworkError)
 860    {
 861       Item::Failed(Message,Cnf);
 862       // move the sigfile back on transient network failures
 863       if(FileExists(DestFile))
 864          Rename(LastGoodSig,Final);
 865
 866       // set the status back to , Item::Failed likes to reset it
 867       Status = pkgAcquire::Item::StatTransientNetworkError;
 868       return;
 869    }
 870
 871    // Delete any existing sigfile when the acquire failed
 872    unlink(Final.c_str());
 873
 874    // queue a pkgAcqMetaIndex with no sigfile
 875    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
 876                        "", IndexTargets, MetaIndexParser);
 877
 878    if (Cnf->LocalOnly == true ||
 879        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 880    {
 881       // Ignore this
 882       Status = StatDone;
 883       Complete = false;
 884       Dequeue();
 885       return;
 886    }
 887
 888    Item::Failed(Message,Cnf);
 889 }
 890                                                                         /*}}}*/
 891 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
 892                                  string URI,string URIDesc,string ShortDesc,
 893                                  string SigFile,
 894                                  const vector<struct IndexTarget*>* IndexTargets,
 895                                  indexRecords* MetaIndexParser) :
 896    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
 897    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
 898 {
 899    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 900    DestFile += URItoFileName(URI);
 901
 902    // Create the item
 903    Desc.Description = URIDesc;
 904    Desc.Owner = this;
 905    Desc.ShortDesc = ShortDesc;
 906    Desc.URI = URI;
 907
 908    QueueURI(Desc);
 909 }
 910                                                                         /*}}}*/
 911 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
 912 // ---------------------------------------------------------------------
 913 /* The only header we use is the last-modified header. */
 914 string pkgAcqMetaIndex::Custom600Headers()
 915 {
 916    string Final = _config->FindDir("Dir::State::lists");
 917    Final += URItoFileName(RealURI);
 918
 919    struct stat Buf;
 920    if (stat(Final.c_str(),&Buf) != 0)
 921       return "\nIndex-File: true";
 922
 923    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 924 }
 925                                                                         /*}}}*/
 926 void pkgAcqMetaIndex::Done(string Message,unsigned long Size,string Hash,       /*{{{*/
 927                            pkgAcquire::MethodConfig *Cfg)
 928 {
 929    Item::Done(Message,Size,Hash,Cfg);
 930
 931    // MetaIndexes are done in two passes: one to download the
 932    // metaindex with an appropriate method, and a second to verify it
 933    // with the gpgv method
 934
 935    if (AuthPass == true)
 936    {
 937       AuthDone(Message);
 938    }
 939    else
 940    {
 941       RetrievalDone(Message);
 942       if (!Complete)
 943          // Still more retrieving to do
 944          return;
 945
 946       if (SigFile == "")
 947       {
 948          // There was no signature file, so we are finished.  Download
 949          // the indexes without verification.
 950          QueueIndexes(false);
 951       }
 952       else
 953       {
 954          // There was a signature file, so pass it to gpgv for
 955          // verification
 956
 957          if (_config->FindB("Debug::pkgAcquire::Auth", false))
 958             std::cerr << "Metaindex acquired, queueing gpg verification ("
 959                       << SigFile << "," << DestFile << ")\n";
 960          AuthPass = true;
 961          Desc.URI = "gpgv:" + SigFile;
 962          QueueURI(Desc);
 963          Mode = "gpgv";
 964       }
 965    }
 966 }
 967                                                                         /*}}}*/
 968 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
 969 {
 970    // We have just finished downloading a Release file (it is not
 971    // verified yet)
 972
 973    string FileName = LookupTag(Message,"Filename");
 974    if (FileName.empty() == true)
 975    {
 976       Status = StatError;
 977       ErrorText = "Method gave a blank filename";
 978       return;
 979    }
 980
 981    if (FileName != DestFile)
 982    {
 983       Local = true;
 984       Desc.URI = "copy:" + FileName;
 985       QueueURI(Desc);
 986       return;
 987    }
 988
 989    // see if the download was a IMSHit
 990    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
 991    Complete = true;
 992
 993    string FinalFile = _config->FindDir("Dir::State::lists");
 994    FinalFile += URItoFileName(RealURI);
 995
 996    // If we get a IMS hit we can remove the empty file in partial
 997    // othersie we move the file in place
 998    if (IMSHit)
 999       unlink(DestFile.c_str());
1000    else
1001       Rename(DestFile,FinalFile);
1002
1003    chmod(FinalFile.c_str(),0644);
1004    DestFile = FinalFile;
1005 }
1006                                                                         /*}}}*/
1007 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1008 {
1009    // At this point, the gpgv method has succeeded, so there is a
1010    // valid signature from a key in the trusted keyring.  We
1011    // perform additional verification of its contents, and use them
1012    // to verify the indexes we are about to download
1013
1014    if (!MetaIndexParser->Load(DestFile))
1015    {
1016       Status = StatAuthError;
1017       ErrorText = MetaIndexParser->ErrorText;
1018       return;
1019    }
1020
1021    if (!VerifyVendor(Message))
1022    {
1023       return;
1024    }
1025
1026    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1027       std::cerr << "Signature verification succeeded: "
1028                 << DestFile << std::endl;
1029
1030    // Download further indexes with verification
1031    QueueIndexes(true);
1032
1033    // Done, move signature file into position
1034
1035    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1036       URItoFileName(RealURI) + ".gpg";
1037    Rename(SigFile,VerifiedSigFile);
1038    chmod(VerifiedSigFile.c_str(),0644);
1039 }
1040                                                                         /*}}}*/
1041 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1042 {
1043    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1044         Target != IndexTargets->end();
1045         Target++)
1046    {
1047       HashString ExpectedIndexHash;
1048       if (verify)
1049       {
1050          const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1051          if (!Record)
1052          {
1053             Status = StatAuthError;
1054             ErrorText = "Unable to find expected entry  "
1055                + (*Target)->MetaKey + " in Meta-index file (malformed Release file?)";
1056             return;
1057          }
1058          ExpectedIndexHash = Record->Hash;
1059          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1060          {
1061             std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1062             std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1063          }
1064          if (ExpectedIndexHash.empty())
1065          {
1066             Status = StatAuthError;
1067             ErrorText = "Unable to find hash sum for "
1068                + (*Target)->MetaKey + " in Meta-index file";
1069             return;
1070          }
1071       }
1072
1073       // Queue Packages file (either diff or full packages files, depending
1074       // on the users option)
1075       if(_config->FindB("Acquire::PDiffs",true) == true)
1076          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1077                              (*Target)->ShortDesc, ExpectedIndexHash);
1078       else
1079          new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description,
1080                             (*Target)->ShortDesc, ExpectedIndexHash);
1081    }
1082 }
1083                                                                         /*}}}*/
1084 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1085 {
1086 //    // Maybe this should be made available from above so we don't have
1087 //    // to read and parse it every time?
1088 //    pkgVendorList List;
1089 //    List.ReadMainList();
1090
1091 //    const Vendor* Vndr = NULL;
1092 //    for (std::vector<string>::const_iterator I = GPGVOutput.begin(); I != GPGVOutput.end(); I++)
1093 //    {
1094 //       string::size_type pos = (*I).find("VALIDSIG ");
1095 //       if (_config->FindB("Debug::Vendor", false))
1096 //          std::cerr << "Looking for VALIDSIG in \"" << (*I) << "\": pos " << pos
1097 //                    << std::endl;
1098 //       if (pos != std::string::npos)
1099 //       {
1100 //          string Fingerprint = (*I).substr(pos+sizeof("VALIDSIG"));
1101 //          if (_config->FindB("Debug::Vendor", false))
1102 //             std::cerr << "Looking for \"" << Fingerprint << "\" in vendor..." <<
1103 //                std::endl;
1104 //          Vndr = List.FindVendor(Fingerprint) != "";
1105 //          if (Vndr != NULL);
1106 //          break;
1107 //       }
1108 //    }
1109    string::size_type pos;
1110
1111    // check for missing sigs (that where not fatal because otherwise we had
1112    // bombed earlier)
1113    string missingkeys;
1114    string msg = _("There is no public key available for the "
1115                   "following key IDs:\n");
1116    pos = Message.find("NO_PUBKEY ");
1117    if (pos != std::string::npos)
1118    {
1119       string::size_type start = pos+strlen("NO_PUBKEY ");
1120       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1121       missingkeys += (Fingerprint);
1122    }
1123    if(!missingkeys.empty())
1124       _error->Warning("%s", string(msg+missingkeys).c_str());
1125
1126    string Transformed = MetaIndexParser->GetExpectedDist();
1127
1128    if (Transformed == "../project/experimental")
1129    {
1130       Transformed = "experimental";
1131    }
1132
1133    pos = Transformed.rfind('/');
1134    if (pos != string::npos)
1135    {
1136       Transformed = Transformed.substr(0, pos);
1137    }
1138
1139    if (Transformed == ".")
1140    {
1141       Transformed = "";
1142    }
1143
1144    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1145    {
1146       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1147       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1148       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1149    }
1150
1151    if (MetaIndexParser->CheckDist(Transformed) == false)
1152    {
1153       // This might become fatal one day
1154 //       Status = StatAuthError;
1155 //       ErrorText = "Conflicting distribution; expected "
1156 //          + MetaIndexParser->GetExpectedDist() + " but got "
1157 //          + MetaIndexParser->GetDist();
1158 //       return false;
1159       if (!Transformed.empty())
1160       {
1161          _error->Warning("Conflicting distribution: %s (expected %s but got %s)",
1162                          Desc.Description.c_str(),
1163                          Transformed.c_str(),
1164                          MetaIndexParser->GetDist().c_str());
1165       }
1166    }
1167
1168    return true;
1169 }
1170                                                                         /*}}}*/
1171 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1172 // ---------------------------------------------------------------------
1173 /* */
1174 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1175 {
1176    if (AuthPass == true)
1177    {
1178       // if we fail the authentication but got the file via a IMS-Hit
1179       // this means that the file wasn't downloaded and that it might be
1180       // just stale (server problem, proxy etc). we delete what we have
1181       // queue it again without i-m-s
1182       // alternatively we could just unlink the file and let the user try again
1183       if (IMSHit)
1184       {
1185          Complete = false;
1186          Local = false;
1187          AuthPass = false;
1188          unlink(DestFile.c_str());
1189
1190          DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1191          DestFile += URItoFileName(RealURI);
1192          Desc.URI = RealURI;
1193          QueueURI(Desc);
1194          return;
1195       }
1196
1197       // gpgv method failed
1198       _error->Warning("GPG error: %s: %s",
1199                       Desc.Description.c_str(),
1200                       LookupTag(Message,"Message").c_str());
1201
1202    }
1203
1204    // No Release file was present, or verification failed, so fall
1205    // back to queueing Packages files without verification
1206    QueueIndexes(false);
1207 }
1208                                                                         /*}}}*/
1209 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1210 // ---------------------------------------------------------------------
1211 /* This just sets up the initial fetch environment and queues the first
1212    possibilitiy */
1213 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1214                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1215                              string &StoreFilename) :
1216                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1217                StoreFilename(StoreFilename), Vf(Version.FileList()),
1218                Trusted(false)
1219 {
1220    Retries = _config->FindI("Acquire::Retries",0);
1221
1222    if (Version.Arch() == 0)
1223    {
1224       _error->Error(_("I wasn't able to locate a file for the %s package. "
1225                       "This might mean you need to manually fix this package. "
1226                       "(due to missing arch)"),
1227                     Version.ParentPkg().Name());
1228       return;
1229    }
1230
1231    /* We need to find a filename to determine the extension. We make the
1232       assumption here that all the available sources for this version share
1233       the same extension.. */
1234    // Skip not source sources, they do not have file fields.
1235    for (; Vf.end() == false; Vf++)
1236    {
1237       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1238          continue;
1239       break;
1240    }
1241
1242    // Does not really matter here.. we are going to fail out below
1243    if (Vf.end() != true)
1244    {
1245       // If this fails to get a file name we will bomb out below.
1246       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1247       if (_error->PendingError() == true)
1248          return;
1249
1250       // Generate the final file name as: package_version_arch.foo
1251       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1252                       QuoteString(Version.VerStr(),"_:") + '_' +
1253                       QuoteString(Version.Arch(),"_:.") +
1254                       "." + flExtension(Parse.FileName());
1255    }
1256
1257    // check if we have one trusted source for the package. if so, switch
1258    // to "TrustedOnly" mode
1259    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; i++)
1260    {
1261       pkgIndexFile *Index;
1262       if (Sources->FindIndex(i.File(),Index) == false)
1263          continue;
1264       if (_config->FindB("Debug::pkgAcquire::Auth", false))
1265       {
1266          std::cerr << "Checking index: " << Index->Describe()
1267                    << "(Trusted=" << Index->IsTrusted() << ")\n";
1268       }
1269       if (Index->IsTrusted()) {
1270          Trusted = true;
1271          break;
1272       }
1273    }
1274
1275    // "allow-unauthenticated" restores apts old fetching behaviour
1276    // that means that e.g. unauthenticated file:// uris are higher
1277    // priority than authenticated http:// uris
1278    if (_config->FindB("APT::Get::AllowUnauthenticated",false) == true)
1279       Trusted = false;
1280
1281    // Select a source
1282    if (QueueNext() == false && _error->PendingError() == false)
1283       _error->Error(_("I wasn't able to locate file for the %s package. "
1284                     "This might mean you need to manually fix this package."),
1285                     Version.ParentPkg().Name());
1286 }
1287                                                                         /*}}}*/
1288 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1289 // ---------------------------------------------------------------------
1290 /* This queues the next available file version for download. It checks if
1291    the archive is already available in the cache and stashs the MD5 for
1292    checking later. */
1293 bool pkgAcqArchive::QueueNext()
1294 {
1295    for (; Vf.end() == false; Vf++)
1296    {
1297       // Ignore not source sources
1298       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1299          continue;
1300
1301       // Try to cross match against the source list
1302       pkgIndexFile *Index;
1303       if (Sources->FindIndex(Vf.File(),Index) == false)
1304             continue;
1305
1306       // only try to get a trusted package from another source if that source
1307       // is also trusted
1308       if(Trusted && !Index->IsTrusted())
1309          continue;
1310
1311       // Grab the text package record
1312       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1313       if (_error->PendingError() == true)
1314          return false;
1315
1316       string PkgFile = Parse.FileName();
1317       if(Parse.SHA256Hash() != "")
1318          ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
1319       else if (Parse.SHA1Hash() != "")
1320          ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
1321       else
1322          ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1323       if (PkgFile.empty() == true)
1324          return _error->Error(_("The package index files are corrupted. No Filename: "
1325                               "field for package %s."),
1326                               Version.ParentPkg().Name());
1327
1328       Desc.URI = Index->ArchiveURI(PkgFile);
1329       Desc.Description = Index->ArchiveInfo(Version);
1330       Desc.Owner = this;
1331       Desc.ShortDesc = Version.ParentPkg().Name();
1332
1333       // See if we already have the file. (Legacy filenames)
1334       FileSize = Version->Size;
1335       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
1336       struct stat Buf;
1337       if (stat(FinalFile.c_str(),&Buf) == 0)
1338       {
1339          // Make sure the size matches
1340          if ((unsigned)Buf.st_size == Version->Size)
1341          {
1342             Complete = true;
1343             Local = true;
1344             Status = StatDone;
1345             StoreFilename = DestFile = FinalFile;
1346             return true;
1347          }
1348
1349          /* Hmm, we have a file and its size does not match, this means it is
1350             an old style mismatched arch */
1351          unlink(FinalFile.c_str());
1352       }
1353
1354       // Check it again using the new style output filenames
1355       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
1356       if (stat(FinalFile.c_str(),&Buf) == 0)
1357       {
1358          // Make sure the size matches
1359          if ((unsigned)Buf.st_size == Version->Size)
1360          {
1361             Complete = true;
1362             Local = true;
1363             Status = StatDone;
1364             StoreFilename = DestFile = FinalFile;
1365             return true;
1366          }
1367
1368          /* Hmm, we have a file and its size does not match, this shouldnt
1369             happen.. */
1370          unlink(FinalFile.c_str());
1371       }
1372
1373       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
1374
1375       // Check the destination file
1376       if (stat(DestFile.c_str(),&Buf) == 0)
1377       {
1378          // Hmm, the partial file is too big, erase it
1379          if ((unsigned)Buf.st_size > Version->Size)
1380             unlink(DestFile.c_str());
1381          else
1382             PartialSize = Buf.st_size;
1383       }
1384
1385       // Create the item
1386       Local = false;
1387       Desc.URI = Index->ArchiveURI(PkgFile);
1388       Desc.Description = Index->ArchiveInfo(Version);
1389       Desc.Owner = this;
1390       Desc.ShortDesc = Version.ParentPkg().Name();
1391       QueueURI(Desc);
1392
1393       Vf++;
1394       return true;
1395    }
1396    return false;
1397 }
1398                                                                         /*}}}*/
1399 // AcqArchive::Done - Finished fetching                                 /*{{{*/
1400 // ---------------------------------------------------------------------
1401 /* */
1402 void pkgAcqArchive::Done(string Message,unsigned long Size,string CalcHash,
1403                          pkgAcquire::MethodConfig *Cfg)
1404 {
1405    Item::Done(Message,Size,CalcHash,Cfg);
1406
1407    // Check the size
1408    if (Size != Version->Size)
1409    {
1410       Status = StatError;
1411       ErrorText = _("Size mismatch");
1412       return;
1413    }
1414
1415    // Check the hash
1416    if(ExpectedHash.toStr() != CalcHash)
1417    {
1418       Status = StatError;
1419       ErrorText = _("Hash Sum mismatch");
1420       if(FileExists(DestFile))
1421          Rename(DestFile,DestFile + ".FAILED");
1422       return;
1423    }
1424
1425    // Grab the output filename
1426    string FileName = LookupTag(Message,"Filename");
1427    if (FileName.empty() == true)
1428    {
1429       Status = StatError;
1430       ErrorText = "Method gave a blank filename";
1431       return;
1432    }
1433
1434    Complete = true;
1435
1436    // Reference filename
1437    if (FileName != DestFile)
1438    {
1439       StoreFilename = DestFile = FileName;
1440       Local = true;
1441       return;
1442    }
1443
1444    // Done, move it into position
1445    string FinalFile = _config->FindDir("Dir::Cache::Archives");
1446    FinalFile += flNotDir(StoreFilename);
1447    Rename(DestFile,FinalFile);
1448
1449    StoreFilename = DestFile = FinalFile;
1450    Complete = true;
1451 }
1452                                                                         /*}}}*/
1453 // AcqArchive::Failed - Failure handler                                 /*{{{*/
1454 // ---------------------------------------------------------------------
1455 /* Here we try other sources */
1456 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1457 {
1458    ErrorText = LookupTag(Message,"Message");
1459
1460    /* We don't really want to retry on failed media swaps, this prevents
1461       that. An interesting observation is that permanent failures are not
1462       recorded. */
1463    if (Cnf->Removable == true &&
1464        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1465    {
1466       // Vf = Version.FileList();
1467       while (Vf.end() == false) Vf++;
1468       StoreFilename = string();
1469       Item::Failed(Message,Cnf);
1470       return;
1471    }
1472
1473    if (QueueNext() == false)
1474    {
1475       // This is the retry counter
1476       if (Retries != 0 &&
1477           Cnf->LocalOnly == false &&
1478           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1479       {
1480          Retries--;
1481          Vf = Version.FileList();
1482          if (QueueNext() == true)
1483             return;
1484       }
1485
1486       StoreFilename = string();
1487       Item::Failed(Message,Cnf);
1488    }
1489 }
1490                                                                         /*}}}*/
1491 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
1492 // ---------------------------------------------------------------------
1493 bool pkgAcqArchive::IsTrusted()
1494 {
1495    return Trusted;
1496 }
1497                                                                         /*}}}*/
1498 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
1499 // ---------------------------------------------------------------------
1500 /* */
1501 void pkgAcqArchive::Finished()
1502 {
1503    if (Status == pkgAcquire::Item::StatDone &&
1504        Complete == true)
1505       return;
1506    StoreFilename = string();
1507 }
1508                                                                         /*}}}*/
1509 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
1510 // ---------------------------------------------------------------------
1511 /* The file is added to the queue */
1512 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
1513                        unsigned long Size,string Dsc,string ShortDesc,
1514                        const string &DestDir, const string &DestFilename) :
1515                        Item(Owner), ExpectedHash(Hash)
1516 {
1517    Retries = _config->FindI("Acquire::Retries",0);
1518
1519    if(!DestFilename.empty())
1520       DestFile = DestFilename;
1521    else if(!DestDir.empty())
1522       DestFile = DestDir + "/" + flNotDir(URI);
1523    else
1524       DestFile = flNotDir(URI);
1525
1526    // Create the item
1527    Desc.URI = URI;
1528    Desc.Description = Dsc;
1529    Desc.Owner = this;
1530
1531    // Set the short description to the archive component
1532    Desc.ShortDesc = ShortDesc;
1533
1534    // Get the transfer sizes
1535    FileSize = Size;
1536    struct stat Buf;
1537    if (stat(DestFile.c_str(),&Buf) == 0)
1538    {
1539       // Hmm, the partial file is too big, erase it
1540       if ((unsigned)Buf.st_size > Size)
1541          unlink(DestFile.c_str());
1542       else
1543          PartialSize = Buf.st_size;
1544    }
1545
1546    QueueURI(Desc);
1547 }
1548                                                                         /*}}}*/
1549 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
1550 // ---------------------------------------------------------------------
1551 /* */
1552 void pkgAcqFile::Done(string Message,unsigned long Size,string CalcHash,
1553                       pkgAcquire::MethodConfig *Cnf)
1554 {
1555    Item::Done(Message,Size,CalcHash,Cnf);
1556
1557    // Check the hash
1558    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
1559    {
1560       Status = StatError;
1561       ErrorText = "Hash Sum mismatch";
1562       Rename(DestFile,DestFile + ".FAILED");
1563       return;
1564    }
1565
1566    string FileName = LookupTag(Message,"Filename");
1567    if (FileName.empty() == true)
1568    {
1569       Status = StatError;
1570       ErrorText = "Method gave a blank filename";
1571       return;
1572    }
1573
1574    Complete = true;
1575
1576    // The files timestamp matches
1577    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1578       return;
1579
1580    // We have to copy it into place
1581    if (FileName != DestFile)
1582    {
1583       Local = true;
1584       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
1585           Cnf->Removable == true)
1586       {
1587          Desc.URI = "copy:" + FileName;
1588          QueueURI(Desc);
1589          return;
1590       }
1591
1592       // Erase the file if it is a symlink so we can overwrite it
1593       struct stat St;
1594       if (lstat(DestFile.c_str(),&St) == 0)
1595       {
1596          if (S_ISLNK(St.st_mode) != 0)
1597             unlink(DestFile.c_str());
1598       }
1599
1600       // Symlink the file
1601       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
1602       {
1603          ErrorText = "Link to " + DestFile + " failure ";
1604          Status = StatError;
1605          Complete = false;
1606       }
1607    }
1608 }
1609                                                                         /*}}}*/
1610 // AcqFile::Failed - Failure handler                                    /*{{{*/
1611 // ---------------------------------------------------------------------
1612 /* Here we try other sources */
1613 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1614 {
1615    ErrorText = LookupTag(Message,"Message");
1616
1617    // This is the retry counter
1618    if (Retries != 0 &&
1619        Cnf->LocalOnly == false &&
1620        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1621    {
1622       Retries--;
1623       QueueURI(Desc);
1624       return;
1625    }
1626
1627    Item::Failed(Message,Cnf);
1628 }
1629                                                                         /*}}}*/