apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <apt-pkg/acquire-item.h>
  17 #include <apt-pkg/configuration.h>
  18 #include <apt-pkg/aptconfiguration.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/vendorlist.h>
  21 #include <apt-pkg/error.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <apt-pkg/fileutl.h>
  24 #include <apt-pkg/md5.h>
  25 #include <apt-pkg/sha1.h>
  26 #include <apt-pkg/tagfile.h>
  27
  28 #include <apti18n.h>
  29
  30 #include <sys/stat.h>
  31 #include <unistd.h>
  32 #include <errno.h>
  33 #include <string>
  34 #include <sstream>
  35 #include <stdio.h>
  36                                                                         /*}}}*/
  37
  38 using namespace std;
  39
  40 // Acquire::Item::Item - Constructor                                    /*{{{*/
  41 // ---------------------------------------------------------------------
  42 /* */
  43 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  44                        PartialSize(0), Mode(0), ID(0), Complete(false),
  45                        Local(false), QueueCounter(0)
  46 {
  47    Owner->Add(this);
  48    Status = StatIdle;
  49 }
  50                                                                         /*}}}*/
  51 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  52 // ---------------------------------------------------------------------
  53 /* */
  54 pkgAcquire::Item::~Item()
  55 {
  56    Owner->Remove(this);
  57 }
  58                                                                         /*}}}*/
  59 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  60 // ---------------------------------------------------------------------
  61 /* We return to an idle state if there are still other queues that could
  62    fetch this object */
  63 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  64 {
  65    Status = StatIdle;
  66    ErrorText = LookupTag(Message,"Message");
  67    if (QueueCounter <= 1)
  68    {
  69       /* This indicates that the file is not available right now but might
  70          be sometime later. If we do a retry cycle then this should be
  71          retried [CDROMs] */
  72       if (Cnf->LocalOnly == true &&
  73           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  74       {
  75          Status = StatIdle;
  76          Dequeue();
  77          return;
  78       }
  79
  80       Status = StatError;
  81       Dequeue();
  82    }
  83 }
  84                                                                         /*}}}*/
  85 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
  86 // ---------------------------------------------------------------------
  87 /* Stash status and the file size. Note that setting Complete means
  88    sub-phases of the acquire process such as decompresion are operating */
  89 void pkgAcquire::Item::Start(string /*Message*/,unsigned long Size)
  90 {
  91    Status = StatFetching;
  92    if (FileSize == 0 && Complete == false)
  93       FileSize = Size;
  94 }
  95                                                                         /*}}}*/
  96 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
  97 // ---------------------------------------------------------------------
  98 /* */
  99 void pkgAcquire::Item::Done(string Message,unsigned long Size,string Hash,
 100                             pkgAcquire::MethodConfig *Cnf)
 101 {
 102    // We just downloaded something..
 103    string FileName = LookupTag(Message,"Filename");
 104    // we only inform the Log class if it was actually not a local thing
 105    if (Complete == false && !Local && FileName == DestFile)
 106    {
 107       if (Owner->Log != 0)
 108          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 109    }
 110
 111    if (FileSize == 0)
 112       FileSize= Size;
 113
 114    Status = StatDone;
 115    ErrorText = string();
 116    Owner->Dequeue(this);
 117 }
 118                                                                         /*}}}*/
 119 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 120 // ---------------------------------------------------------------------
 121 /* This helper function is used by alot of item methods as thier final
 122    step */
 123 void pkgAcquire::Item::Rename(string From,string To)
 124 {
 125    if (rename(From.c_str(),To.c_str()) != 0)
 126    {
 127       char S[300];
 128       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 129               From.c_str(),To.c_str());
 130       Status = StatError;
 131       ErrorText = S;
 132    }
 133 }
 134                                                                         /*}}}*/
 135 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 136 // ---------------------------------------------------------------------
 137 /* Get the DiffIndex file first and see if there are patches availabe
 138  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 139  * patches. If anything goes wrong in that process, it will fall back to
 140  * the original packages file
 141  */
 142 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 143                                  string URI,string URIDesc,string ShortDesc,
 144                                  HashString ExpectedHash)
 145    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 146      Description(URIDesc)
 147 {
 148
 149    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 150
 151    Desc.Description = URIDesc + "/DiffIndex";
 152    Desc.Owner = this;
 153    Desc.ShortDesc = ShortDesc;
 154    Desc.URI = URI + ".diff/Index";
 155
 156    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 157    DestFile += URItoFileName(URI) + string(".DiffIndex");
 158
 159    if(Debug)
 160       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 161
 162    // look for the current package file
 163    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 164    CurrentPackagesFile += URItoFileName(RealURI);
 165
 166    // FIXME: this file:/ check is a hack to prevent fetching
 167    //        from local sources. this is really silly, and
 168    //        should be fixed cleanly as soon as possible
 169    if(!FileExists(CurrentPackagesFile) ||
 170       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 171    {
 172       // we don't have a pkg file or we don't want to queue
 173       if(Debug)
 174          std::clog << "No index file, local or canceld by user" << std::endl;
 175       Failed("", NULL);
 176       return;
 177    }
 178
 179    if(Debug)
 180       std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): "
 181                 << CurrentPackagesFile << std::endl;
 182
 183    QueueURI(Desc);
 184
 185 }
 186                                                                         /*}}}*/
 187 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 188 // ---------------------------------------------------------------------
 189 /* The only header we use is the last-modified header. */
 190 string pkgAcqDiffIndex::Custom600Headers()
 191 {
 192    string Final = _config->FindDir("Dir::State::lists");
 193    Final += URItoFileName(RealURI) + string(".IndexDiff");
 194
 195    if(Debug)
 196       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 197
 198    struct stat Buf;
 199    if (stat(Final.c_str(),&Buf) != 0)
 200       return "\nIndex-File: true";
 201
 202    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 203 }
 204                                                                         /*}}}*/
 205 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 206 {
 207    if(Debug)
 208       std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile
 209                 << std::endl;
 210
 211    pkgTagSection Tags;
 212    string ServerSha1;
 213    vector<DiffInfo> available_patches;
 214
 215    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 216    pkgTagFile TF(&Fd);
 217    if (_error->PendingError() == true)
 218       return false;
 219
 220    if(TF.Step(Tags) == true)
 221    {
 222       bool found = false;
 223       DiffInfo d;
 224       string size;
 225
 226       string const tmp = Tags.FindS("SHA1-Current");
 227       std::stringstream ss(tmp);
 228       ss >> ServerSha1 >> size;
 229       unsigned long const ServerSize = atol(size.c_str());
 230
 231       FileFd fd(CurrentPackagesFile, FileFd::ReadOnlyGzip);
 232       SHA1Summation SHA1;
 233       SHA1.AddFD(fd.Fd(), fd.Size());
 234       string const local_sha1 = SHA1.Result();
 235
 236       if(local_sha1 == ServerSha1)
 237       {
 238          // we have the same sha1 as the server
 239          if(Debug)
 240             std::clog << "Package file is up-to-date" << std::endl;
 241          // set found to true, this will queue a pkgAcqIndexDiffs with
 242          // a empty availabe_patches
 243          found = true;
 244       }
 245       else
 246       {
 247          if(Debug)
 248             std::clog << "SHA1-Current: " << ServerSha1 << std::endl;
 249
 250          // check the historie and see what patches we need
 251          string const history = Tags.FindS("SHA1-History");
 252          std::stringstream hist(history);
 253          while(hist >> d.sha1 >> size >> d.file)
 254          {
 255             // read until the first match is found
 256             // from that point on, we probably need all diffs
 257             if(d.sha1 == local_sha1)
 258                found=true;
 259             else if (found == false)
 260                continue;
 261
 262             if(Debug)
 263                std::clog << "Need to get diff: " << d.file << std::endl;
 264             available_patches.push_back(d);
 265          }
 266
 267          if (available_patches.empty() == false)
 268          {
 269             // patching with too many files is rather slow compared to a fast download
 270             unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 271             if (fileLimit != 0 && fileLimit < available_patches.size())
 272             {
 273                if (Debug)
 274                   std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 275                         << ") so fallback to complete download" << std::endl;
 276                return false;
 277             }
 278
 279             // see if the patches are too big
 280             found = false; // it was true and it will be true again at the end
 281             d = *available_patches.begin();
 282             string const firstPatch = d.file;
 283             unsigned long patchesSize = 0;
 284             std::stringstream patches(Tags.FindS("SHA1-Patches"));
 285             while(patches >> d.sha1 >> size >> d.file)
 286             {
 287                if (firstPatch == d.file)
 288                   found = true;
 289                else if (found == false)
 290                   continue;
 291
 292                patchesSize += atol(size.c_str());
 293             }
 294             unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 295             if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 296             {
 297                if (Debug)
 298                   std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 299                         << ") so fallback to complete download" << std::endl;
 300                return false;
 301             }
 302          }
 303       }
 304
 305       // we have something, queue the next diff
 306       if(found)
 307       {
 308          // queue the diffs
 309          string::size_type const last_space = Description.rfind(" ");
 310          if(last_space != string::npos)
 311             Description.erase(last_space, Description.size()-last_space);
 312          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 313                               ExpectedHash, ServerSha1, available_patches);
 314          Complete = false;
 315          Status = StatDone;
 316          Dequeue();
 317          return true;
 318       }
 319    }
 320
 321    // Nothing found, report and return false
 322    // Failing here is ok, if we return false later, the full
 323    // IndexFile is queued
 324    if(Debug)
 325       std::clog << "Can't find a patch in the index file" << std::endl;
 326    return false;
 327 }
 328                                                                         /*}}}*/
 329 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)      /*{{{*/
 330 {
 331    if(Debug)
 332       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << std::endl
 333                 << "Falling back to normal index file aquire" << std::endl;
 334
 335    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 336                    ExpectedHash);
 337
 338    Complete = false;
 339    Status = StatDone;
 340    Dequeue();
 341 }
 342                                                                         /*}}}*/
 343 void pkgAcqDiffIndex::Done(string Message,unsigned long Size,string Md5Hash,    /*{{{*/
 344                            pkgAcquire::MethodConfig *Cnf)
 345 {
 346    if(Debug)
 347       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 348
 349    Item::Done(Message,Size,Md5Hash,Cnf);
 350
 351    string FinalFile;
 352    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 353
 354    // sucess in downloading the index
 355    // rename the index
 356    FinalFile += string(".IndexDiff");
 357    if(Debug)
 358       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 359                 << std::endl;
 360    Rename(DestFile,FinalFile);
 361    chmod(FinalFile.c_str(),0644);
 362    DestFile = FinalFile;
 363
 364    if(!ParseDiffIndex(DestFile))
 365       return Failed("", NULL);
 366
 367    Complete = true;
 368    Status = StatDone;
 369    Dequeue();
 370    return;
 371 }
 372                                                                         /*}}}*/
 373 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 374 // ---------------------------------------------------------------------
 375 /* The package diff is added to the queue. one object is constructed
 376  * for each diff and the index
 377  */
 378 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 379                                    string URI,string URIDesc,string ShortDesc,
 380                                    HashString ExpectedHash,
 381                                    string ServerSha1,
 382                                    vector<DiffInfo> diffs)
 383    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 384      available_patches(diffs), ServerSha1(ServerSha1)
 385 {
 386
 387    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 388    DestFile += URItoFileName(URI);
 389
 390    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 391
 392    Description = URIDesc;
 393    Desc.Owner = this;
 394    Desc.ShortDesc = ShortDesc;
 395
 396    if(available_patches.size() == 0)
 397    {
 398       // we are done (yeah!)
 399       Finish(true);
 400    }
 401    else
 402    {
 403       // get the next diff
 404       State = StateFetchDiff;
 405       QueueNextDiff();
 406    }
 407 }
 408                                                                         /*}}}*/
 409 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)     /*{{{*/
 410 {
 411    if(Debug)
 412       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl
 413                 << "Falling back to normal index file aquire" << std::endl;
 414    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 415                    ExpectedHash);
 416    Finish();
 417 }
 418                                                                         /*}}}*/
 419 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 420 void pkgAcqIndexDiffs::Finish(bool allDone)
 421 {
 422    // we restore the original name, this is required, otherwise
 423    // the file will be cleaned
 424    if(allDone)
 425    {
 426       DestFile = _config->FindDir("Dir::State::lists");
 427       DestFile += URItoFileName(RealURI);
 428
 429       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 430       {
 431          Status = StatAuthError;
 432          ErrorText = _("MD5Sum mismatch");
 433          Rename(DestFile,DestFile + ".FAILED");
 434          Dequeue();
 435          return;
 436       }
 437
 438       // this is for the "real" finish
 439       Complete = true;
 440       Status = StatDone;
 441       Dequeue();
 442       if(Debug)
 443          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 444       return;
 445    }
 446
 447    if(Debug)
 448       std::clog << "Finishing: " << Desc.URI << std::endl;
 449    Complete = false;
 450    Status = StatDone;
 451    Dequeue();
 452    return;
 453 }
 454                                                                         /*}}}*/
 455 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 456 {
 457
 458    // calc sha1 of the just patched file
 459    string FinalFile = _config->FindDir("Dir::State::lists");
 460    FinalFile += URItoFileName(RealURI);
 461
 462    FileFd fd(FinalFile, FileFd::ReadOnlyGzip);
 463    SHA1Summation SHA1;
 464    SHA1.AddFD(fd.Fd(), fd.Size());
 465    string local_sha1 = string(SHA1.Result());
 466    if(Debug)
 467       std::clog << "QueueNextDiff: "
 468                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 469
 470    // final file reached before all patches are applied
 471    if(local_sha1 == ServerSha1)
 472    {
 473       Finish(true);
 474       return true;
 475    }
 476
 477    // remove all patches until the next matching patch is found
 478    // this requires the Index file to be ordered
 479    for(vector<DiffInfo>::iterator I=available_patches.begin();
 480        available_patches.size() > 0 &&
 481           I != available_patches.end() &&
 482           (*I).sha1 != local_sha1;
 483        I++)
 484    {
 485       available_patches.erase(I);
 486    }
 487
 488    // error checking and falling back if no patch was found
 489    if(available_patches.size() == 0)
 490    {
 491       Failed("", NULL);
 492       return false;
 493    }
 494
 495    // queue the right diff
 496    Desc.URI = string(RealURI) + ".diff/" + available_patches[0].file + ".gz";
 497    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 498    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 499    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 500
 501    if(Debug)
 502       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 503
 504    QueueURI(Desc);
 505
 506    return true;
 507 }
 508                                                                         /*}}}*/
 509 void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash,   /*{{{*/
 510                             pkgAcquire::MethodConfig *Cnf)
 511 {
 512    if(Debug)
 513       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 514
 515    Item::Done(Message,Size,Md5Hash,Cnf);
 516
 517    string FinalFile;
 518    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 519
 520    // sucess in downloading a diff, enter ApplyDiff state
 521    if(State == StateFetchDiff)
 522    {
 523
 524       if(Debug)
 525          std::clog << "Sending to gzip method: " << FinalFile << std::endl;
 526
 527       string FileName = LookupTag(Message,"Filename");
 528       State = StateUnzipDiff;
 529       Local = true;
 530       Desc.URI = "gzip:" + FileName;
 531       DestFile += ".decomp";
 532       QueueURI(Desc);
 533       Mode = "gzip";
 534       return;
 535    }
 536
 537    // sucess in downloading a diff, enter ApplyDiff state
 538    if(State == StateUnzipDiff)
 539    {
 540
 541       // rred excepts the patch as $FinalFile.ed
 542       Rename(DestFile,FinalFile+".ed");
 543
 544       if(Debug)
 545          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 546
 547       State = StateApplyDiff;
 548       Local = true;
 549       Desc.URI = "rred:" + FinalFile;
 550       QueueURI(Desc);
 551       Mode = "rred";
 552       return;
 553    }
 554
 555
 556    // success in download/apply a diff, queue next (if needed)
 557    if(State == StateApplyDiff)
 558    {
 559       // remove the just applied patch
 560       available_patches.erase(available_patches.begin());
 561
 562       // move into place
 563       if(Debug)
 564       {
 565          std::clog << "Moving patched file in place: " << std::endl
 566                    << DestFile << " -> " << FinalFile << std::endl;
 567       }
 568       Rename(DestFile,FinalFile);
 569       chmod(FinalFile.c_str(),0644);
 570
 571       // see if there is more to download
 572       if(available_patches.size() > 0) {
 573          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 574                               ExpectedHash, ServerSha1, available_patches);
 575          return Finish();
 576       } else
 577          return Finish(true);
 578    }
 579 }
 580                                                                         /*}}}*/
 581 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 582 // ---------------------------------------------------------------------
 583 /* The package file is added to the queue and a second class is
 584    instantiated to fetch the revision file */
 585 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 586                          string URI,string URIDesc,string ShortDesc,
 587                          HashString ExpectedHash, string comprExt)
 588    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 589 {
 590    Decompression = false;
 591    Erase = false;
 592
 593    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 594    DestFile += URItoFileName(URI);
 595
 596    if(comprExt.empty())
 597    {
 598       // autoselect the compression method
 599       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 600       if (types.empty() == true)
 601          comprExt = "plain";
 602       else
 603          comprExt = "." + types[0];
 604    }
 605    CompressionExtension = ((comprExt == "plain" || comprExt == ".") ? "" : comprExt);
 606
 607    Desc.URI = URI + CompressionExtension;
 608
 609    Desc.Description = URIDesc;
 610    Desc.Owner = this;
 611    Desc.ShortDesc = ShortDesc;
 612
 613    QueueURI(Desc);
 614 }
 615                                                                         /*}}}*/
 616 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 617 // ---------------------------------------------------------------------
 618 /* The only header we use is the last-modified header. */
 619 string pkgAcqIndex::Custom600Headers()
 620 {
 621    string Final = _config->FindDir("Dir::State::lists");
 622    Final += URItoFileName(RealURI);
 623    if (_config->FindB("Acquire::GzipIndexes",false))
 624       Final += ".gz";
 625
 626    struct stat Buf;
 627    if (stat(Final.c_str(),&Buf) != 0)
 628       return "\nIndex-File: true";
 629
 630    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 631 }
 632                                                                         /*}}}*/
 633 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
 634 {
 635    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 636
 637    for (std::vector<std::string>::const_iterator t = types.begin();
 638         t != types.end(); t++)
 639    {
 640       // jump over all already tried compression types
 641       const unsigned int nameLen = Desc.URI.size() - (*t).size();
 642       if(Desc.URI.substr(nameLen) != *t)
 643          continue;
 644
 645       // we want to try it with the next extension (and make sure to
 646       // not skip over the end)
 647       t++;
 648       if (t == types.end())
 649          break;
 650
 651       // queue new download
 652       Desc.URI = Desc.URI.substr(0, nameLen) + *t;
 653       new pkgAcqIndex(Owner, RealURI, Desc.Description, Desc.ShortDesc,
 654       ExpectedHash, string(".").append(*t));
 655
 656       Status = StatDone;
 657       Complete = false;
 658       Dequeue();
 659       return;
 660    }
 661
 662    // on decompression failure, remove bad versions in partial/
 663    if(Decompression && Erase) {
 664       string s = _config->FindDir("Dir::State::lists") + "partial/";
 665       s += URItoFileName(RealURI);
 666       unlink(s.c_str());
 667    }
 668
 669    Item::Failed(Message,Cnf);
 670 }
 671                                                                         /*}}}*/
 672 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
 673 // ---------------------------------------------------------------------
 674 /* This goes through a number of states.. On the initial fetch the
 675    method could possibly return an alternate filename which points
 676    to the uncompressed version of the file. If this is so the file
 677    is copied into the partial directory. In all other cases the file
 678    is decompressed with a gzip uri. */
 679 void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash,
 680                        pkgAcquire::MethodConfig *Cfg)
 681 {
 682    Item::Done(Message,Size,Hash,Cfg);
 683
 684    if (Decompression == true)
 685    {
 686       if (_config->FindB("Debug::pkgAcquire::Auth", false))
 687       {
 688          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
 689          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
 690       }
 691
 692       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
 693       {
 694          Status = StatAuthError;
 695          ErrorText = _("Hash Sum mismatch");
 696          Rename(DestFile,DestFile + ".FAILED");
 697          return;
 698       }
 699       // Done, move it into position
 700       string FinalFile = _config->FindDir("Dir::State::lists");
 701       FinalFile += URItoFileName(RealURI);
 702       Rename(DestFile,FinalFile);
 703       chmod(FinalFile.c_str(),0644);
 704
 705       /* We restore the original name to DestFile so that the clean operation
 706          will work OK */
 707       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 708       DestFile += URItoFileName(RealURI);
 709
 710       // Remove the compressed version.
 711       if (Erase == true)
 712          unlink(DestFile.c_str());
 713       return;
 714    }
 715
 716    Erase = false;
 717    Complete = true;
 718
 719    // Handle the unzipd case
 720    string FileName = LookupTag(Message,"Alt-Filename");
 721    if (FileName.empty() == false)
 722    {
 723       // The files timestamp matches
 724       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
 725          return;
 726       Decompression = true;
 727       Local = true;
 728       DestFile += ".decomp";
 729       Desc.URI = "copy:" + FileName;
 730       QueueURI(Desc);
 731       Mode = "copy";
 732       return;
 733    }
 734
 735    FileName = LookupTag(Message,"Filename");
 736    if (FileName.empty() == true)
 737    {
 738       Status = StatError;
 739       ErrorText = "Method gave a blank filename";
 740    }
 741
 742    string compExt = flExtension(flNotDir(URI(Desc.URI).Path));
 743
 744    // The files timestamp matches
 745    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) {
 746        if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
 747           // Update DestFile for .gz suffix so that the clean operation keeps it
 748           DestFile += ".gz";
 749       return;
 750     }
 751
 752    if (FileName == DestFile)
 753       Erase = true;
 754    else
 755       Local = true;
 756
 757    string decompProg;
 758
 759    // If we enable compressed indexes and already have gzip, keep it
 760    if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) {
 761       string FinalFile = _config->FindDir("Dir::State::lists");
 762       FinalFile += URItoFileName(RealURI) + ".gz";
 763       //if(Debug)
 764       //   std::clog << "pkgAcqIndex: keeping gzipped " << FinalFile << endl;
 765       Rename(DestFile,FinalFile);
 766       chmod(FinalFile.c_str(),0644);
 767
 768       // Update DestFile for .gz suffix so that the clean operation keeps it
 769       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 770       DestFile += URItoFileName(RealURI) + ".gz";
 771       return;
 772     }
 773
 774    // get the binary name for your used compression type
 775    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
 776    if(decompProg.empty() == false);
 777    // flExtensions returns the full name if no extension is found
 778    // this is why we have this complicated compare operation here
 779    // FIMXE: add a new flJustExtension() that return "" if no
 780    //        extension is found and use that above so that it can
 781    //        be tested against ""
 782    else if(compExt == flNotDir(URI(Desc.URI).Path))
 783       decompProg = "copy";
 784    else {
 785       _error->Error("Unsupported extension: %s", compExt.c_str());
 786       return;
 787    }
 788
 789    Decompression = true;
 790    DestFile += ".decomp";
 791    Desc.URI = decompProg + ":" + FileName;
 792    QueueURI(Desc);
 793    Mode = decompProg.c_str();
 794 }
 795                                                                         /*}}}*/
 796 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
 797 // ---------------------------------------------------------------------
 798 /* The Translation file is added to the queue */
 799 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
 800                             string URI,string URIDesc,string ShortDesc)
 801   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
 802 {
 803 }
 804                                                                         /*}}}*/
 805 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
 806 // ---------------------------------------------------------------------
 807 /* */
 808 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 809 {
 810    if (Cnf->LocalOnly == true ||
 811        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 812    {
 813       // Ignore this
 814       Status = StatDone;
 815       Complete = false;
 816       Dequeue();
 817       return;
 818    }
 819
 820    Item::Failed(Message,Cnf);
 821 }
 822                                                                         /*}}}*/
 823 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
 824                              string URI,string URIDesc,string ShortDesc,
 825                              string MetaIndexURI, string MetaIndexURIDesc,
 826                              string MetaIndexShortDesc,
 827                              const vector<IndexTarget*>* IndexTargets,
 828                              indexRecords* MetaIndexParser) :
 829    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
 830    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
 831    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
 832 {
 833    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 834    DestFile += URItoFileName(URI);
 835
 836    // remove any partial downloaded sig-file in partial/.
 837    // it may confuse proxies and is too small to warrant a
 838    // partial download anyway
 839    unlink(DestFile.c_str());
 840
 841    // Create the item
 842    Desc.Description = URIDesc;
 843    Desc.Owner = this;
 844    Desc.ShortDesc = ShortDesc;
 845    Desc.URI = URI;
 846
 847    string Final = _config->FindDir("Dir::State::lists");
 848    Final += URItoFileName(RealURI);
 849    struct stat Buf;
 850    if (stat(Final.c_str(),&Buf) == 0)
 851    {
 852       // File was already in place.  It needs to be re-downloaded/verified
 853       // because Release might have changed, we do give it a differnt
 854       // name than DestFile because otherwise the http method will
 855       // send If-Range requests and there are too many broken servers
 856       // out there that do not understand them
 857       LastGoodSig = DestFile+".reverify";
 858       Rename(Final,LastGoodSig);
 859    }
 860
 861    QueueURI(Desc);
 862 }
 863                                                                         /*}}}*/
 864 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
 865 // ---------------------------------------------------------------------
 866 /* The only header we use is the last-modified header. */
 867 string pkgAcqMetaSig::Custom600Headers()
 868 {
 869    struct stat Buf;
 870    if (stat(LastGoodSig.c_str(),&Buf) != 0)
 871       return "\nIndex-File: true";
 872
 873    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 874 }
 875
 876 void pkgAcqMetaSig::Done(string Message,unsigned long Size,string MD5,
 877                          pkgAcquire::MethodConfig *Cfg)
 878 {
 879    Item::Done(Message,Size,MD5,Cfg);
 880
 881    string FileName = LookupTag(Message,"Filename");
 882    if (FileName.empty() == true)
 883    {
 884       Status = StatError;
 885       ErrorText = "Method gave a blank filename";
 886       return;
 887    }
 888
 889    if (FileName != DestFile)
 890    {
 891       // We have to copy it into place
 892       Local = true;
 893       Desc.URI = "copy:" + FileName;
 894       QueueURI(Desc);
 895       return;
 896    }
 897
 898    Complete = true;
 899
 900    // put the last known good file back on i-m-s hit (it will
 901    // be re-verified again)
 902    // Else do nothing, we have the new file in DestFile then
 903    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 904       Rename(LastGoodSig, DestFile);
 905
 906    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
 907    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
 908                        DestFile, IndexTargets, MetaIndexParser);
 909
 910 }
 911                                                                         /*}}}*/
 912 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
 913 {
 914    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 915
 916    // if we get a network error we fail gracefully
 917    if(Status == StatTransientNetworkError)
 918    {
 919       Item::Failed(Message,Cnf);
 920       // move the sigfile back on transient network failures
 921       if(FileExists(DestFile))
 922          Rename(LastGoodSig,Final);
 923
 924       // set the status back to , Item::Failed likes to reset it
 925       Status = pkgAcquire::Item::StatTransientNetworkError;
 926       return;
 927    }
 928
 929    // Delete any existing sigfile when the acquire failed
 930    unlink(Final.c_str());
 931
 932    // queue a pkgAcqMetaIndex with no sigfile
 933    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
 934                        "", IndexTargets, MetaIndexParser);
 935
 936    if (Cnf->LocalOnly == true ||
 937        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 938    {
 939       // Ignore this
 940       Status = StatDone;
 941       Complete = false;
 942       Dequeue();
 943       return;
 944    }
 945
 946    Item::Failed(Message,Cnf);
 947 }
 948                                                                         /*}}}*/
 949 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
 950                                  string URI,string URIDesc,string ShortDesc,
 951                                  string SigFile,
 952                                  const vector<struct IndexTarget*>* IndexTargets,
 953                                  indexRecords* MetaIndexParser) :
 954    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
 955    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
 956 {
 957    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 958    DestFile += URItoFileName(URI);
 959
 960    // Create the item
 961    Desc.Description = URIDesc;
 962    Desc.Owner = this;
 963    Desc.ShortDesc = ShortDesc;
 964    Desc.URI = URI;
 965
 966    QueueURI(Desc);
 967 }
 968                                                                         /*}}}*/
 969 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
 970 // ---------------------------------------------------------------------
 971 /* The only header we use is the last-modified header. */
 972 string pkgAcqMetaIndex::Custom600Headers()
 973 {
 974    string Final = _config->FindDir("Dir::State::lists");
 975    Final += URItoFileName(RealURI);
 976
 977    struct stat Buf;
 978    if (stat(Final.c_str(),&Buf) != 0)
 979       return "\nIndex-File: true";
 980
 981    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 982 }
 983                                                                         /*}}}*/
 984 void pkgAcqMetaIndex::Done(string Message,unsigned long Size,string Hash,       /*{{{*/
 985                            pkgAcquire::MethodConfig *Cfg)
 986 {
 987    Item::Done(Message,Size,Hash,Cfg);
 988
 989    // MetaIndexes are done in two passes: one to download the
 990    // metaindex with an appropriate method, and a second to verify it
 991    // with the gpgv method
 992
 993    if (AuthPass == true)
 994    {
 995       AuthDone(Message);
 996    }
 997    else
 998    {
 999       RetrievalDone(Message);
1000       if (!Complete)
1001          // Still more retrieving to do
1002          return;
1003
1004       if (SigFile == "")
1005       {
1006          // There was no signature file, so we are finished.  Download
1007          // the indexes without verification.
1008          QueueIndexes(false);
1009       }
1010       else
1011       {
1012          // There was a signature file, so pass it to gpgv for
1013          // verification
1014
1015          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1016             std::cerr << "Metaindex acquired, queueing gpg verification ("
1017                       << SigFile << "," << DestFile << ")\n";
1018          AuthPass = true;
1019          Desc.URI = "gpgv:" + SigFile;
1020          QueueURI(Desc);
1021          Mode = "gpgv";
1022       }
1023    }
1024 }
1025                                                                         /*}}}*/
1026 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
1027 {
1028    // We have just finished downloading a Release file (it is not
1029    // verified yet)
1030
1031    string FileName = LookupTag(Message,"Filename");
1032    if (FileName.empty() == true)
1033    {
1034       Status = StatError;
1035       ErrorText = "Method gave a blank filename";
1036       return;
1037    }
1038
1039    if (FileName != DestFile)
1040    {
1041       Local = true;
1042       Desc.URI = "copy:" + FileName;
1043       QueueURI(Desc);
1044       return;
1045    }
1046
1047    // see if the download was a IMSHit
1048    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1049    Complete = true;
1050
1051    string FinalFile = _config->FindDir("Dir::State::lists");
1052    FinalFile += URItoFileName(RealURI);
1053
1054    // If we get a IMS hit we can remove the empty file in partial
1055    // othersie we move the file in place
1056    if (IMSHit)
1057       unlink(DestFile.c_str());
1058    else
1059       Rename(DestFile,FinalFile);
1060
1061    chmod(FinalFile.c_str(),0644);
1062    DestFile = FinalFile;
1063 }
1064                                                                         /*}}}*/
1065 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1066 {
1067    // At this point, the gpgv method has succeeded, so there is a
1068    // valid signature from a key in the trusted keyring.  We
1069    // perform additional verification of its contents, and use them
1070    // to verify the indexes we are about to download
1071
1072    if (!MetaIndexParser->Load(DestFile))
1073    {
1074       Status = StatAuthError;
1075       ErrorText = MetaIndexParser->ErrorText;
1076       return;
1077    }
1078
1079    if (!VerifyVendor(Message))
1080    {
1081       return;
1082    }
1083
1084    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1085       std::cerr << "Signature verification succeeded: "
1086                 << DestFile << std::endl;
1087
1088    // Download further indexes with verification
1089    QueueIndexes(true);
1090
1091    // Done, move signature file into position
1092
1093    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1094       URItoFileName(RealURI) + ".gpg";
1095    Rename(SigFile,VerifiedSigFile);
1096    chmod(VerifiedSigFile.c_str(),0644);
1097 }
1098                                                                         /*}}}*/
1099 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1100 {
1101    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1102         Target != IndexTargets->end();
1103         Target++)
1104    {
1105       HashString ExpectedIndexHash;
1106       if (verify)
1107       {
1108          const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1109          if (!Record)
1110          {
1111             Status = StatAuthError;
1112             ErrorText = "Unable to find expected entry  "
1113                + (*Target)->MetaKey + " in Meta-index file (malformed Release file?)";
1114             return;
1115          }
1116          ExpectedIndexHash = Record->Hash;
1117          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1118          {
1119             std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1120             std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1121          }
1122          if (ExpectedIndexHash.empty())
1123          {
1124             Status = StatAuthError;
1125             ErrorText = "Unable to find hash sum for "
1126                + (*Target)->MetaKey + " in Meta-index file";
1127             return;
1128          }
1129       }
1130
1131       // Queue Packages file (either diff or full packages files, depending
1132       // on the users option)
1133       if(_config->FindB("Acquire::PDiffs",true) == true)
1134          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1135                              (*Target)->ShortDesc, ExpectedIndexHash);
1136       else
1137          new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description,
1138                             (*Target)->ShortDesc, ExpectedIndexHash);
1139    }
1140 }
1141                                                                         /*}}}*/
1142 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1143 {
1144 //    // Maybe this should be made available from above so we don't have
1145 //    // to read and parse it every time?
1146 //    pkgVendorList List;
1147 //    List.ReadMainList();
1148
1149 //    const Vendor* Vndr = NULL;
1150 //    for (std::vector<string>::const_iterator I = GPGVOutput.begin(); I != GPGVOutput.end(); I++)
1151 //    {
1152 //       string::size_type pos = (*I).find("VALIDSIG ");
1153 //       if (_config->FindB("Debug::Vendor", false))
1154 //          std::cerr << "Looking for VALIDSIG in \"" << (*I) << "\": pos " << pos
1155 //                    << std::endl;
1156 //       if (pos != std::string::npos)
1157 //       {
1158 //          string Fingerprint = (*I).substr(pos+sizeof("VALIDSIG"));
1159 //          if (_config->FindB("Debug::Vendor", false))
1160 //             std::cerr << "Looking for \"" << Fingerprint << "\" in vendor..." <<
1161 //                std::endl;
1162 //          Vndr = List.FindVendor(Fingerprint) != "";
1163 //          if (Vndr != NULL);
1164 //          break;
1165 //       }
1166 //    }
1167    string::size_type pos;
1168
1169    // check for missing sigs (that where not fatal because otherwise we had
1170    // bombed earlier)
1171    string missingkeys;
1172    string msg = _("There is no public key available for the "
1173                   "following key IDs:\n");
1174    pos = Message.find("NO_PUBKEY ");
1175    if (pos != std::string::npos)
1176    {
1177       string::size_type start = pos+strlen("NO_PUBKEY ");
1178       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1179       missingkeys += (Fingerprint);
1180    }
1181    if(!missingkeys.empty())
1182       _error->Warning("%s", string(msg+missingkeys).c_str());
1183
1184    string Transformed = MetaIndexParser->GetExpectedDist();
1185
1186    if (Transformed == "../project/experimental")
1187    {
1188       Transformed = "experimental";
1189    }
1190
1191    pos = Transformed.rfind('/');
1192    if (pos != string::npos)
1193    {
1194       Transformed = Transformed.substr(0, pos);
1195    }
1196
1197    if (Transformed == ".")
1198    {
1199       Transformed = "";
1200    }
1201
1202    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1203    {
1204       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1205       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1206       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1207    }
1208
1209    if (MetaIndexParser->CheckDist(Transformed) == false)
1210    {
1211       // This might become fatal one day
1212 //       Status = StatAuthError;
1213 //       ErrorText = "Conflicting distribution; expected "
1214 //          + MetaIndexParser->GetExpectedDist() + " but got "
1215 //          + MetaIndexParser->GetDist();
1216 //       return false;
1217       if (!Transformed.empty())
1218       {
1219          _error->Warning("Conflicting distribution: %s (expected %s but got %s)",
1220                          Desc.Description.c_str(),
1221                          Transformed.c_str(),
1222                          MetaIndexParser->GetDist().c_str());
1223       }
1224    }
1225
1226    return true;
1227 }
1228                                                                         /*}}}*/
1229 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1230 // ---------------------------------------------------------------------
1231 /* */
1232 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1233 {
1234    if (AuthPass == true)
1235    {
1236       // if we fail the authentication but got the file via a IMS-Hit
1237       // this means that the file wasn't downloaded and that it might be
1238       // just stale (server problem, proxy etc). we delete what we have
1239       // queue it again without i-m-s
1240       // alternatively we could just unlink the file and let the user try again
1241       if (IMSHit)
1242       {
1243          Complete = false;
1244          Local = false;
1245          AuthPass = false;
1246          unlink(DestFile.c_str());
1247
1248          DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1249          DestFile += URItoFileName(RealURI);
1250          Desc.URI = RealURI;
1251          QueueURI(Desc);
1252          return;
1253       }
1254
1255       // gpgv method failed
1256       _error->Warning("GPG error: %s: %s",
1257                       Desc.Description.c_str(),
1258                       LookupTag(Message,"Message").c_str());
1259
1260    }
1261
1262    // No Release file was present, or verification failed, so fall
1263    // back to queueing Packages files without verification
1264    QueueIndexes(false);
1265 }
1266                                                                         /*}}}*/
1267 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1268 // ---------------------------------------------------------------------
1269 /* This just sets up the initial fetch environment and queues the first
1270    possibilitiy */
1271 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1272                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1273                              string &StoreFilename) :
1274                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1275                StoreFilename(StoreFilename), Vf(Version.FileList()),
1276                Trusted(false)
1277 {
1278    Retries = _config->FindI("Acquire::Retries",0);
1279
1280    if (Version.Arch() == 0)
1281    {
1282       _error->Error(_("I wasn't able to locate a file for the %s package. "
1283                       "This might mean you need to manually fix this package. "
1284                       "(due to missing arch)"),
1285                     Version.ParentPkg().Name());
1286       return;
1287    }
1288
1289    /* We need to find a filename to determine the extension. We make the
1290       assumption here that all the available sources for this version share
1291       the same extension.. */
1292    // Skip not source sources, they do not have file fields.
1293    for (; Vf.end() == false; Vf++)
1294    {
1295       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1296          continue;
1297       break;
1298    }
1299
1300    // Does not really matter here.. we are going to fail out below
1301    if (Vf.end() != true)
1302    {
1303       // If this fails to get a file name we will bomb out below.
1304       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1305       if (_error->PendingError() == true)
1306          return;
1307
1308       // Generate the final file name as: package_version_arch.foo
1309       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1310                       QuoteString(Version.VerStr(),"_:") + '_' +
1311                       QuoteString(Version.Arch(),"_:.") +
1312                       "." + flExtension(Parse.FileName());
1313    }
1314
1315    // check if we have one trusted source for the package. if so, switch
1316    // to "TrustedOnly" mode
1317    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; i++)
1318    {
1319       pkgIndexFile *Index;
1320       if (Sources->FindIndex(i.File(),Index) == false)
1321          continue;
1322       if (_config->FindB("Debug::pkgAcquire::Auth", false))
1323       {
1324          std::cerr << "Checking index: " << Index->Describe()
1325                    << "(Trusted=" << Index->IsTrusted() << ")\n";
1326       }
1327       if (Index->IsTrusted()) {
1328          Trusted = true;
1329          break;
1330       }
1331    }
1332
1333    // "allow-unauthenticated" restores apts old fetching behaviour
1334    // that means that e.g. unauthenticated file:// uris are higher
1335    // priority than authenticated http:// uris
1336    if (_config->FindB("APT::Get::AllowUnauthenticated",false) == true)
1337       Trusted = false;
1338
1339    // Select a source
1340    if (QueueNext() == false && _error->PendingError() == false)
1341       _error->Error(_("I wasn't able to locate file for the %s package. "
1342                     "This might mean you need to manually fix this package."),
1343                     Version.ParentPkg().Name());
1344 }
1345                                                                         /*}}}*/
1346 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1347 // ---------------------------------------------------------------------
1348 /* This queues the next available file version for download. It checks if
1349    the archive is already available in the cache and stashs the MD5 for
1350    checking later. */
1351 bool pkgAcqArchive::QueueNext()
1352 {
1353    for (; Vf.end() == false; Vf++)
1354    {
1355       // Ignore not source sources
1356       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1357          continue;
1358
1359       // Try to cross match against the source list
1360       pkgIndexFile *Index;
1361       if (Sources->FindIndex(Vf.File(),Index) == false)
1362             continue;
1363
1364       // only try to get a trusted package from another source if that source
1365       // is also trusted
1366       if(Trusted && !Index->IsTrusted())
1367          continue;
1368
1369       // Grab the text package record
1370       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1371       if (_error->PendingError() == true)
1372          return false;
1373
1374       string PkgFile = Parse.FileName();
1375       if(Parse.SHA256Hash() != "")
1376          ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
1377       else if (Parse.SHA1Hash() != "")
1378          ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
1379       else
1380          ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1381       if (PkgFile.empty() == true)
1382          return _error->Error(_("The package index files are corrupted. No Filename: "
1383                               "field for package %s."),
1384                               Version.ParentPkg().Name());
1385
1386       Desc.URI = Index->ArchiveURI(PkgFile);
1387       Desc.Description = Index->ArchiveInfo(Version);
1388       Desc.Owner = this;
1389       Desc.ShortDesc = Version.ParentPkg().Name();
1390
1391       // See if we already have the file. (Legacy filenames)
1392       FileSize = Version->Size;
1393       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
1394       struct stat Buf;
1395       if (stat(FinalFile.c_str(),&Buf) == 0)
1396       {
1397          // Make sure the size matches
1398          if ((unsigned)Buf.st_size == Version->Size)
1399          {
1400             Complete = true;
1401             Local = true;
1402             Status = StatDone;
1403             StoreFilename = DestFile = FinalFile;
1404             return true;
1405          }
1406
1407          /* Hmm, we have a file and its size does not match, this means it is
1408             an old style mismatched arch */
1409          unlink(FinalFile.c_str());
1410       }
1411
1412       // Check it again using the new style output filenames
1413       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
1414       if (stat(FinalFile.c_str(),&Buf) == 0)
1415       {
1416          // Make sure the size matches
1417          if ((unsigned)Buf.st_size == Version->Size)
1418          {
1419             Complete = true;
1420             Local = true;
1421             Status = StatDone;
1422             StoreFilename = DestFile = FinalFile;
1423             return true;
1424          }
1425
1426          /* Hmm, we have a file and its size does not match, this shouldnt
1427             happen.. */
1428          unlink(FinalFile.c_str());
1429       }
1430
1431       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
1432
1433       // Check the destination file
1434       if (stat(DestFile.c_str(),&Buf) == 0)
1435       {
1436          // Hmm, the partial file is too big, erase it
1437          if ((unsigned)Buf.st_size > Version->Size)
1438             unlink(DestFile.c_str());
1439          else
1440             PartialSize = Buf.st_size;
1441       }
1442
1443       // Create the item
1444       Local = false;
1445       Desc.URI = Index->ArchiveURI(PkgFile);
1446       Desc.Description = Index->ArchiveInfo(Version);
1447       Desc.Owner = this;
1448       Desc.ShortDesc = Version.ParentPkg().Name();
1449       QueueURI(Desc);
1450
1451       Vf++;
1452       return true;
1453    }
1454    return false;
1455 }
1456                                                                         /*}}}*/
1457 // AcqArchive::Done - Finished fetching                                 /*{{{*/
1458 // ---------------------------------------------------------------------
1459 /* */
1460 void pkgAcqArchive::Done(string Message,unsigned long Size,string CalcHash,
1461                          pkgAcquire::MethodConfig *Cfg)
1462 {
1463    Item::Done(Message,Size,CalcHash,Cfg);
1464
1465    // Check the size
1466    if (Size != Version->Size)
1467    {
1468       Status = StatError;
1469       ErrorText = _("Size mismatch");
1470       return;
1471    }
1472
1473    // Check the hash
1474    if(ExpectedHash.toStr() != CalcHash)
1475    {
1476       Status = StatError;
1477       ErrorText = _("Hash Sum mismatch");
1478       if(FileExists(DestFile))
1479          Rename(DestFile,DestFile + ".FAILED");
1480       return;
1481    }
1482
1483    // Grab the output filename
1484    string FileName = LookupTag(Message,"Filename");
1485    if (FileName.empty() == true)
1486    {
1487       Status = StatError;
1488       ErrorText = "Method gave a blank filename";
1489       return;
1490    }
1491
1492    Complete = true;
1493
1494    // Reference filename
1495    if (FileName != DestFile)
1496    {
1497       StoreFilename = DestFile = FileName;
1498       Local = true;
1499       return;
1500    }
1501
1502    // Done, move it into position
1503    string FinalFile = _config->FindDir("Dir::Cache::Archives");
1504    FinalFile += flNotDir(StoreFilename);
1505    Rename(DestFile,FinalFile);
1506
1507    StoreFilename = DestFile = FinalFile;
1508    Complete = true;
1509 }
1510                                                                         /*}}}*/
1511 // AcqArchive::Failed - Failure handler                                 /*{{{*/
1512 // ---------------------------------------------------------------------
1513 /* Here we try other sources */
1514 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1515 {
1516    ErrorText = LookupTag(Message,"Message");
1517
1518    /* We don't really want to retry on failed media swaps, this prevents
1519       that. An interesting observation is that permanent failures are not
1520       recorded. */
1521    if (Cnf->Removable == true &&
1522        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1523    {
1524       // Vf = Version.FileList();
1525       while (Vf.end() == false) Vf++;
1526       StoreFilename = string();
1527       Item::Failed(Message,Cnf);
1528       return;
1529    }
1530
1531    if (QueueNext() == false)
1532    {
1533       // This is the retry counter
1534       if (Retries != 0 &&
1535           Cnf->LocalOnly == false &&
1536           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1537       {
1538          Retries--;
1539          Vf = Version.FileList();
1540          if (QueueNext() == true)
1541             return;
1542       }
1543
1544       StoreFilename = string();
1545       Item::Failed(Message,Cnf);
1546    }
1547 }
1548                                                                         /*}}}*/
1549 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
1550 // ---------------------------------------------------------------------
1551 bool pkgAcqArchive::IsTrusted()
1552 {
1553    return Trusted;
1554 }
1555                                                                         /*}}}*/
1556 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
1557 // ---------------------------------------------------------------------
1558 /* */
1559 void pkgAcqArchive::Finished()
1560 {
1561    if (Status == pkgAcquire::Item::StatDone &&
1562        Complete == true)
1563       return;
1564    StoreFilename = string();
1565 }
1566                                                                         /*}}}*/
1567 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
1568 // ---------------------------------------------------------------------
1569 /* The file is added to the queue */
1570 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
1571                        unsigned long Size,string Dsc,string ShortDesc,
1572                        const string &DestDir, const string &DestFilename,
1573                        bool IsIndexFile) :
1574                        Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile)
1575 {
1576    Retries = _config->FindI("Acquire::Retries",0);
1577
1578    if(!DestFilename.empty())
1579       DestFile = DestFilename;
1580    else if(!DestDir.empty())
1581       DestFile = DestDir + "/" + flNotDir(URI);
1582    else
1583       DestFile = flNotDir(URI);
1584
1585    // Create the item
1586    Desc.URI = URI;
1587    Desc.Description = Dsc;
1588    Desc.Owner = this;
1589
1590    // Set the short description to the archive component
1591    Desc.ShortDesc = ShortDesc;
1592
1593    // Get the transfer sizes
1594    FileSize = Size;
1595    struct stat Buf;
1596    if (stat(DestFile.c_str(),&Buf) == 0)
1597    {
1598       // Hmm, the partial file is too big, erase it
1599       if ((unsigned)Buf.st_size > Size)
1600          unlink(DestFile.c_str());
1601       else
1602          PartialSize = Buf.st_size;
1603    }
1604
1605    QueueURI(Desc);
1606 }
1607                                                                         /*}}}*/
1608 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
1609 // ---------------------------------------------------------------------
1610 /* */
1611 void pkgAcqFile::Done(string Message,unsigned long Size,string CalcHash,
1612                       pkgAcquire::MethodConfig *Cnf)
1613 {
1614    Item::Done(Message,Size,CalcHash,Cnf);
1615
1616    // Check the hash
1617    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
1618    {
1619       Status = StatError;
1620       ErrorText = "Hash Sum mismatch";
1621       Rename(DestFile,DestFile + ".FAILED");
1622       return;
1623    }
1624
1625    string FileName = LookupTag(Message,"Filename");
1626    if (FileName.empty() == true)
1627    {
1628       Status = StatError;
1629       ErrorText = "Method gave a blank filename";
1630       return;
1631    }
1632
1633    Complete = true;
1634
1635    // The files timestamp matches
1636    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1637       return;
1638
1639    // We have to copy it into place
1640    if (FileName != DestFile)
1641    {
1642       Local = true;
1643       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
1644           Cnf->Removable == true)
1645       {
1646          Desc.URI = "copy:" + FileName;
1647          QueueURI(Desc);
1648          return;
1649       }
1650
1651       // Erase the file if it is a symlink so we can overwrite it
1652       struct stat St;
1653       if (lstat(DestFile.c_str(),&St) == 0)
1654       {
1655          if (S_ISLNK(St.st_mode) != 0)
1656             unlink(DestFile.c_str());
1657       }
1658
1659       // Symlink the file
1660       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
1661       {
1662          ErrorText = "Link to " + DestFile + " failure ";
1663          Status = StatError;
1664          Complete = false;
1665       }
1666    }
1667 }
1668                                                                         /*}}}*/
1669 // AcqFile::Failed - Failure handler                                    /*{{{*/
1670 // ---------------------------------------------------------------------
1671 /* Here we try other sources */
1672 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1673 {
1674    ErrorText = LookupTag(Message,"Message");
1675
1676    // This is the retry counter
1677    if (Retries != 0 &&
1678        Cnf->LocalOnly == false &&
1679        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1680    {
1681       Retries--;
1682       QueueURI(Desc);
1683       return;
1684    }
1685
1686    Item::Failed(Message,Cnf);
1687 }
1688                                                                         /*}}}*/
1689 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
1690 // ---------------------------------------------------------------------
1691 /* The only header we use is the last-modified header. */
1692 string pkgAcqFile::Custom600Headers()
1693 {
1694    if (IsIndexFile)
1695       return "\nIndex-File: true";
1696    else
1697       return "";
1698 }
1699                                                                         /*}}}*/