apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <apt-pkg/acquire-item.h>
  17 #include <apt-pkg/configuration.h>
  18 #include <apt-pkg/aptconfiguration.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/vendorlist.h>
  21 #include <apt-pkg/error.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <apt-pkg/fileutl.h>
  24 #include <apt-pkg/md5.h>
  25 #include <apt-pkg/sha1.h>
  26 #include <apt-pkg/tagfile.h>
  27
  28 #include <apti18n.h>
  29
  30 #include <sys/stat.h>
  31 #include <unistd.h>
  32 #include <errno.h>
  33 #include <string>
  34 #include <sstream>
  35 #include <stdio.h>
  36 #include <ctime>
  37                                                                         /*}}}*/
  38
  39 using namespace std;
  40
  41 // Acquire::Item::Item - Constructor                                    /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* */
  44 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  45                        PartialSize(0), Mode(0), ID(0), Complete(false),
  46                        Local(false), QueueCounter(0)
  47 {
  48    Owner->Add(this);
  49    Status = StatIdle;
  50 }
  51                                                                         /*}}}*/
  52 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  53 // ---------------------------------------------------------------------
  54 /* */
  55 pkgAcquire::Item::~Item()
  56 {
  57    Owner->Remove(this);
  58 }
  59                                                                         /*}}}*/
  60 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  61 // ---------------------------------------------------------------------
  62 /* We return to an idle state if there are still other queues that could
  63    fetch this object */
  64 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  65 {
  66    Status = StatIdle;
  67    ErrorText = LookupTag(Message,"Message");
  68    UsedMirror =  LookupTag(Message,"UsedMirror");
  69    if (QueueCounter <= 1)
  70    {
  71       /* This indicates that the file is not available right now but might
  72          be sometime later. If we do a retry cycle then this should be
  73          retried [CDROMs] */
  74       if (Cnf->LocalOnly == true &&
  75           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  76       {
  77          Status = StatIdle;
  78          Dequeue();
  79          return;
  80       }
  81
  82       Status = StatError;
  83       Dequeue();
  84    }
  85
  86    // report mirror failure back to LP if we actually use a mirror
  87    string FailReason = LookupTag(Message, "FailReason");
  88    if(FailReason.size() != 0)
  89       ReportMirrorFailure(FailReason);
  90    else
  91       ReportMirrorFailure(ErrorText);
  92 }
  93                                                                         /*}}}*/
  94 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
  95 // ---------------------------------------------------------------------
  96 /* Stash status and the file size. Note that setting Complete means
  97    sub-phases of the acquire process such as decompresion are operating */
  98 void pkgAcquire::Item::Start(string /*Message*/,unsigned long Size)
  99 {
 100    Status = StatFetching;
 101    if (FileSize == 0 && Complete == false)
 102       FileSize = Size;
 103 }
 104                                                                         /*}}}*/
 105 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
 106 // ---------------------------------------------------------------------
 107 /* */
 108 void pkgAcquire::Item::Done(string Message,unsigned long Size,string Hash,
 109                             pkgAcquire::MethodConfig *Cnf)
 110 {
 111    // We just downloaded something..
 112    string FileName = LookupTag(Message,"Filename");
 113    UsedMirror =  LookupTag(Message,"UsedMirror");
 114    if (Complete == false && !Local && FileName == DestFile)
 115    {
 116       if (Owner->Log != 0)
 117          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 118    }
 119
 120    if (FileSize == 0)
 121       FileSize= Size;
 122    Status = StatDone;
 123    ErrorText = string();
 124    Owner->Dequeue(this);
 125 }
 126                                                                         /*}}}*/
 127 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 128 // ---------------------------------------------------------------------
 129 /* This helper function is used by alot of item methods as thier final
 130    step */
 131 void pkgAcquire::Item::Rename(string From,string To)
 132 {
 133    if (rename(From.c_str(),To.c_str()) != 0)
 134    {
 135       char S[300];
 136       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 137               From.c_str(),To.c_str());
 138       Status = StatError;
 139       ErrorText = S;
 140    }
 141 }
 142                                                                         /*}}}*/
 143
 144 void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
 145 {
 146    // we only act if a mirror was used at all
 147    if(UsedMirror.empty())
 148       return;
 149 #if 0
 150    std::cerr << "\nReportMirrorFailure: "
 151              << UsedMirror
 152              << " Uri: " << DescURI()
 153              << " FailCode: "
 154              << FailCode << std::endl;
 155 #endif
 156    const char *Args[40];
 157    unsigned int i = 0;
 158    string report = _config->Find("Methods::Mirror::ProblemReporting",
 159                                  "/usr/lib/apt/apt-report-mirror-failure");
 160    if(!FileExists(report))
 161       return;
 162    Args[i++] = report.c_str();
 163    Args[i++] = UsedMirror.c_str();
 164    Args[i++] = DescURI().c_str();
 165    Args[i++] = FailCode.c_str();
 166    Args[i++] = NULL;
 167    pid_t pid = ExecFork();
 168    if(pid < 0)
 169    {
 170       _error->Error("ReportMirrorFailure Fork failed");
 171       return;
 172    }
 173    else if(pid == 0)
 174    {
 175       execvp(Args[0], (char**)Args);
 176       std::cerr << "Could not exec " << Args[0] << std::endl;
 177       _exit(100);
 178    }
 179    if(!ExecWait(pid, "report-mirror-failure"))
 180    {
 181       _error->Warning("Couldn't report problem to '%s'",
 182                       _config->Find("Methods::Mirror::ProblemReporting").c_str());
 183    }
 184 }
 185
 186 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 187 // ---------------------------------------------------------------------
 188 /* Get the DiffIndex file first and see if there are patches availabe
 189  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 190  * patches. If anything goes wrong in that process, it will fall back to
 191  * the original packages file
 192  */
 193 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 194                                  string URI,string URIDesc,string ShortDesc,
 195                                  HashString ExpectedHash)
 196    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 197      Description(URIDesc)
 198 {
 199
 200    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 201
 202    Desc.Description = URIDesc + "/DiffIndex";
 203    Desc.Owner = this;
 204    Desc.ShortDesc = ShortDesc;
 205    Desc.URI = URI + ".diff/Index";
 206
 207    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 208    DestFile += URItoFileName(URI) + string(".DiffIndex");
 209
 210    if(Debug)
 211       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 212
 213    // look for the current package file
 214    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 215    CurrentPackagesFile += URItoFileName(RealURI);
 216
 217    // FIXME: this file:/ check is a hack to prevent fetching
 218    //        from local sources. this is really silly, and
 219    //        should be fixed cleanly as soon as possible
 220    if(!FileExists(CurrentPackagesFile) ||
 221       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 222    {
 223       // we don't have a pkg file or we don't want to queue
 224       if(Debug)
 225          std::clog << "No index file, local or canceld by user" << std::endl;
 226       Failed("", NULL);
 227       return;
 228    }
 229
 230    if(Debug)
 231       std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): "
 232                 << CurrentPackagesFile << std::endl;
 233
 234    QueueURI(Desc);
 235
 236 }
 237                                                                         /*}}}*/
 238 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 239 // ---------------------------------------------------------------------
 240 /* The only header we use is the last-modified header. */
 241 string pkgAcqDiffIndex::Custom600Headers()
 242 {
 243    string Final = _config->FindDir("Dir::State::lists");
 244    Final += URItoFileName(RealURI) + string(".IndexDiff");
 245
 246    if(Debug)
 247       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 248
 249    struct stat Buf;
 250    if (stat(Final.c_str(),&Buf) != 0)
 251       return "\nIndex-File: true";
 252
 253    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 254 }
 255                                                                         /*}}}*/
 256 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 257 {
 258    if(Debug)
 259       std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile
 260                 << std::endl;
 261
 262    pkgTagSection Tags;
 263    string ServerSha1;
 264    vector<DiffInfo> available_patches;
 265
 266    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 267    pkgTagFile TF(&Fd);
 268    if (_error->PendingError() == true)
 269       return false;
 270
 271    if(TF.Step(Tags) == true)
 272    {
 273       bool found = false;
 274       DiffInfo d;
 275       string size;
 276
 277       string const tmp = Tags.FindS("SHA1-Current");
 278       std::stringstream ss(tmp);
 279       ss >> ServerSha1 >> size;
 280       unsigned long const ServerSize = atol(size.c_str());
 281
 282       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 283       SHA1Summation SHA1;
 284       SHA1.AddFD(fd.Fd(), fd.Size());
 285       string const local_sha1 = SHA1.Result();
 286
 287       if(local_sha1 == ServerSha1)
 288       {
 289          // we have the same sha1 as the server
 290          if(Debug)
 291             std::clog << "Package file is up-to-date" << std::endl;
 292          // set found to true, this will queue a pkgAcqIndexDiffs with
 293          // a empty availabe_patches
 294          found = true;
 295       }
 296       else
 297       {
 298          if(Debug)
 299             std::clog << "SHA1-Current: " << ServerSha1 << std::endl;
 300
 301          // check the historie and see what patches we need
 302          string const history = Tags.FindS("SHA1-History");
 303          std::stringstream hist(history);
 304          while(hist >> d.sha1 >> size >> d.file)
 305          {
 306             // read until the first match is found
 307             // from that point on, we probably need all diffs
 308             if(d.sha1 == local_sha1)
 309                found=true;
 310             else if (found == false)
 311                continue;
 312
 313             if(Debug)
 314                std::clog << "Need to get diff: " << d.file << std::endl;
 315             available_patches.push_back(d);
 316          }
 317
 318          if (available_patches.empty() == false)
 319          {
 320             // patching with too many files is rather slow compared to a fast download
 321             unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 322             if (fileLimit != 0 && fileLimit < available_patches.size())
 323             {
 324                if (Debug)
 325                   std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 326                         << ") so fallback to complete download" << std::endl;
 327                return false;
 328             }
 329
 330             // see if the patches are too big
 331             found = false; // it was true and it will be true again at the end
 332             d = *available_patches.begin();
 333             string const firstPatch = d.file;
 334             unsigned long patchesSize = 0;
 335             std::stringstream patches(Tags.FindS("SHA1-Patches"));
 336             while(patches >> d.sha1 >> size >> d.file)
 337             {
 338                if (firstPatch == d.file)
 339                   found = true;
 340                else if (found == false)
 341                   continue;
 342
 343                patchesSize += atol(size.c_str());
 344             }
 345             unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 346             if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 347             {
 348                if (Debug)
 349                   std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 350                         << ") so fallback to complete download" << std::endl;
 351                return false;
 352             }
 353          }
 354       }
 355
 356       // we have something, queue the next diff
 357       if(found)
 358       {
 359          // queue the diffs
 360          string::size_type const last_space = Description.rfind(" ");
 361          if(last_space != string::npos)
 362             Description.erase(last_space, Description.size()-last_space);
 363          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 364                               ExpectedHash, ServerSha1, available_patches);
 365          Complete = false;
 366          Status = StatDone;
 367          Dequeue();
 368          return true;
 369       }
 370    }
 371
 372    // Nothing found, report and return false
 373    // Failing here is ok, if we return false later, the full
 374    // IndexFile is queued
 375    if(Debug)
 376       std::clog << "Can't find a patch in the index file" << std::endl;
 377    return false;
 378 }
 379                                                                         /*}}}*/
 380 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)      /*{{{*/
 381 {
 382    if(Debug)
 383       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << std::endl
 384                 << "Falling back to normal index file aquire" << std::endl;
 385
 386    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 387                    ExpectedHash);
 388
 389    Complete = false;
 390    Status = StatDone;
 391    Dequeue();
 392 }
 393                                                                         /*}}}*/
 394 void pkgAcqDiffIndex::Done(string Message,unsigned long Size,string Md5Hash,    /*{{{*/
 395                            pkgAcquire::MethodConfig *Cnf)
 396 {
 397    if(Debug)
 398       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 399
 400    Item::Done(Message,Size,Md5Hash,Cnf);
 401
 402    string FinalFile;
 403    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 404
 405    // sucess in downloading the index
 406    // rename the index
 407    FinalFile += string(".IndexDiff");
 408    if(Debug)
 409       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 410                 << std::endl;
 411    Rename(DestFile,FinalFile);
 412    chmod(FinalFile.c_str(),0644);
 413    DestFile = FinalFile;
 414
 415    if(!ParseDiffIndex(DestFile))
 416       return Failed("", NULL);
 417
 418    Complete = true;
 419    Status = StatDone;
 420    Dequeue();
 421    return;
 422 }
 423                                                                         /*}}}*/
 424 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 425 // ---------------------------------------------------------------------
 426 /* The package diff is added to the queue. one object is constructed
 427  * for each diff and the index
 428  */
 429 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 430                                    string URI,string URIDesc,string ShortDesc,
 431                                    HashString ExpectedHash,
 432                                    string ServerSha1,
 433                                    vector<DiffInfo> diffs)
 434    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 435      available_patches(diffs), ServerSha1(ServerSha1)
 436 {
 437
 438    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 439    DestFile += URItoFileName(URI);
 440
 441    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 442
 443    Description = URIDesc;
 444    Desc.Owner = this;
 445    Desc.ShortDesc = ShortDesc;
 446
 447    if(available_patches.size() == 0)
 448    {
 449       // we are done (yeah!)
 450       Finish(true);
 451    }
 452    else
 453    {
 454       // get the next diff
 455       State = StateFetchDiff;
 456       QueueNextDiff();
 457    }
 458 }
 459                                                                         /*}}}*/
 460 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)     /*{{{*/
 461 {
 462    if(Debug)
 463       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl
 464                 << "Falling back to normal index file aquire" << std::endl;
 465    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 466                    ExpectedHash);
 467    Finish();
 468 }
 469                                                                         /*}}}*/
 470 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 471 void pkgAcqIndexDiffs::Finish(bool allDone)
 472 {
 473    // we restore the original name, this is required, otherwise
 474    // the file will be cleaned
 475    if(allDone)
 476    {
 477       DestFile = _config->FindDir("Dir::State::lists");
 478       DestFile += URItoFileName(RealURI);
 479
 480       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 481       {
 482          Status = StatAuthError;
 483          ErrorText = _("MD5Sum mismatch");
 484          Rename(DestFile,DestFile + ".FAILED");
 485          Dequeue();
 486          return;
 487       }
 488
 489       // this is for the "real" finish
 490       Complete = true;
 491       Status = StatDone;
 492       Dequeue();
 493       if(Debug)
 494          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 495       return;
 496    }
 497
 498    if(Debug)
 499       std::clog << "Finishing: " << Desc.URI << std::endl;
 500    Complete = false;
 501    Status = StatDone;
 502    Dequeue();
 503    return;
 504 }
 505                                                                         /*}}}*/
 506 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 507 {
 508
 509    // calc sha1 of the just patched file
 510    string FinalFile = _config->FindDir("Dir::State::lists");
 511    FinalFile += URItoFileName(RealURI);
 512
 513    FileFd fd(FinalFile, FileFd::ReadOnly);
 514    SHA1Summation SHA1;
 515    SHA1.AddFD(fd.Fd(), fd.Size());
 516    string local_sha1 = string(SHA1.Result());
 517    if(Debug)
 518       std::clog << "QueueNextDiff: "
 519                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 520
 521    // final file reached before all patches are applied
 522    if(local_sha1 == ServerSha1)
 523    {
 524       Finish(true);
 525       return true;
 526    }
 527
 528    // remove all patches until the next matching patch is found
 529    // this requires the Index file to be ordered
 530    for(vector<DiffInfo>::iterator I=available_patches.begin();
 531        available_patches.size() > 0 &&
 532           I != available_patches.end() &&
 533           (*I).sha1 != local_sha1;
 534        I++)
 535    {
 536       available_patches.erase(I);
 537    }
 538
 539    // error checking and falling back if no patch was found
 540    if(available_patches.size() == 0)
 541    {
 542       Failed("", NULL);
 543       return false;
 544    }
 545
 546    // queue the right diff
 547    Desc.URI = string(RealURI) + ".diff/" + available_patches[0].file + ".gz";
 548    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 549    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 550    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 551
 552    if(Debug)
 553       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 554
 555    QueueURI(Desc);
 556
 557    return true;
 558 }
 559                                                                         /*}}}*/
 560 void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash,   /*{{{*/
 561                             pkgAcquire::MethodConfig *Cnf)
 562 {
 563    if(Debug)
 564       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 565
 566    Item::Done(Message,Size,Md5Hash,Cnf);
 567
 568    string FinalFile;
 569    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 570
 571    // sucess in downloading a diff, enter ApplyDiff state
 572    if(State == StateFetchDiff)
 573    {
 574
 575       if(Debug)
 576          std::clog << "Sending to gzip method: " << FinalFile << std::endl;
 577
 578       string FileName = LookupTag(Message,"Filename");
 579       State = StateUnzipDiff;
 580       Local = true;
 581       Desc.URI = "gzip:" + FileName;
 582       DestFile += ".decomp";
 583       QueueURI(Desc);
 584       Mode = "gzip";
 585       return;
 586    }
 587
 588    // sucess in downloading a diff, enter ApplyDiff state
 589    if(State == StateUnzipDiff)
 590    {
 591
 592       // rred excepts the patch as $FinalFile.ed
 593       Rename(DestFile,FinalFile+".ed");
 594
 595       if(Debug)
 596          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 597
 598       State = StateApplyDiff;
 599       Local = true;
 600       Desc.URI = "rred:" + FinalFile;
 601       QueueURI(Desc);
 602       Mode = "rred";
 603       return;
 604    }
 605
 606
 607    // success in download/apply a diff, queue next (if needed)
 608    if(State == StateApplyDiff)
 609    {
 610       // remove the just applied patch
 611       available_patches.erase(available_patches.begin());
 612
 613       // move into place
 614       if(Debug)
 615       {
 616          std::clog << "Moving patched file in place: " << std::endl
 617                    << DestFile << " -> " << FinalFile << std::endl;
 618       }
 619       Rename(DestFile,FinalFile);
 620       chmod(FinalFile.c_str(),0644);
 621
 622       // see if there is more to download
 623       if(available_patches.size() > 0) {
 624          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 625                               ExpectedHash, ServerSha1, available_patches);
 626          return Finish();
 627       } else
 628          return Finish(true);
 629    }
 630 }
 631                                                                         /*}}}*/
 632 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 633 // ---------------------------------------------------------------------
 634 /* The package file is added to the queue and a second class is
 635    instantiated to fetch the revision file */
 636 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 637                          string URI,string URIDesc,string ShortDesc,
 638                          HashString ExpectedHash, string comprExt)
 639    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 640 {
 641    Decompression = false;
 642    Erase = false;
 643
 644    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 645    DestFile += URItoFileName(URI);
 646
 647    if(comprExt.empty())
 648    {
 649       // autoselect the compression method
 650       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 651       if (types.empty() == true)
 652          comprExt = "plain";
 653       else
 654          comprExt = "." + types[0];
 655    }
 656    CompressionExtension = ((comprExt == "plain" || comprExt == ".") ? "" : comprExt);
 657
 658    Desc.URI = URI + CompressionExtension;
 659
 660    Desc.Description = URIDesc;
 661    Desc.Owner = this;
 662    Desc.ShortDesc = ShortDesc;
 663
 664    QueueURI(Desc);
 665 }
 666                                                                         /*}}}*/
 667 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 668 // ---------------------------------------------------------------------
 669 /* The only header we use is the last-modified header. */
 670 string pkgAcqIndex::Custom600Headers()
 671 {
 672    string Final = _config->FindDir("Dir::State::lists");
 673    Final += URItoFileName(RealURI);
 674
 675    struct stat Buf;
 676    if (stat(Final.c_str(),&Buf) != 0)
 677       return "\nIndex-File: true";
 678    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 679 }
 680                                                                         /*}}}*/
 681 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
 682 {
 683    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 684
 685    for (std::vector<std::string>::const_iterator t = types.begin();
 686         t != types.end(); t++)
 687    {
 688       // jump over all already tried compression types
 689       const unsigned int nameLen = Desc.URI.size() - (*t).size();
 690       if(Desc.URI.substr(nameLen) != *t)
 691          continue;
 692
 693       // we want to try it with the next extension (and make sure to
 694       // not skip over the end)
 695       t++;
 696       if (t == types.end())
 697          break;
 698
 699       // queue new download
 700       Desc.URI = Desc.URI.substr(0, nameLen) + *t;
 701       new pkgAcqIndex(Owner, RealURI, Desc.Description, Desc.ShortDesc,
 702       ExpectedHash, string(".").append(*t));
 703
 704       Status = StatDone;
 705       Complete = false;
 706       Dequeue();
 707       return;
 708    }
 709
 710    // on decompression failure, remove bad versions in partial/
 711    if(Decompression && Erase) {
 712       string s = _config->FindDir("Dir::State::lists") + "partial/";
 713       s += URItoFileName(RealURI);
 714       unlink(s.c_str());
 715    }
 716
 717    Item::Failed(Message,Cnf);
 718 }
 719                                                                         /*}}}*/
 720 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
 721 // ---------------------------------------------------------------------
 722 /* This goes through a number of states.. On the initial fetch the
 723    method could possibly return an alternate filename which points
 724    to the uncompressed version of the file. If this is so the file
 725    is copied into the partial directory. In all other cases the file
 726    is decompressed with a gzip uri. */
 727 void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash,
 728                        pkgAcquire::MethodConfig *Cfg)
 729 {
 730    Item::Done(Message,Size,Hash,Cfg);
 731
 732    if (Decompression == true)
 733    {
 734       if (_config->FindB("Debug::pkgAcquire::Auth", false))
 735       {
 736          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
 737          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
 738       }
 739
 740       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
 741       {
 742          Status = StatAuthError;
 743          ErrorText = _("Hash Sum mismatch");
 744          Rename(DestFile,DestFile + ".FAILED");
 745          ReportMirrorFailure("HashChecksumFailure");
 746          return;
 747       }
 748       // Done, move it into position
 749       string FinalFile = _config->FindDir("Dir::State::lists");
 750       FinalFile += URItoFileName(RealURI);
 751       Rename(DestFile,FinalFile);
 752       chmod(FinalFile.c_str(),0644);
 753
 754       /* We restore the original name to DestFile so that the clean operation
 755          will work OK */
 756       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 757       DestFile += URItoFileName(RealURI);
 758
 759       // Remove the compressed version.
 760       if (Erase == true)
 761          unlink(DestFile.c_str());
 762       return;
 763    }
 764
 765    Erase = false;
 766    Complete = true;
 767
 768    // Handle the unzipd case
 769    string FileName = LookupTag(Message,"Alt-Filename");
 770    if (FileName.empty() == false)
 771    {
 772       // The files timestamp matches
 773       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
 774          return;
 775       Decompression = true;
 776       Local = true;
 777       DestFile += ".decomp";
 778       Desc.URI = "copy:" + FileName;
 779       QueueURI(Desc);
 780       Mode = "copy";
 781       return;
 782    }
 783
 784    FileName = LookupTag(Message,"Filename");
 785    if (FileName.empty() == true)
 786    {
 787       Status = StatError;
 788       ErrorText = "Method gave a blank filename";
 789    }
 790
 791    // The files timestamp matches
 792    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 793       return;
 794
 795    if (FileName == DestFile)
 796       Erase = true;
 797    else
 798       Local = true;
 799
 800    string compExt = flExtension(flNotDir(URI(Desc.URI).Path));
 801    string decompProg;
 802
 803    // get the binary name for your used compression type
 804    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
 805    if(decompProg.empty() == false);
 806    // flExtensions returns the full name if no extension is found
 807    // this is why we have this complicated compare operation here
 808    // FIMXE: add a new flJustExtension() that return "" if no
 809    //        extension is found and use that above so that it can
 810    //        be tested against ""
 811    else if(compExt == flNotDir(URI(Desc.URI).Path))
 812       decompProg = "copy";
 813    else {
 814       _error->Error("Unsupported extension: %s", compExt.c_str());
 815       return;
 816    }
 817
 818    Decompression = true;
 819    DestFile += ".decomp";
 820    Desc.URI = decompProg + ":" + FileName;
 821    QueueURI(Desc);
 822    Mode = decompProg.c_str();
 823 }
 824                                                                         /*}}}*/
 825 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
 826 // ---------------------------------------------------------------------
 827 /* The Translation file is added to the queue */
 828 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
 829                             string URI,string URIDesc,string ShortDesc)
 830   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
 831 {
 832 }
 833                                                                         /*}}}*/
 834 // AcqIndexTrans::Custom600Headers - Insert custom request headers      /*{{{*/
 835 // ---------------------------------------------------------------------
 836 string pkgAcqIndexTrans::Custom600Headers()
 837 {
 838    return "\nFail-Ignore: true";
 839 }
 840                                                                         /*}}}*/
 841 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
 842 // ---------------------------------------------------------------------
 843 /* */
 844 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 845 {
 846    if (Cnf->LocalOnly == true ||
 847        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 848    {
 849       // Ignore this
 850       Status = StatDone;
 851       Complete = false;
 852       Dequeue();
 853       return;
 854    }
 855
 856    Item::Failed(Message,Cnf);
 857 }
 858                                                                         /*}}}*/
 859 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
 860                              string URI,string URIDesc,string ShortDesc,
 861                              string MetaIndexURI, string MetaIndexURIDesc,
 862                              string MetaIndexShortDesc,
 863                              const vector<IndexTarget*>* IndexTargets,
 864                              indexRecords* MetaIndexParser) :
 865    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
 866    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
 867    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
 868 {
 869    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 870    DestFile += URItoFileName(URI);
 871
 872    // remove any partial downloaded sig-file in partial/.
 873    // it may confuse proxies and is too small to warrant a
 874    // partial download anyway
 875    unlink(DestFile.c_str());
 876
 877    // Create the item
 878    Desc.Description = URIDesc;
 879    Desc.Owner = this;
 880    Desc.ShortDesc = ShortDesc;
 881    Desc.URI = URI;
 882
 883    string Final = _config->FindDir("Dir::State::lists");
 884    Final += URItoFileName(RealURI);
 885    struct stat Buf;
 886    if (stat(Final.c_str(),&Buf) == 0)
 887    {
 888       // File was already in place.  It needs to be re-downloaded/verified
 889       // because Release might have changed, we do give it a differnt
 890       // name than DestFile because otherwise the http method will
 891       // send If-Range requests and there are too many broken servers
 892       // out there that do not understand them
 893       LastGoodSig = DestFile+".reverify";
 894       Rename(Final,LastGoodSig);
 895    }
 896
 897    QueueURI(Desc);
 898 }
 899                                                                         /*}}}*/
 900 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
 901 // ---------------------------------------------------------------------
 902 /* The only header we use is the last-modified header. */
 903 string pkgAcqMetaSig::Custom600Headers()
 904 {
 905    struct stat Buf;
 906    if (stat(LastGoodSig.c_str(),&Buf) != 0)
 907       return "\nIndex-File: true";
 908
 909    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 910 }
 911
 912 void pkgAcqMetaSig::Done(string Message,unsigned long Size,string MD5,
 913                          pkgAcquire::MethodConfig *Cfg)
 914 {
 915    Item::Done(Message,Size,MD5,Cfg);
 916
 917    string FileName = LookupTag(Message,"Filename");
 918    if (FileName.empty() == true)
 919    {
 920       Status = StatError;
 921       ErrorText = "Method gave a blank filename";
 922       return;
 923    }
 924
 925    if (FileName != DestFile)
 926    {
 927       // We have to copy it into place
 928       Local = true;
 929       Desc.URI = "copy:" + FileName;
 930       QueueURI(Desc);
 931       return;
 932    }
 933
 934    Complete = true;
 935
 936    // put the last known good file back on i-m-s hit (it will
 937    // be re-verified again)
 938    // Else do nothing, we have the new file in DestFile then
 939    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 940       Rename(LastGoodSig, DestFile);
 941
 942    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
 943    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc,
 944                        MetaIndexShortDesc,  DestFile, IndexTargets,
 945                        MetaIndexParser);
 946
 947 }
 948                                                                         /*}}}*/
 949 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
 950 {
 951    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 952
 953    // if we get a network error we fail gracefully
 954    if(Status == StatTransientNetworkError)
 955    {
 956       Item::Failed(Message,Cnf);
 957       // move the sigfile back on transient network failures
 958       if(FileExists(LastGoodSig))
 959          Rename(LastGoodSig,Final);
 960
 961       // set the status back to , Item::Failed likes to reset it
 962       Status = pkgAcquire::Item::StatTransientNetworkError;
 963       return;
 964    }
 965
 966    // Delete any existing sigfile when the acquire failed
 967    unlink(Final.c_str());
 968
 969    // queue a pkgAcqMetaIndex with no sigfile
 970    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
 971                        "", IndexTargets, MetaIndexParser);
 972
 973    if (Cnf->LocalOnly == true ||
 974        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 975    {
 976       // Ignore this
 977       Status = StatDone;
 978       Complete = false;
 979       Dequeue();
 980       return;
 981    }
 982
 983    Item::Failed(Message,Cnf);
 984 }
 985                                                                         /*}}}*/
 986 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
 987                                  string URI,string URIDesc,string ShortDesc,
 988                                  string SigFile,
 989                                  const vector<struct IndexTarget*>* IndexTargets,
 990                                  indexRecords* MetaIndexParser) :
 991    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
 992    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
 993 {
 994    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 995    DestFile += URItoFileName(URI);
 996
 997    // Create the item
 998    Desc.Description = URIDesc;
 999    Desc.Owner = this;
1000    Desc.ShortDesc = ShortDesc;
1001    Desc.URI = URI;
1002
1003    QueueURI(Desc);
1004 }
1005                                                                         /*}}}*/
1006 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
1007 // ---------------------------------------------------------------------
1008 /* The only header we use is the last-modified header. */
1009 string pkgAcqMetaIndex::Custom600Headers()
1010 {
1011    string Final = _config->FindDir("Dir::State::lists");
1012    Final += URItoFileName(RealURI);
1013
1014    struct stat Buf;
1015    if (stat(Final.c_str(),&Buf) != 0)
1016       return "\nIndex-File: true";
1017
1018    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1019 }
1020                                                                         /*}}}*/
1021 void pkgAcqMetaIndex::Done(string Message,unsigned long Size,string Hash,       /*{{{*/
1022                            pkgAcquire::MethodConfig *Cfg)
1023 {
1024    Item::Done(Message,Size,Hash,Cfg);
1025
1026    // MetaIndexes are done in two passes: one to download the
1027    // metaindex with an appropriate method, and a second to verify it
1028    // with the gpgv method
1029
1030    if (AuthPass == true)
1031    {
1032       AuthDone(Message);
1033
1034       // all cool, move Release file into place
1035       Complete = true;
1036
1037       string FinalFile = _config->FindDir("Dir::State::lists");
1038       FinalFile += URItoFileName(RealURI);
1039       Rename(DestFile,FinalFile);
1040       chmod(FinalFile.c_str(),0644);
1041       DestFile = FinalFile;
1042    }
1043    else
1044    {
1045       RetrievalDone(Message);
1046       if (!Complete)
1047          // Still more retrieving to do
1048          return;
1049
1050       if (SigFile == "")
1051       {
1052          // There was no signature file, so we are finished.  Download
1053          // the indexes without verification.
1054          QueueIndexes(false);
1055       }
1056       else
1057       {
1058          // There was a signature file, so pass it to gpgv for
1059          // verification
1060
1061          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1062             std::cerr << "Metaindex acquired, queueing gpg verification ("
1063                       << SigFile << "," << DestFile << ")\n";
1064          AuthPass = true;
1065          Desc.URI = "gpgv:" + SigFile;
1066          QueueURI(Desc);
1067          Mode = "gpgv";
1068       }
1069    }
1070 }
1071                                                                         /*}}}*/
1072 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
1073 {
1074    // We have just finished downloading a Release file (it is not
1075    // verified yet)
1076
1077    string FileName = LookupTag(Message,"Filename");
1078    if (FileName.empty() == true)
1079    {
1080       Status = StatError;
1081       ErrorText = "Method gave a blank filename";
1082       return;
1083    }
1084
1085    if (FileName != DestFile)
1086    {
1087       Local = true;
1088       Desc.URI = "copy:" + FileName;
1089       QueueURI(Desc);
1090       return;
1091    }
1092
1093    // make sure to verify against the right file on I-M-S hit
1094    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1095    if(IMSHit)
1096    {
1097       string FinalFile = _config->FindDir("Dir::State::lists");
1098       FinalFile += URItoFileName(RealURI);
1099       DestFile = FinalFile;
1100    }
1101    Complete = true;
1102 }
1103                                                                         /*}}}*/
1104 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1105 {
1106    // At this point, the gpgv method has succeeded, so there is a
1107    // valid signature from a key in the trusted keyring.  We
1108    // perform additional verification of its contents, and use them
1109    // to verify the indexes we are about to download
1110
1111    if (!MetaIndexParser->Load(DestFile))
1112    {
1113       Status = StatAuthError;
1114       ErrorText = MetaIndexParser->ErrorText;
1115       return;
1116    }
1117
1118    if (!VerifyVendor(Message))
1119    {
1120       return;
1121    }
1122
1123    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1124       std::cerr << "Signature verification succeeded: "
1125                 << DestFile << std::endl;
1126
1127    // Download further indexes with verification
1128    QueueIndexes(true);
1129
1130    // Done, move signature file into position
1131    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1132       URItoFileName(RealURI) + ".gpg";
1133    Rename(SigFile,VerifiedSigFile);
1134    chmod(VerifiedSigFile.c_str(),0644);
1135 }
1136                                                                         /*}}}*/
1137 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1138 {
1139    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1140         Target != IndexTargets->end();
1141         Target++)
1142    {
1143       HashString ExpectedIndexHash;
1144       if (verify)
1145       {
1146          const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1147          if (!Record)
1148          {
1149             Status = StatAuthError;
1150             ErrorText = "Unable to find expected entry  "
1151                + (*Target)->MetaKey + " in Meta-index file (malformed Release file?)";
1152             return;
1153          }
1154          ExpectedIndexHash = Record->Hash;
1155          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1156          {
1157             std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1158             std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1159          }
1160          if (ExpectedIndexHash.empty())
1161          {
1162             Status = StatAuthError;
1163             ErrorText = "Unable to find hash sum for "
1164                + (*Target)->MetaKey + " in Meta-index file";
1165             return;
1166          }
1167       }
1168
1169       /* Queue Packages file (either diff or full packages files, depending
1170          on the users option) - we also check if the PDiff Index file is listed
1171          in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
1172          instead, but passing the required info to it is to much hassle */
1173       if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
1174           MetaIndexParser->Exists(string((*Target)->MetaKey).append(".diff/Index")) == true))
1175          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1176                              (*Target)->ShortDesc, ExpectedIndexHash);
1177       else
1178          new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description,
1179                             (*Target)->ShortDesc, ExpectedIndexHash);
1180    }
1181 }
1182                                                                         /*}}}*/
1183 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1184 {
1185 //    // Maybe this should be made available from above so we don't have
1186 //    // to read and parse it every time?
1187 //    pkgVendorList List;
1188 //    List.ReadMainList();
1189
1190 //    const Vendor* Vndr = NULL;
1191 //    for (std::vector<string>::const_iterator I = GPGVOutput.begin(); I != GPGVOutput.end(); I++)
1192 //    {
1193 //       string::size_type pos = (*I).find("VALIDSIG ");
1194 //       if (_config->FindB("Debug::Vendor", false))
1195 //          std::cerr << "Looking for VALIDSIG in \"" << (*I) << "\": pos " << pos
1196 //                    << std::endl;
1197 //       if (pos != std::string::npos)
1198 //       {
1199 //          string Fingerprint = (*I).substr(pos+sizeof("VALIDSIG"));
1200 //          if (_config->FindB("Debug::Vendor", false))
1201 //             std::cerr << "Looking for \"" << Fingerprint << "\" in vendor..." <<
1202 //                std::endl;
1203 //          Vndr = List.FindVendor(Fingerprint) != "";
1204 //          if (Vndr != NULL);
1205 //          break;
1206 //       }
1207 //    }
1208    string::size_type pos;
1209
1210    // check for missing sigs (that where not fatal because otherwise we had
1211    // bombed earlier)
1212    string missingkeys;
1213    string msg = _("There is no public key available for the "
1214                   "following key IDs:\n");
1215    pos = Message.find("NO_PUBKEY ");
1216    if (pos != std::string::npos)
1217    {
1218       string::size_type start = pos+strlen("NO_PUBKEY ");
1219       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1220       missingkeys += (Fingerprint);
1221    }
1222    if(!missingkeys.empty())
1223       _error->Warning("%s", string(msg+missingkeys).c_str());
1224
1225    string Transformed = MetaIndexParser->GetExpectedDist();
1226
1227    if (Transformed == "../project/experimental")
1228    {
1229       Transformed = "experimental";
1230    }
1231
1232    pos = Transformed.rfind('/');
1233    if (pos != string::npos)
1234    {
1235       Transformed = Transformed.substr(0, pos);
1236    }
1237
1238    if (Transformed == ".")
1239    {
1240       Transformed = "";
1241    }
1242
1243    if (_config->FindB("Acquire::Check-Valid-Until", true) == true &&
1244        MetaIndexParser->GetValidUntil() > 0) {
1245       time_t const invalid_since = time(NULL) - MetaIndexParser->GetValidUntil();
1246       if (invalid_since > 0)
1247          // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1248          // the time since then the file is invalid - formated in the same way as in
1249          // the download progress display (e.g. 7d 3h 42min 1s)
1250          return _error->Error(_("Release file expired, ignoring %s (invalid since %s)"),
1251                               RealURI.c_str(), TimeToStr(invalid_since).c_str());
1252    }
1253
1254    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1255    {
1256       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1257       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1258       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1259    }
1260
1261    if (MetaIndexParser->CheckDist(Transformed) == false)
1262    {
1263       // This might become fatal one day
1264 //       Status = StatAuthError;
1265 //       ErrorText = "Conflicting distribution; expected "
1266 //          + MetaIndexParser->GetExpectedDist() + " but got "
1267 //          + MetaIndexParser->GetDist();
1268 //       return false;
1269       if (!Transformed.empty())
1270       {
1271          _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1272                          Desc.Description.c_str(),
1273                          Transformed.c_str(),
1274                          MetaIndexParser->GetDist().c_str());
1275       }
1276    }
1277
1278    return true;
1279 }
1280                                                                         /*}}}*/
1281 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1282 // ---------------------------------------------------------------------
1283 /* */
1284 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1285 {
1286    if (AuthPass == true)
1287    {
1288       // gpgv method failed, if we have a good signature
1289       string LastGoodSigFile = _config->FindDir("Dir::State::lists") +
1290          "partial/" + URItoFileName(RealURI) + ".gpg.reverify";
1291       if(FileExists(LastGoodSigFile))
1292       {
1293          string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1294             URItoFileName(RealURI) + ".gpg";
1295          Rename(LastGoodSigFile,VerifiedSigFile);
1296          Status = StatTransientNetworkError;
1297          _error->Warning(_("A error occurred during the signature "
1298                            "verification. The repository is not updated "
1299                            "and the previous index files will be used."
1300                            "GPG error: %s: %s\n"),
1301                          Desc.Description.c_str(),
1302                          LookupTag(Message,"Message").c_str());
1303          RunScripts("APT::Update::Auth-Failure");
1304          return;
1305       } else {
1306          _error->Warning(_("GPG error: %s: %s"),
1307                          Desc.Description.c_str(),
1308                          LookupTag(Message,"Message").c_str());
1309       }
1310       // gpgv method failed
1311       ReportMirrorFailure("GPGFailure");
1312    }
1313
1314    // No Release file was present, or verification failed, so fall
1315    // back to queueing Packages files without verification
1316    QueueIndexes(false);
1317 }
1318                                                                         /*}}}*/
1319 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1320 // ---------------------------------------------------------------------
1321 /* This just sets up the initial fetch environment and queues the first
1322    possibilitiy */
1323 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1324                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1325                              string &StoreFilename) :
1326                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1327                StoreFilename(StoreFilename), Vf(Version.FileList()),
1328                Trusted(false)
1329 {
1330    Retries = _config->FindI("Acquire::Retries",0);
1331
1332    if (Version.Arch() == 0)
1333    {
1334       _error->Error(_("I wasn't able to locate a file for the %s package. "
1335                       "This might mean you need to manually fix this package. "
1336                       "(due to missing arch)"),
1337                     Version.ParentPkg().Name());
1338       return;
1339    }
1340
1341    /* We need to find a filename to determine the extension. We make the
1342       assumption here that all the available sources for this version share
1343       the same extension.. */
1344    // Skip not source sources, they do not have file fields.
1345    for (; Vf.end() == false; Vf++)
1346    {
1347       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1348          continue;
1349       break;
1350    }
1351
1352    // Does not really matter here.. we are going to fail out below
1353    if (Vf.end() != true)
1354    {
1355       // If this fails to get a file name we will bomb out below.
1356       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1357       if (_error->PendingError() == true)
1358          return;
1359
1360       // Generate the final file name as: package_version_arch.foo
1361       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1362                       QuoteString(Version.VerStr(),"_:") + '_' +
1363                       QuoteString(Version.Arch(),"_:.") +
1364                       "." + flExtension(Parse.FileName());
1365    }
1366
1367    // check if we have one trusted source for the package. if so, switch
1368    // to "TrustedOnly" mode
1369    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; i++)
1370    {
1371       pkgIndexFile *Index;
1372       if (Sources->FindIndex(i.File(),Index) == false)
1373          continue;
1374       if (_config->FindB("Debug::pkgAcquire::Auth", false))
1375       {
1376          std::cerr << "Checking index: " << Index->Describe()
1377                    << "(Trusted=" << Index->IsTrusted() << ")\n";
1378       }
1379       if (Index->IsTrusted()) {
1380          Trusted = true;
1381          break;
1382       }
1383    }
1384
1385    // "allow-unauthenticated" restores apts old fetching behaviour
1386    // that means that e.g. unauthenticated file:// uris are higher
1387    // priority than authenticated http:// uris
1388    if (_config->FindB("APT::Get::AllowUnauthenticated",false) == true)
1389       Trusted = false;
1390
1391    // Select a source
1392    if (QueueNext() == false && _error->PendingError() == false)
1393       _error->Error(_("I wasn't able to locate file for the %s package. "
1394                     "This might mean you need to manually fix this package."),
1395                     Version.ParentPkg().Name());
1396 }
1397                                                                         /*}}}*/
1398 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1399 // ---------------------------------------------------------------------
1400 /* This queues the next available file version for download. It checks if
1401    the archive is already available in the cache and stashs the MD5 for
1402    checking later. */
1403 bool pkgAcqArchive::QueueNext()
1404 {
1405    string const ForceHash = _config->Find("Acquire::ForceHash");
1406    for (; Vf.end() == false; Vf++)
1407    {
1408       // Ignore not source sources
1409       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1410          continue;
1411
1412       // Try to cross match against the source list
1413       pkgIndexFile *Index;
1414       if (Sources->FindIndex(Vf.File(),Index) == false)
1415             continue;
1416
1417       // only try to get a trusted package from another source if that source
1418       // is also trusted
1419       if(Trusted && !Index->IsTrusted())
1420          continue;
1421
1422       // Grab the text package record
1423       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1424       if (_error->PendingError() == true)
1425          return false;
1426
1427       string PkgFile = Parse.FileName();
1428       if (ForceHash.empty() == false)
1429       {
1430          if(stringcasecmp(ForceHash, "sha256") == 0)
1431             ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
1432          else if (stringcasecmp(ForceHash, "sha1") == 0)
1433             ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
1434          else
1435             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1436       }
1437       else
1438       {
1439          string Hash;
1440          if ((Hash = Parse.SHA256Hash()).empty() == false)
1441             ExpectedHash = HashString("SHA256", Hash);
1442          else if ((Hash = Parse.SHA1Hash()).empty() == false)
1443             ExpectedHash = HashString("SHA1", Hash);
1444          else
1445             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1446       }
1447       if (PkgFile.empty() == true)
1448          return _error->Error(_("The package index files are corrupted. No Filename: "
1449                               "field for package %s."),
1450                               Version.ParentPkg().Name());
1451
1452       Desc.URI = Index->ArchiveURI(PkgFile);
1453       Desc.Description = Index->ArchiveInfo(Version);
1454       Desc.Owner = this;
1455       Desc.ShortDesc = Version.ParentPkg().Name();
1456
1457       // See if we already have the file. (Legacy filenames)
1458       FileSize = Version->Size;
1459       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
1460       struct stat Buf;
1461       if (stat(FinalFile.c_str(),&Buf) == 0)
1462       {
1463          // Make sure the size matches
1464          if ((unsigned)Buf.st_size == Version->Size)
1465          {
1466             Complete = true;
1467             Local = true;
1468             Status = StatDone;
1469             StoreFilename = DestFile = FinalFile;
1470             return true;
1471          }
1472
1473          /* Hmm, we have a file and its size does not match, this means it is
1474             an old style mismatched arch */
1475          unlink(FinalFile.c_str());
1476       }
1477
1478       // Check it again using the new style output filenames
1479       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
1480       if (stat(FinalFile.c_str(),&Buf) == 0)
1481       {
1482          // Make sure the size matches
1483          if ((unsigned)Buf.st_size == Version->Size)
1484          {
1485             Complete = true;
1486             Local = true;
1487             Status = StatDone;
1488             StoreFilename = DestFile = FinalFile;
1489             return true;
1490          }
1491
1492          /* Hmm, we have a file and its size does not match, this shouldnt
1493             happen.. */
1494          unlink(FinalFile.c_str());
1495       }
1496
1497       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
1498
1499       // Check the destination file
1500       if (stat(DestFile.c_str(),&Buf) == 0)
1501       {
1502          // Hmm, the partial file is too big, erase it
1503          if ((unsigned)Buf.st_size > Version->Size)
1504             unlink(DestFile.c_str());
1505          else
1506             PartialSize = Buf.st_size;
1507       }
1508
1509       // Create the item
1510       Local = false;
1511       Desc.URI = Index->ArchiveURI(PkgFile);
1512       Desc.Description = Index->ArchiveInfo(Version);
1513       Desc.Owner = this;
1514       Desc.ShortDesc = Version.ParentPkg().Name();
1515       QueueURI(Desc);
1516
1517       Vf++;
1518       return true;
1519    }
1520    return false;
1521 }
1522                                                                         /*}}}*/
1523 // AcqArchive::Done - Finished fetching                                 /*{{{*/
1524 // ---------------------------------------------------------------------
1525 /* */
1526 void pkgAcqArchive::Done(string Message,unsigned long Size,string CalcHash,
1527                          pkgAcquire::MethodConfig *Cfg)
1528 {
1529    Item::Done(Message,Size,CalcHash,Cfg);
1530
1531    // Check the size
1532    if (Size != Version->Size)
1533    {
1534       Status = StatError;
1535       ErrorText = _("Size mismatch");
1536       return;
1537    }
1538
1539    // Check the hash
1540    if(ExpectedHash.toStr() != CalcHash)
1541    {
1542       Status = StatError;
1543       ErrorText = _("Hash Sum mismatch");
1544       if(FileExists(DestFile))
1545          Rename(DestFile,DestFile + ".FAILED");
1546       return;
1547    }
1548
1549    // Grab the output filename
1550    string FileName = LookupTag(Message,"Filename");
1551    if (FileName.empty() == true)
1552    {
1553       Status = StatError;
1554       ErrorText = "Method gave a blank filename";
1555       return;
1556    }
1557
1558    Complete = true;
1559
1560    // Reference filename
1561    if (FileName != DestFile)
1562    {
1563       StoreFilename = DestFile = FileName;
1564       Local = true;
1565       return;
1566    }
1567
1568    // Done, move it into position
1569    string FinalFile = _config->FindDir("Dir::Cache::Archives");
1570    FinalFile += flNotDir(StoreFilename);
1571    Rename(DestFile,FinalFile);
1572
1573    StoreFilename = DestFile = FinalFile;
1574    Complete = true;
1575 }
1576                                                                         /*}}}*/
1577 // AcqArchive::Failed - Failure handler                                 /*{{{*/
1578 // ---------------------------------------------------------------------
1579 /* Here we try other sources */
1580 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1581 {
1582    ErrorText = LookupTag(Message,"Message");
1583
1584    /* We don't really want to retry on failed media swaps, this prevents
1585       that. An interesting observation is that permanent failures are not
1586       recorded. */
1587    if (Cnf->Removable == true &&
1588        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1589    {
1590       // Vf = Version.FileList();
1591       while (Vf.end() == false) Vf++;
1592       StoreFilename = string();
1593       Item::Failed(Message,Cnf);
1594       return;
1595    }
1596
1597    if (QueueNext() == false)
1598    {
1599       // This is the retry counter
1600       if (Retries != 0 &&
1601           Cnf->LocalOnly == false &&
1602           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1603       {
1604          Retries--;
1605          Vf = Version.FileList();
1606          if (QueueNext() == true)
1607             return;
1608       }
1609
1610       StoreFilename = string();
1611       Item::Failed(Message,Cnf);
1612    }
1613 }
1614                                                                         /*}}}*/
1615 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
1616 // ---------------------------------------------------------------------
1617 bool pkgAcqArchive::IsTrusted()
1618 {
1619    return Trusted;
1620 }
1621                                                                         /*}}}*/
1622 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
1623 // ---------------------------------------------------------------------
1624 /* */
1625 void pkgAcqArchive::Finished()
1626 {
1627    if (Status == pkgAcquire::Item::StatDone &&
1628        Complete == true)
1629       return;
1630    StoreFilename = string();
1631 }
1632                                                                         /*}}}*/
1633 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
1634 // ---------------------------------------------------------------------
1635 /* The file is added to the queue */
1636 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
1637                        unsigned long Size,string Dsc,string ShortDesc,
1638                        const string &DestDir, const string &DestFilename,
1639                        bool IsIndexFile) :
1640                        Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile)
1641 {
1642    Retries = _config->FindI("Acquire::Retries",0);
1643
1644    if(!DestFilename.empty())
1645       DestFile = DestFilename;
1646    else if(!DestDir.empty())
1647       DestFile = DestDir + "/" + flNotDir(URI);
1648    else
1649       DestFile = flNotDir(URI);
1650
1651    // Create the item
1652    Desc.URI = URI;
1653    Desc.Description = Dsc;
1654    Desc.Owner = this;
1655
1656    // Set the short description to the archive component
1657    Desc.ShortDesc = ShortDesc;
1658
1659    // Get the transfer sizes
1660    FileSize = Size;
1661    struct stat Buf;
1662    if (stat(DestFile.c_str(),&Buf) == 0)
1663    {
1664       // Hmm, the partial file is too big, erase it
1665       if ((unsigned)Buf.st_size > Size)
1666          unlink(DestFile.c_str());
1667       else
1668          PartialSize = Buf.st_size;
1669    }
1670
1671    QueueURI(Desc);
1672 }
1673                                                                         /*}}}*/
1674 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
1675 // ---------------------------------------------------------------------
1676 /* */
1677 void pkgAcqFile::Done(string Message,unsigned long Size,string CalcHash,
1678                       pkgAcquire::MethodConfig *Cnf)
1679 {
1680    Item::Done(Message,Size,CalcHash,Cnf);
1681
1682    // Check the hash
1683    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
1684    {
1685       Status = StatError;
1686       ErrorText = "Hash Sum mismatch";
1687       Rename(DestFile,DestFile + ".FAILED");
1688       return;
1689    }
1690
1691    string FileName = LookupTag(Message,"Filename");
1692    if (FileName.empty() == true)
1693    {
1694       Status = StatError;
1695       ErrorText = "Method gave a blank filename";
1696       return;
1697    }
1698
1699    Complete = true;
1700
1701    // The files timestamp matches
1702    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1703       return;
1704
1705    // We have to copy it into place
1706    if (FileName != DestFile)
1707    {
1708       Local = true;
1709       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
1710           Cnf->Removable == true)
1711       {
1712          Desc.URI = "copy:" + FileName;
1713          QueueURI(Desc);
1714          return;
1715       }
1716
1717       // Erase the file if it is a symlink so we can overwrite it
1718       struct stat St;
1719       if (lstat(DestFile.c_str(),&St) == 0)
1720       {
1721          if (S_ISLNK(St.st_mode) != 0)
1722             unlink(DestFile.c_str());
1723       }
1724
1725       // Symlink the file
1726       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
1727       {
1728          ErrorText = "Link to " + DestFile + " failure ";
1729          Status = StatError;
1730          Complete = false;
1731       }
1732    }
1733 }
1734                                                                         /*}}}*/
1735 // AcqFile::Failed - Failure handler                                    /*{{{*/
1736 // ---------------------------------------------------------------------
1737 /* Here we try other sources */
1738 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1739 {
1740    ErrorText = LookupTag(Message,"Message");
1741
1742    // This is the retry counter
1743    if (Retries != 0 &&
1744        Cnf->LocalOnly == false &&
1745        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1746    {
1747       Retries--;
1748       QueueURI(Desc);
1749       return;
1750    }
1751
1752    Item::Failed(Message,Cnf);
1753 }
1754                                                                         /*}}}*/
1755 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
1756 // ---------------------------------------------------------------------
1757 /* The only header we use is the last-modified header. */
1758 string pkgAcqFile::Custom600Headers()
1759 {
1760    if (IsIndexFile)
1761       return "\nIndex-File: true";
1762    return "";
1763 }
1764                                                                         /*}}}*/