apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <apt-pkg/acquire-item.h>
  17 #include <apt-pkg/configuration.h>
  18 #include <apt-pkg/aptconfiguration.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/vendorlist.h>
  21 #include <apt-pkg/error.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <apt-pkg/fileutl.h>
  24 #include <apt-pkg/md5.h>
  25 #include <apt-pkg/sha1.h>
  26 #include <apt-pkg/tagfile.h>
  27
  28 #include <apti18n.h>
  29
  30 #include <sys/stat.h>
  31 #include <unistd.h>
  32 #include <errno.h>
  33 #include <string>
  34 #include <sstream>
  35 #include <stdio.h>
  36 #include <ctime>
  37                                                                         /*}}}*/
  38
  39 using namespace std;
  40
  41 // Acquire::Item::Item - Constructor                                    /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* */
  44 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  45                        PartialSize(0), Mode(0), ID(0), Complete(false),
  46                        Local(false), QueueCounter(0)
  47 {
  48    Owner->Add(this);
  49    Status = StatIdle;
  50 }
  51                                                                         /*}}}*/
  52 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  53 // ---------------------------------------------------------------------
  54 /* */
  55 pkgAcquire::Item::~Item()
  56 {
  57    Owner->Remove(this);
  58 }
  59                                                                         /*}}}*/
  60 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  61 // ---------------------------------------------------------------------
  62 /* We return to an idle state if there are still other queues that could
  63    fetch this object */
  64 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  65 {
  66    Status = StatIdle;
  67    ErrorText = LookupTag(Message,"Message");
  68    UsedMirror =  LookupTag(Message,"UsedMirror");
  69    if (QueueCounter <= 1)
  70    {
  71       /* This indicates that the file is not available right now but might
  72          be sometime later. If we do a retry cycle then this should be
  73          retried [CDROMs] */
  74       if (Cnf->LocalOnly == true &&
  75           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  76       {
  77          Status = StatIdle;
  78          Dequeue();
  79          return;
  80       }
  81
  82       Status = StatError;
  83       Dequeue();
  84    }
  85
  86    // report mirror failure back to LP if we actually use a mirror
  87    string FailReason = LookupTag(Message, "FailReason");
  88    if(FailReason.size() != 0)
  89       ReportMirrorFailure(FailReason);
  90    else
  91       ReportMirrorFailure(ErrorText);
  92 }
  93                                                                         /*}}}*/
  94 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
  95 // ---------------------------------------------------------------------
  96 /* Stash status and the file size. Note that setting Complete means
  97    sub-phases of the acquire process such as decompresion are operating */
  98 void pkgAcquire::Item::Start(string /*Message*/,unsigned long Size)
  99 {
 100    Status = StatFetching;
 101    if (FileSize == 0 && Complete == false)
 102       FileSize = Size;
 103 }
 104                                                                         /*}}}*/
 105 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
 106 // ---------------------------------------------------------------------
 107 /* */
 108 void pkgAcquire::Item::Done(string Message,unsigned long Size,string Hash,
 109                             pkgAcquire::MethodConfig *Cnf)
 110 {
 111    // We just downloaded something..
 112    string FileName = LookupTag(Message,"Filename");
 113    UsedMirror =  LookupTag(Message,"UsedMirror");
 114    if (Complete == false && !Local && FileName == DestFile)
 115    {
 116       if (Owner->Log != 0)
 117          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 118    }
 119
 120    if (FileSize == 0)
 121       FileSize= Size;
 122    Status = StatDone;
 123    ErrorText = string();
 124    Owner->Dequeue(this);
 125 }
 126                                                                         /*}}}*/
 127 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 128 // ---------------------------------------------------------------------
 129 /* This helper function is used by alot of item methods as thier final
 130    step */
 131 void pkgAcquire::Item::Rename(string From,string To)
 132 {
 133    if (rename(From.c_str(),To.c_str()) != 0)
 134    {
 135       char S[300];
 136       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 137               From.c_str(),To.c_str());
 138       Status = StatError;
 139       ErrorText = S;
 140    }
 141 }
 142                                                                         /*}}}*/
 143 // Acquire::Item::ReportMirrorFailure                                   /*{{{*/
 144 // ---------------------------------------------------------------------
 145 void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
 146 {
 147    // we only act if a mirror was used at all
 148    if(UsedMirror.empty())
 149       return;
 150 #if 0
 151    std::cerr << "\nReportMirrorFailure: "
 152              << UsedMirror
 153              << " Uri: " << DescURI()
 154              << " FailCode: "
 155              << FailCode << std::endl;
 156 #endif
 157    const char *Args[40];
 158    unsigned int i = 0;
 159    string report = _config->Find("Methods::Mirror::ProblemReporting",
 160                                  "/usr/lib/apt/apt-report-mirror-failure");
 161    if(!FileExists(report))
 162       return;
 163    Args[i++] = report.c_str();
 164    Args[i++] = UsedMirror.c_str();
 165    Args[i++] = DescURI().c_str();
 166    Args[i++] = FailCode.c_str();
 167    Args[i++] = NULL;
 168    pid_t pid = ExecFork();
 169    if(pid < 0)
 170    {
 171       _error->Error("ReportMirrorFailure Fork failed");
 172       return;
 173    }
 174    else if(pid == 0)
 175    {
 176       execvp(Args[0], (char**)Args);
 177       std::cerr << "Could not exec " << Args[0] << std::endl;
 178       _exit(100);
 179    }
 180    if(!ExecWait(pid, "report-mirror-failure"))
 181    {
 182       _error->Warning("Couldn't report problem to '%s'",
 183                       _config->Find("Methods::Mirror::ProblemReporting").c_str());
 184    }
 185 }
 186                                                                         /*}}}*/
 187 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 188 // ---------------------------------------------------------------------
 189 /* Get the DiffIndex file first and see if there are patches availabe
 190  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 191  * patches. If anything goes wrong in that process, it will fall back to
 192  * the original packages file
 193  */
 194 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 195                                  string URI,string URIDesc,string ShortDesc,
 196                                  HashString ExpectedHash)
 197    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 198      Description(URIDesc)
 199 {
 200
 201    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 202
 203    Desc.Description = URIDesc + "/DiffIndex";
 204    Desc.Owner = this;
 205    Desc.ShortDesc = ShortDesc;
 206    Desc.URI = URI + ".diff/Index";
 207
 208    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 209    DestFile += URItoFileName(URI) + string(".DiffIndex");
 210
 211    if(Debug)
 212       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 213
 214    // look for the current package file
 215    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 216    CurrentPackagesFile += URItoFileName(RealURI);
 217
 218    // FIXME: this file:/ check is a hack to prevent fetching
 219    //        from local sources. this is really silly, and
 220    //        should be fixed cleanly as soon as possible
 221    if(!FileExists(CurrentPackagesFile) ||
 222       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 223    {
 224       // we don't have a pkg file or we don't want to queue
 225       if(Debug)
 226          std::clog << "No index file, local or canceld by user" << std::endl;
 227       Failed("", NULL);
 228       return;
 229    }
 230
 231    if(Debug)
 232       std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): "
 233                 << CurrentPackagesFile << std::endl;
 234
 235    QueueURI(Desc);
 236
 237 }
 238                                                                         /*}}}*/
 239 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 240 // ---------------------------------------------------------------------
 241 /* The only header we use is the last-modified header. */
 242 string pkgAcqDiffIndex::Custom600Headers()
 243 {
 244    string Final = _config->FindDir("Dir::State::lists");
 245    Final += URItoFileName(RealURI) + string(".IndexDiff");
 246
 247    if(Debug)
 248       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 249
 250    struct stat Buf;
 251    if (stat(Final.c_str(),&Buf) != 0)
 252       return "\nIndex-File: true";
 253
 254    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 255 }
 256                                                                         /*}}}*/
 257 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 258 {
 259    if(Debug)
 260       std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile
 261                 << std::endl;
 262
 263    pkgTagSection Tags;
 264    string ServerSha1;
 265    vector<DiffInfo> available_patches;
 266
 267    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 268    pkgTagFile TF(&Fd);
 269    if (_error->PendingError() == true)
 270       return false;
 271
 272    if(TF.Step(Tags) == true)
 273    {
 274       bool found = false;
 275       DiffInfo d;
 276       string size;
 277
 278       string const tmp = Tags.FindS("SHA1-Current");
 279       std::stringstream ss(tmp);
 280       ss >> ServerSha1 >> size;
 281       unsigned long const ServerSize = atol(size.c_str());
 282
 283       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 284       SHA1Summation SHA1;
 285       SHA1.AddFD(fd.Fd(), fd.Size());
 286       string const local_sha1 = SHA1.Result();
 287
 288       if(local_sha1 == ServerSha1)
 289       {
 290          // we have the same sha1 as the server
 291          if(Debug)
 292             std::clog << "Package file is up-to-date" << std::endl;
 293          // set found to true, this will queue a pkgAcqIndexDiffs with
 294          // a empty availabe_patches
 295          found = true;
 296       }
 297       else
 298       {
 299          if(Debug)
 300             std::clog << "SHA1-Current: " << ServerSha1 << " and we start at "<< fd.Name() << " " << fd.Size() << " " << local_sha1 << std::endl;
 301
 302          // check the historie and see what patches we need
 303          string const history = Tags.FindS("SHA1-History");
 304          std::stringstream hist(history);
 305          while(hist >> d.sha1 >> size >> d.file)
 306          {
 307             // read until the first match is found
 308             // from that point on, we probably need all diffs
 309             if(d.sha1 == local_sha1)
 310                found=true;
 311             else if (found == false)
 312                continue;
 313
 314             if(Debug)
 315                std::clog << "Need to get diff: " << d.file << std::endl;
 316             available_patches.push_back(d);
 317          }
 318
 319          if (available_patches.empty() == false)
 320          {
 321             // patching with too many files is rather slow compared to a fast download
 322             unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 323             if (fileLimit != 0 && fileLimit < available_patches.size())
 324             {
 325                if (Debug)
 326                   std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 327                         << ") so fallback to complete download" << std::endl;
 328                return false;
 329             }
 330
 331             // see if the patches are too big
 332             found = false; // it was true and it will be true again at the end
 333             d = *available_patches.begin();
 334             string const firstPatch = d.file;
 335             unsigned long patchesSize = 0;
 336             std::stringstream patches(Tags.FindS("SHA1-Patches"));
 337             while(patches >> d.sha1 >> size >> d.file)
 338             {
 339                if (firstPatch == d.file)
 340                   found = true;
 341                else if (found == false)
 342                   continue;
 343
 344                patchesSize += atol(size.c_str());
 345             }
 346             unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 347             if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 348             {
 349                if (Debug)
 350                   std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 351                         << ") so fallback to complete download" << std::endl;
 352                return false;
 353             }
 354          }
 355       }
 356
 357       // we have something, queue the next diff
 358       if(found)
 359       {
 360          // queue the diffs
 361          string::size_type const last_space = Description.rfind(" ");
 362          if(last_space != string::npos)
 363             Description.erase(last_space, Description.size()-last_space);
 364          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 365                               ExpectedHash, ServerSha1, available_patches);
 366          Complete = false;
 367          Status = StatDone;
 368          Dequeue();
 369          return true;
 370       }
 371    }
 372
 373    // Nothing found, report and return false
 374    // Failing here is ok, if we return false later, the full
 375    // IndexFile is queued
 376    if(Debug)
 377       std::clog << "Can't find a patch in the index file" << std::endl;
 378    return false;
 379 }
 380                                                                         /*}}}*/
 381 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)      /*{{{*/
 382 {
 383    if(Debug)
 384       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << std::endl
 385                 << "Falling back to normal index file aquire" << std::endl;
 386
 387    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 388                    ExpectedHash);
 389
 390    Complete = false;
 391    Status = StatDone;
 392    Dequeue();
 393 }
 394                                                                         /*}}}*/
 395 void pkgAcqDiffIndex::Done(string Message,unsigned long Size,string Md5Hash,    /*{{{*/
 396                            pkgAcquire::MethodConfig *Cnf)
 397 {
 398    if(Debug)
 399       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 400
 401    Item::Done(Message,Size,Md5Hash,Cnf);
 402
 403    string FinalFile;
 404    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 405
 406    // sucess in downloading the index
 407    // rename the index
 408    FinalFile += string(".IndexDiff");
 409    if(Debug)
 410       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 411                 << std::endl;
 412    Rename(DestFile,FinalFile);
 413    chmod(FinalFile.c_str(),0644);
 414    DestFile = FinalFile;
 415
 416    if(!ParseDiffIndex(DestFile))
 417       return Failed("", NULL);
 418
 419    Complete = true;
 420    Status = StatDone;
 421    Dequeue();
 422    return;
 423 }
 424                                                                         /*}}}*/
 425 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 426 // ---------------------------------------------------------------------
 427 /* The package diff is added to the queue. one object is constructed
 428  * for each diff and the index
 429  */
 430 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 431                                    string URI,string URIDesc,string ShortDesc,
 432                                    HashString ExpectedHash,
 433                                    string ServerSha1,
 434                                    vector<DiffInfo> diffs)
 435    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 436      available_patches(diffs), ServerSha1(ServerSha1)
 437 {
 438
 439    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 440    DestFile += URItoFileName(URI);
 441
 442    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 443
 444    Description = URIDesc;
 445    Desc.Owner = this;
 446    Desc.ShortDesc = ShortDesc;
 447
 448    if(available_patches.size() == 0)
 449    {
 450       // we are done (yeah!)
 451       Finish(true);
 452    }
 453    else
 454    {
 455       // get the next diff
 456       State = StateFetchDiff;
 457       QueueNextDiff();
 458    }
 459 }
 460                                                                         /*}}}*/
 461 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)     /*{{{*/
 462 {
 463    if(Debug)
 464       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl
 465                 << "Falling back to normal index file aquire" << std::endl;
 466    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 467                    ExpectedHash);
 468    Finish();
 469 }
 470                                                                         /*}}}*/
 471 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 472 void pkgAcqIndexDiffs::Finish(bool allDone)
 473 {
 474    // we restore the original name, this is required, otherwise
 475    // the file will be cleaned
 476    if(allDone)
 477    {
 478       DestFile = _config->FindDir("Dir::State::lists");
 479       DestFile += URItoFileName(RealURI);
 480
 481       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 482       {
 483          Status = StatAuthError;
 484          ErrorText = _("MD5Sum mismatch");
 485          Rename(DestFile,DestFile + ".FAILED");
 486          Dequeue();
 487          return;
 488       }
 489
 490       // this is for the "real" finish
 491       Complete = true;
 492       Status = StatDone;
 493       Dequeue();
 494       if(Debug)
 495          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 496       return;
 497    }
 498
 499    if(Debug)
 500       std::clog << "Finishing: " << Desc.URI << std::endl;
 501    Complete = false;
 502    Status = StatDone;
 503    Dequeue();
 504    return;
 505 }
 506                                                                         /*}}}*/
 507 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 508 {
 509
 510    // calc sha1 of the just patched file
 511    string FinalFile = _config->FindDir("Dir::State::lists");
 512    FinalFile += URItoFileName(RealURI);
 513
 514    FileFd fd(FinalFile, FileFd::ReadOnly);
 515    SHA1Summation SHA1;
 516    SHA1.AddFD(fd.Fd(), fd.Size());
 517    string local_sha1 = string(SHA1.Result());
 518    if(Debug)
 519       std::clog << "QueueNextDiff: "
 520                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 521
 522    // final file reached before all patches are applied
 523    if(local_sha1 == ServerSha1)
 524    {
 525       Finish(true);
 526       return true;
 527    }
 528
 529    // remove all patches until the next matching patch is found
 530    // this requires the Index file to be ordered
 531    for(vector<DiffInfo>::iterator I=available_patches.begin();
 532        available_patches.size() > 0 &&
 533           I != available_patches.end() &&
 534           (*I).sha1 != local_sha1;
 535        I++)
 536    {
 537       available_patches.erase(I);
 538    }
 539
 540    // error checking and falling back if no patch was found
 541    if(available_patches.size() == 0)
 542    {
 543       Failed("", NULL);
 544       return false;
 545    }
 546
 547    // queue the right diff
 548    Desc.URI = string(RealURI) + ".diff/" + available_patches[0].file + ".gz";
 549    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 550    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 551    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 552
 553    if(Debug)
 554       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 555
 556    QueueURI(Desc);
 557
 558    return true;
 559 }
 560                                                                         /*}}}*/
 561 void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash,   /*{{{*/
 562                             pkgAcquire::MethodConfig *Cnf)
 563 {
 564    if(Debug)
 565       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 566
 567    Item::Done(Message,Size,Md5Hash,Cnf);
 568
 569    string FinalFile;
 570    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 571
 572    // sucess in downloading a diff, enter ApplyDiff state
 573    if(State == StateFetchDiff)
 574    {
 575
 576       if(Debug)
 577          std::clog << "Sending to gzip method: " << FinalFile << std::endl;
 578
 579       string FileName = LookupTag(Message,"Filename");
 580       State = StateUnzipDiff;
 581       Local = true;
 582       Desc.URI = "gzip:" + FileName;
 583       DestFile += ".decomp";
 584       QueueURI(Desc);
 585       Mode = "gzip";
 586       return;
 587    }
 588
 589    // sucess in downloading a diff, enter ApplyDiff state
 590    if(State == StateUnzipDiff)
 591    {
 592
 593       // rred excepts the patch as $FinalFile.ed
 594       Rename(DestFile,FinalFile+".ed");
 595
 596       if(Debug)
 597          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 598
 599       State = StateApplyDiff;
 600       Local = true;
 601       Desc.URI = "rred:" + FinalFile;
 602       QueueURI(Desc);
 603       Mode = "rred";
 604       return;
 605    }
 606
 607
 608    // success in download/apply a diff, queue next (if needed)
 609    if(State == StateApplyDiff)
 610    {
 611       // remove the just applied patch
 612       available_patches.erase(available_patches.begin());
 613
 614       // move into place
 615       if(Debug)
 616       {
 617          std::clog << "Moving patched file in place: " << std::endl
 618                    << DestFile << " -> " << FinalFile << std::endl;
 619       }
 620       Rename(DestFile,FinalFile);
 621       chmod(FinalFile.c_str(),0644);
 622
 623       // see if there is more to download
 624       if(available_patches.size() > 0) {
 625          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 626                               ExpectedHash, ServerSha1, available_patches);
 627          return Finish();
 628       } else
 629          return Finish(true);
 630    }
 631 }
 632                                                                         /*}}}*/
 633 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 634 // ---------------------------------------------------------------------
 635 /* The package file is added to the queue and a second class is
 636    instantiated to fetch the revision file */
 637 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 638                          string URI,string URIDesc,string ShortDesc,
 639                          HashString ExpectedHash, string comprExt)
 640    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 641 {
 642    Decompression = false;
 643    Erase = false;
 644
 645    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 646    DestFile += URItoFileName(URI);
 647
 648    if(comprExt.empty())
 649    {
 650       // autoselect the compression method
 651       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 652       if (types.empty() == true)
 653          comprExt = "plain";
 654       else
 655          comprExt = "." + types[0];
 656    }
 657    CompressionExtension = ((comprExt == "plain" || comprExt == ".") ? "" : comprExt);
 658
 659    Desc.URI = URI + CompressionExtension;
 660
 661    Desc.Description = URIDesc;
 662    Desc.Owner = this;
 663    Desc.ShortDesc = ShortDesc;
 664
 665    QueueURI(Desc);
 666 }
 667                                                                         /*}}}*/
 668 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 669 // ---------------------------------------------------------------------
 670 /* The only header we use is the last-modified header. */
 671 string pkgAcqIndex::Custom600Headers()
 672 {
 673    string Final = _config->FindDir("Dir::State::lists");
 674    Final += URItoFileName(RealURI);
 675    if (_config->FindB("Acquire::GzipIndexes",false))
 676       Final += ".gz";
 677
 678    struct stat Buf;
 679    if (stat(Final.c_str(),&Buf) != 0)
 680       return "\nIndex-File: true";
 681    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 682 }
 683                                                                         /*}}}*/
 684 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
 685 {
 686    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 687
 688    for (std::vector<std::string>::const_iterator t = types.begin();
 689         t != types.end(); t++)
 690    {
 691       // jump over all already tried compression types
 692       const unsigned int nameLen = Desc.URI.size() - (*t).size();
 693       if(Desc.URI.substr(nameLen) != *t)
 694          continue;
 695
 696       // we want to try it with the next extension (and make sure to
 697       // not skip over the end)
 698       t++;
 699       if (t == types.end())
 700          break;
 701
 702       // queue new download
 703       Desc.URI = Desc.URI.substr(0, nameLen) + *t;
 704       new pkgAcqIndex(Owner, RealURI, Desc.Description, Desc.ShortDesc,
 705       ExpectedHash, string(".").append(*t));
 706
 707       Status = StatDone;
 708       Complete = false;
 709       Dequeue();
 710       return;
 711    }
 712
 713    // on decompression failure, remove bad versions in partial/
 714    if(Decompression && Erase) {
 715       string s = _config->FindDir("Dir::State::lists") + "partial/";
 716       s += URItoFileName(RealURI);
 717       unlink(s.c_str());
 718    }
 719
 720    Item::Failed(Message,Cnf);
 721 }
 722                                                                         /*}}}*/
 723 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
 724 // ---------------------------------------------------------------------
 725 /* This goes through a number of states.. On the initial fetch the
 726    method could possibly return an alternate filename which points
 727    to the uncompressed version of the file. If this is so the file
 728    is copied into the partial directory. In all other cases the file
 729    is decompressed with a gzip uri. */
 730 void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash,
 731                        pkgAcquire::MethodConfig *Cfg)
 732 {
 733    Item::Done(Message,Size,Hash,Cfg);
 734
 735    if (Decompression == true)
 736    {
 737       if (_config->FindB("Debug::pkgAcquire::Auth", false))
 738       {
 739          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
 740          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
 741       }
 742
 743       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
 744       {
 745          Status = StatAuthError;
 746          ErrorText = _("Hash Sum mismatch");
 747          Rename(DestFile,DestFile + ".FAILED");
 748          ReportMirrorFailure("HashChecksumFailure");
 749          return;
 750       }
 751       // Done, move it into position
 752       string FinalFile = _config->FindDir("Dir::State::lists");
 753       FinalFile += URItoFileName(RealURI);
 754       Rename(DestFile,FinalFile);
 755       chmod(FinalFile.c_str(),0644);
 756
 757       /* We restore the original name to DestFile so that the clean operation
 758          will work OK */
 759       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 760       DestFile += URItoFileName(RealURI);
 761
 762       // Remove the compressed version.
 763       if (Erase == true)
 764          unlink(DestFile.c_str());
 765       return;
 766    }
 767
 768    Erase = false;
 769    Complete = true;
 770
 771    // Handle the unzipd case
 772    string FileName = LookupTag(Message,"Alt-Filename");
 773    if (FileName.empty() == false)
 774    {
 775       // The files timestamp matches
 776       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
 777          return;
 778       Decompression = true;
 779       Local = true;
 780       DestFile += ".decomp";
 781       Desc.URI = "copy:" + FileName;
 782       QueueURI(Desc);
 783       Mode = "copy";
 784       return;
 785    }
 786
 787    FileName = LookupTag(Message,"Filename");
 788    if (FileName.empty() == true)
 789    {
 790       Status = StatError;
 791       ErrorText = "Method gave a blank filename";
 792    }
 793
 794    string compExt = flExtension(flNotDir(URI(Desc.URI).Path));
 795
 796    // The files timestamp matches
 797    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) {
 798        if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
 799           // Update DestFile for .gz suffix so that the clean operation keeps it
 800           DestFile += ".gz";
 801       return;
 802     }
 803
 804    if (FileName == DestFile)
 805       Erase = true;
 806    else
 807       Local = true;
 808
 809    string decompProg;
 810
 811    // If we enable compressed indexes and already have gzip, keep it
 812    if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) {
 813       string FinalFile = _config->FindDir("Dir::State::lists");
 814       FinalFile += URItoFileName(RealURI) + ".gz";
 815       Rename(DestFile,FinalFile);
 816       chmod(FinalFile.c_str(),0644);
 817
 818       // Update DestFile for .gz suffix so that the clean operation keeps it
 819       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 820       DestFile += URItoFileName(RealURI) + ".gz";
 821       return;
 822     }
 823
 824    // get the binary name for your used compression type
 825    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
 826    if(decompProg.empty() == false);
 827    // flExtensions returns the full name if no extension is found
 828    // this is why we have this complicated compare operation here
 829    // FIMXE: add a new flJustExtension() that return "" if no
 830    //        extension is found and use that above so that it can
 831    //        be tested against ""
 832    else if(compExt == flNotDir(URI(Desc.URI).Path))
 833       decompProg = "copy";
 834    else {
 835       _error->Error("Unsupported extension: %s", compExt.c_str());
 836       return;
 837    }
 838
 839    Decompression = true;
 840    DestFile += ".decomp";
 841    Desc.URI = decompProg + ":" + FileName;
 842    QueueURI(Desc);
 843    Mode = decompProg.c_str();
 844 }
 845                                                                         /*}}}*/
 846 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
 847 // ---------------------------------------------------------------------
 848 /* The Translation file is added to the queue */
 849 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
 850                             string URI,string URIDesc,string ShortDesc)
 851   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
 852 {
 853 }
 854                                                                         /*}}}*/
 855 // AcqIndexTrans::Custom600Headers - Insert custom request headers      /*{{{*/
 856 // ---------------------------------------------------------------------
 857 string pkgAcqIndexTrans::Custom600Headers()
 858 {
 859    string Final = _config->FindDir("Dir::State::lists");
 860    Final += URItoFileName(RealURI);
 861
 862    struct stat Buf;
 863    if (stat(Final.c_str(),&Buf) != 0)
 864       return "\nFail-Ignore: true";
 865    return "\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 866 }
 867                                                                         /*}}}*/
 868 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
 869 // ---------------------------------------------------------------------
 870 /* */
 871 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 872 {
 873    if (Cnf->LocalOnly == true ||
 874        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 875    {
 876       // Ignore this
 877       Status = StatDone;
 878       Complete = false;
 879       Dequeue();
 880       return;
 881    }
 882
 883    Item::Failed(Message,Cnf);
 884 }
 885                                                                         /*}}}*/
 886 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
 887                              string URI,string URIDesc,string ShortDesc,
 888                              string MetaIndexURI, string MetaIndexURIDesc,
 889                              string MetaIndexShortDesc,
 890                              const vector<IndexTarget*>* IndexTargets,
 891                              indexRecords* MetaIndexParser) :
 892    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
 893    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
 894    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
 895 {
 896    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 897    DestFile += URItoFileName(URI);
 898
 899    // remove any partial downloaded sig-file in partial/.
 900    // it may confuse proxies and is too small to warrant a
 901    // partial download anyway
 902    unlink(DestFile.c_str());
 903
 904    // Create the item
 905    Desc.Description = URIDesc;
 906    Desc.Owner = this;
 907    Desc.ShortDesc = ShortDesc;
 908    Desc.URI = URI;
 909
 910    string Final = _config->FindDir("Dir::State::lists");
 911    Final += URItoFileName(RealURI);
 912    struct stat Buf;
 913    if (stat(Final.c_str(),&Buf) == 0)
 914    {
 915       // File was already in place.  It needs to be re-downloaded/verified
 916       // because Release might have changed, we do give it a differnt
 917       // name than DestFile because otherwise the http method will
 918       // send If-Range requests and there are too many broken servers
 919       // out there that do not understand them
 920       LastGoodSig = DestFile+".reverify";
 921       Rename(Final,LastGoodSig);
 922    }
 923
 924    QueueURI(Desc);
 925 }
 926                                                                         /*}}}*/
 927 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
 928 // ---------------------------------------------------------------------
 929 /* The only header we use is the last-modified header. */
 930 string pkgAcqMetaSig::Custom600Headers()
 931 {
 932    struct stat Buf;
 933    if (stat(LastGoodSig.c_str(),&Buf) != 0)
 934       return "\nIndex-File: true";
 935
 936    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 937 }
 938
 939 void pkgAcqMetaSig::Done(string Message,unsigned long Size,string MD5,
 940                          pkgAcquire::MethodConfig *Cfg)
 941 {
 942    Item::Done(Message,Size,MD5,Cfg);
 943
 944    string FileName = LookupTag(Message,"Filename");
 945    if (FileName.empty() == true)
 946    {
 947       Status = StatError;
 948       ErrorText = "Method gave a blank filename";
 949       return;
 950    }
 951
 952    if (FileName != DestFile)
 953    {
 954       // We have to copy it into place
 955       Local = true;
 956       Desc.URI = "copy:" + FileName;
 957       QueueURI(Desc);
 958       return;
 959    }
 960
 961    Complete = true;
 962
 963    // put the last known good file back on i-m-s hit (it will
 964    // be re-verified again)
 965    // Else do nothing, we have the new file in DestFile then
 966    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 967       Rename(LastGoodSig, DestFile);
 968
 969    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
 970    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc,
 971                        MetaIndexShortDesc,  DestFile, IndexTargets,
 972                        MetaIndexParser);
 973
 974 }
 975                                                                         /*}}}*/
 976 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
 977 {
 978    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 979
 980    // if we get a network error we fail gracefully
 981    if(Status == StatTransientNetworkError)
 982    {
 983       Item::Failed(Message,Cnf);
 984       // move the sigfile back on transient network failures
 985       if(FileExists(LastGoodSig))
 986          Rename(LastGoodSig,Final);
 987
 988       // set the status back to , Item::Failed likes to reset it
 989       Status = pkgAcquire::Item::StatTransientNetworkError;
 990       return;
 991    }
 992
 993    // Delete any existing sigfile when the acquire failed
 994    unlink(Final.c_str());
 995
 996    // queue a pkgAcqMetaIndex with no sigfile
 997    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
 998                        "", IndexTargets, MetaIndexParser);
 999
1000    if (Cnf->LocalOnly == true ||
1001        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1002    {
1003       // Ignore this
1004       Status = StatDone;
1005       Complete = false;
1006       Dequeue();
1007       return;
1008    }
1009
1010    Item::Failed(Message,Cnf);
1011 }
1012                                                                         /*}}}*/
1013 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
1014                                  string URI,string URIDesc,string ShortDesc,
1015                                  string SigFile,
1016                                  const vector<struct IndexTarget*>* IndexTargets,
1017                                  indexRecords* MetaIndexParser) :
1018    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
1019    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
1020 {
1021    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1022    DestFile += URItoFileName(URI);
1023
1024    // Create the item
1025    Desc.Description = URIDesc;
1026    Desc.Owner = this;
1027    Desc.ShortDesc = ShortDesc;
1028    Desc.URI = URI;
1029
1030    QueueURI(Desc);
1031 }
1032                                                                         /*}}}*/
1033 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
1034 // ---------------------------------------------------------------------
1035 /* The only header we use is the last-modified header. */
1036 string pkgAcqMetaIndex::Custom600Headers()
1037 {
1038    string Final = _config->FindDir("Dir::State::lists");
1039    Final += URItoFileName(RealURI);
1040
1041    struct stat Buf;
1042    if (stat(Final.c_str(),&Buf) != 0)
1043       return "\nIndex-File: true";
1044
1045    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1046 }
1047                                                                         /*}}}*/
1048 void pkgAcqMetaIndex::Done(string Message,unsigned long Size,string Hash,       /*{{{*/
1049                            pkgAcquire::MethodConfig *Cfg)
1050 {
1051    Item::Done(Message,Size,Hash,Cfg);
1052
1053    // MetaIndexes are done in two passes: one to download the
1054    // metaindex with an appropriate method, and a second to verify it
1055    // with the gpgv method
1056
1057    if (AuthPass == true)
1058    {
1059       AuthDone(Message);
1060
1061       // all cool, move Release file into place
1062       Complete = true;
1063    }
1064    else
1065    {
1066       RetrievalDone(Message);
1067       if (!Complete)
1068          // Still more retrieving to do
1069          return;
1070
1071       if (SigFile == "")
1072       {
1073          // There was no signature file, so we are finished.  Download
1074          // the indexes without verification.
1075          QueueIndexes(false);
1076       }
1077       else
1078       {
1079          // There was a signature file, so pass it to gpgv for
1080          // verification
1081
1082          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1083             std::cerr << "Metaindex acquired, queueing gpg verification ("
1084                       << SigFile << "," << DestFile << ")\n";
1085          AuthPass = true;
1086          Desc.URI = "gpgv:" + SigFile;
1087          QueueURI(Desc);
1088          Mode = "gpgv";
1089          return;
1090       }
1091    }
1092
1093    if (Complete == true)
1094    {
1095       string FinalFile = _config->FindDir("Dir::State::lists");
1096       FinalFile += URItoFileName(RealURI);
1097       Rename(DestFile,FinalFile);
1098       chmod(FinalFile.c_str(),0644);
1099       DestFile = FinalFile;
1100    }
1101 }
1102                                                                         /*}}}*/
1103 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
1104 {
1105    // We have just finished downloading a Release file (it is not
1106    // verified yet)
1107
1108    string FileName = LookupTag(Message,"Filename");
1109    if (FileName.empty() == true)
1110    {
1111       Status = StatError;
1112       ErrorText = "Method gave a blank filename";
1113       return;
1114    }
1115
1116    if (FileName != DestFile)
1117    {
1118       Local = true;
1119       Desc.URI = "copy:" + FileName;
1120       QueueURI(Desc);
1121       return;
1122    }
1123
1124    // make sure to verify against the right file on I-M-S hit
1125    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1126    if(IMSHit)
1127    {
1128       string FinalFile = _config->FindDir("Dir::State::lists");
1129       FinalFile += URItoFileName(RealURI);
1130       DestFile = FinalFile;
1131    }
1132    Complete = true;
1133 }
1134                                                                         /*}}}*/
1135 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1136 {
1137    // At this point, the gpgv method has succeeded, so there is a
1138    // valid signature from a key in the trusted keyring.  We
1139    // perform additional verification of its contents, and use them
1140    // to verify the indexes we are about to download
1141
1142    if (!MetaIndexParser->Load(DestFile))
1143    {
1144       Status = StatAuthError;
1145       ErrorText = MetaIndexParser->ErrorText;
1146       return;
1147    }
1148
1149    if (!VerifyVendor(Message))
1150    {
1151       return;
1152    }
1153
1154    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1155       std::cerr << "Signature verification succeeded: "
1156                 << DestFile << std::endl;
1157
1158    // Download further indexes with verification
1159    QueueIndexes(true);
1160
1161    // Done, move signature file into position
1162    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1163       URItoFileName(RealURI) + ".gpg";
1164    Rename(SigFile,VerifiedSigFile);
1165    chmod(VerifiedSigFile.c_str(),0644);
1166 }
1167                                                                         /*}}}*/
1168 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1169 {
1170    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1171         Target != IndexTargets->end();
1172         Target++)
1173    {
1174       HashString ExpectedIndexHash;
1175       if (verify)
1176       {
1177          const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1178          if (!Record)
1179          {
1180             Status = StatAuthError;
1181             ErrorText = "Unable to find expected entry  "
1182                + (*Target)->MetaKey + " in Meta-index file (malformed Release file?)";
1183             return;
1184          }
1185          ExpectedIndexHash = Record->Hash;
1186          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1187          {
1188             std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1189             std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1190          }
1191          if (ExpectedIndexHash.empty())
1192          {
1193             Status = StatAuthError;
1194             ErrorText = "Unable to find hash sum for "
1195                + (*Target)->MetaKey + " in Meta-index file";
1196             return;
1197          }
1198       }
1199
1200       /* Queue Packages file (either diff or full packages files, depending
1201          on the users option) - we also check if the PDiff Index file is listed
1202          in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
1203          instead, but passing the required info to it is to much hassle */
1204       if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
1205           MetaIndexParser->Exists(string((*Target)->MetaKey).append(".diff/Index")) == true))
1206          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1207                              (*Target)->ShortDesc, ExpectedIndexHash);
1208       else
1209          new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description,
1210                             (*Target)->ShortDesc, ExpectedIndexHash);
1211    }
1212 }
1213                                                                         /*}}}*/
1214 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1215 {
1216 //    // Maybe this should be made available from above so we don't have
1217 //    // to read and parse it every time?
1218 //    pkgVendorList List;
1219 //    List.ReadMainList();
1220
1221 //    const Vendor* Vndr = NULL;
1222 //    for (std::vector<string>::const_iterator I = GPGVOutput.begin(); I != GPGVOutput.end(); I++)
1223 //    {
1224 //       string::size_type pos = (*I).find("VALIDSIG ");
1225 //       if (_config->FindB("Debug::Vendor", false))
1226 //          std::cerr << "Looking for VALIDSIG in \"" << (*I) << "\": pos " << pos
1227 //                    << std::endl;
1228 //       if (pos != std::string::npos)
1229 //       {
1230 //          string Fingerprint = (*I).substr(pos+sizeof("VALIDSIG"));
1231 //          if (_config->FindB("Debug::Vendor", false))
1232 //             std::cerr << "Looking for \"" << Fingerprint << "\" in vendor..." <<
1233 //                std::endl;
1234 //          Vndr = List.FindVendor(Fingerprint) != "";
1235 //          if (Vndr != NULL);
1236 //          break;
1237 //       }
1238 //    }
1239    string::size_type pos;
1240
1241    // check for missing sigs (that where not fatal because otherwise we had
1242    // bombed earlier)
1243    string missingkeys;
1244    string msg = _("There is no public key available for the "
1245                   "following key IDs:\n");
1246    pos = Message.find("NO_PUBKEY ");
1247    if (pos != std::string::npos)
1248    {
1249       string::size_type start = pos+strlen("NO_PUBKEY ");
1250       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1251       missingkeys += (Fingerprint);
1252    }
1253    if(!missingkeys.empty())
1254       _error->Warning("%s", string(msg+missingkeys).c_str());
1255
1256    string Transformed = MetaIndexParser->GetExpectedDist();
1257
1258    if (Transformed == "../project/experimental")
1259    {
1260       Transformed = "experimental";
1261    }
1262
1263    pos = Transformed.rfind('/');
1264    if (pos != string::npos)
1265    {
1266       Transformed = Transformed.substr(0, pos);
1267    }
1268
1269    if (Transformed == ".")
1270    {
1271       Transformed = "";
1272    }
1273
1274    if (_config->FindB("Acquire::Check-Valid-Until", true) == true &&
1275        MetaIndexParser->GetValidUntil() > 0) {
1276       time_t const invalid_since = time(NULL) - MetaIndexParser->GetValidUntil();
1277       if (invalid_since > 0)
1278          // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1279          // the time since then the file is invalid - formated in the same way as in
1280          // the download progress display (e.g. 7d 3h 42min 1s)
1281          return _error->Error(_("Release file expired, ignoring %s (invalid since %s)"),
1282                               RealURI.c_str(), TimeToStr(invalid_since).c_str());
1283    }
1284
1285    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1286    {
1287       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1288       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1289       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1290    }
1291
1292    if (MetaIndexParser->CheckDist(Transformed) == false)
1293    {
1294       // This might become fatal one day
1295 //       Status = StatAuthError;
1296 //       ErrorText = "Conflicting distribution; expected "
1297 //          + MetaIndexParser->GetExpectedDist() + " but got "
1298 //          + MetaIndexParser->GetDist();
1299 //       return false;
1300       if (!Transformed.empty())
1301       {
1302          _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1303                          Desc.Description.c_str(),
1304                          Transformed.c_str(),
1305                          MetaIndexParser->GetDist().c_str());
1306       }
1307    }
1308
1309    return true;
1310 }
1311                                                                         /*}}}*/
1312 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1313 // ---------------------------------------------------------------------
1314 /* */
1315 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1316 {
1317    if (AuthPass == true)
1318    {
1319       // gpgv method failed, if we have a good signature
1320       string LastGoodSigFile = _config->FindDir("Dir::State::lists") +
1321          "partial/" + URItoFileName(RealURI) + ".gpg.reverify";
1322       if(FileExists(LastGoodSigFile))
1323       {
1324          string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1325             URItoFileName(RealURI) + ".gpg";
1326          Rename(LastGoodSigFile,VerifiedSigFile);
1327          Status = StatTransientNetworkError;
1328          _error->Warning(_("A error occurred during the signature "
1329                            "verification. The repository is not updated "
1330                            "and the previous index files will be used. "
1331                            "GPG error: %s: %s\n"),
1332                          Desc.Description.c_str(),
1333                          LookupTag(Message,"Message").c_str());
1334          RunScripts("APT::Update::Auth-Failure");
1335          return;
1336       } else {
1337          _error->Warning(_("GPG error: %s: %s"),
1338                          Desc.Description.c_str(),
1339                          LookupTag(Message,"Message").c_str());
1340       }
1341       // gpgv method failed
1342       ReportMirrorFailure("GPGFailure");
1343    }
1344
1345    // No Release file was present, or verification failed, so fall
1346    // back to queueing Packages files without verification
1347    QueueIndexes(false);
1348 }
1349                                                                         /*}}}*/
1350 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1351 // ---------------------------------------------------------------------
1352 /* This just sets up the initial fetch environment and queues the first
1353    possibilitiy */
1354 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1355                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1356                              string &StoreFilename) :
1357                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1358                StoreFilename(StoreFilename), Vf(Version.FileList()),
1359                Trusted(false)
1360 {
1361    Retries = _config->FindI("Acquire::Retries",0);
1362
1363    if (Version.Arch() == 0)
1364    {
1365       _error->Error(_("I wasn't able to locate a file for the %s package. "
1366                       "This might mean you need to manually fix this package. "
1367                       "(due to missing arch)"),
1368                     Version.ParentPkg().Name());
1369       return;
1370    }
1371
1372    /* We need to find a filename to determine the extension. We make the
1373       assumption here that all the available sources for this version share
1374       the same extension.. */
1375    // Skip not source sources, they do not have file fields.
1376    for (; Vf.end() == false; Vf++)
1377    {
1378       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1379          continue;
1380       break;
1381    }
1382
1383    // Does not really matter here.. we are going to fail out below
1384    if (Vf.end() != true)
1385    {
1386       // If this fails to get a file name we will bomb out below.
1387       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1388       if (_error->PendingError() == true)
1389          return;
1390
1391       // Generate the final file name as: package_version_arch.foo
1392       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1393                       QuoteString(Version.VerStr(),"_:") + '_' +
1394                       QuoteString(Version.Arch(),"_:.") +
1395                       "." + flExtension(Parse.FileName());
1396    }
1397
1398    // check if we have one trusted source for the package. if so, switch
1399    // to "TrustedOnly" mode
1400    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; i++)
1401    {
1402       pkgIndexFile *Index;
1403       if (Sources->FindIndex(i.File(),Index) == false)
1404          continue;
1405       if (_config->FindB("Debug::pkgAcquire::Auth", false))
1406       {
1407          std::cerr << "Checking index: " << Index->Describe()
1408                    << "(Trusted=" << Index->IsTrusted() << ")\n";
1409       }
1410       if (Index->IsTrusted()) {
1411          Trusted = true;
1412          break;
1413       }
1414    }
1415
1416    // "allow-unauthenticated" restores apts old fetching behaviour
1417    // that means that e.g. unauthenticated file:// uris are higher
1418    // priority than authenticated http:// uris
1419    if (_config->FindB("APT::Get::AllowUnauthenticated",false) == true)
1420       Trusted = false;
1421
1422    // Select a source
1423    if (QueueNext() == false && _error->PendingError() == false)
1424       _error->Error(_("I wasn't able to locate file for the %s package. "
1425                     "This might mean you need to manually fix this package."),
1426                     Version.ParentPkg().Name());
1427 }
1428                                                                         /*}}}*/
1429 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1430 // ---------------------------------------------------------------------
1431 /* This queues the next available file version for download. It checks if
1432    the archive is already available in the cache and stashs the MD5 for
1433    checking later. */
1434 bool pkgAcqArchive::QueueNext()
1435 {
1436    string const ForceHash = _config->Find("Acquire::ForceHash");
1437    for (; Vf.end() == false; Vf++)
1438    {
1439       // Ignore not source sources
1440       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1441          continue;
1442
1443       // Try to cross match against the source list
1444       pkgIndexFile *Index;
1445       if (Sources->FindIndex(Vf.File(),Index) == false)
1446             continue;
1447
1448       // only try to get a trusted package from another source if that source
1449       // is also trusted
1450       if(Trusted && !Index->IsTrusted())
1451          continue;
1452
1453       // Grab the text package record
1454       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1455       if (_error->PendingError() == true)
1456          return false;
1457
1458       string PkgFile = Parse.FileName();
1459       if (ForceHash.empty() == false)
1460       {
1461          if(stringcasecmp(ForceHash, "sha256") == 0)
1462             ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
1463          else if (stringcasecmp(ForceHash, "sha1") == 0)
1464             ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
1465          else
1466             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1467       }
1468       else
1469       {
1470          string Hash;
1471          if ((Hash = Parse.SHA256Hash()).empty() == false)
1472             ExpectedHash = HashString("SHA256", Hash);
1473          else if ((Hash = Parse.SHA1Hash()).empty() == false)
1474             ExpectedHash = HashString("SHA1", Hash);
1475          else
1476             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1477       }
1478       if (PkgFile.empty() == true)
1479          return _error->Error(_("The package index files are corrupted. No Filename: "
1480                               "field for package %s."),
1481                               Version.ParentPkg().Name());
1482
1483       Desc.URI = Index->ArchiveURI(PkgFile);
1484       Desc.Description = Index->ArchiveInfo(Version);
1485       Desc.Owner = this;
1486       Desc.ShortDesc = Version.ParentPkg().Name();
1487
1488       // See if we already have the file. (Legacy filenames)
1489       FileSize = Version->Size;
1490       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
1491       struct stat Buf;
1492       if (stat(FinalFile.c_str(),&Buf) == 0)
1493       {
1494          // Make sure the size matches
1495          if ((unsigned)Buf.st_size == Version->Size)
1496          {
1497             Complete = true;
1498             Local = true;
1499             Status = StatDone;
1500             StoreFilename = DestFile = FinalFile;
1501             return true;
1502          }
1503
1504          /* Hmm, we have a file and its size does not match, this means it is
1505             an old style mismatched arch */
1506          unlink(FinalFile.c_str());
1507       }
1508
1509       // Check it again using the new style output filenames
1510       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
1511       if (stat(FinalFile.c_str(),&Buf) == 0)
1512       {
1513          // Make sure the size matches
1514          if ((unsigned)Buf.st_size == Version->Size)
1515          {
1516             Complete = true;
1517             Local = true;
1518             Status = StatDone;
1519             StoreFilename = DestFile = FinalFile;
1520             return true;
1521          }
1522
1523          /* Hmm, we have a file and its size does not match, this shouldnt
1524             happen.. */
1525          unlink(FinalFile.c_str());
1526       }
1527
1528       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
1529
1530       // Check the destination file
1531       if (stat(DestFile.c_str(),&Buf) == 0)
1532       {
1533          // Hmm, the partial file is too big, erase it
1534          if ((unsigned)Buf.st_size > Version->Size)
1535             unlink(DestFile.c_str());
1536          else
1537             PartialSize = Buf.st_size;
1538       }
1539
1540       // Create the item
1541       Local = false;
1542       Desc.URI = Index->ArchiveURI(PkgFile);
1543       Desc.Description = Index->ArchiveInfo(Version);
1544       Desc.Owner = this;
1545       Desc.ShortDesc = Version.ParentPkg().Name();
1546       QueueURI(Desc);
1547
1548       Vf++;
1549       return true;
1550    }
1551    return false;
1552 }
1553                                                                         /*}}}*/
1554 // AcqArchive::Done - Finished fetching                                 /*{{{*/
1555 // ---------------------------------------------------------------------
1556 /* */
1557 void pkgAcqArchive::Done(string Message,unsigned long Size,string CalcHash,
1558                          pkgAcquire::MethodConfig *Cfg)
1559 {
1560    Item::Done(Message,Size,CalcHash,Cfg);
1561
1562    // Check the size
1563    if (Size != Version->Size)
1564    {
1565       Status = StatError;
1566       ErrorText = _("Size mismatch");
1567       return;
1568    }
1569
1570    // Check the hash
1571    if(ExpectedHash.toStr() != CalcHash)
1572    {
1573       Status = StatError;
1574       ErrorText = _("Hash Sum mismatch");
1575       if(FileExists(DestFile))
1576          Rename(DestFile,DestFile + ".FAILED");
1577       return;
1578    }
1579
1580    // Grab the output filename
1581    string FileName = LookupTag(Message,"Filename");
1582    if (FileName.empty() == true)
1583    {
1584       Status = StatError;
1585       ErrorText = "Method gave a blank filename";
1586       return;
1587    }
1588
1589    Complete = true;
1590
1591    // Reference filename
1592    if (FileName != DestFile)
1593    {
1594       StoreFilename = DestFile = FileName;
1595       Local = true;
1596       return;
1597    }
1598
1599    // Done, move it into position
1600    string FinalFile = _config->FindDir("Dir::Cache::Archives");
1601    FinalFile += flNotDir(StoreFilename);
1602    Rename(DestFile,FinalFile);
1603
1604    StoreFilename = DestFile = FinalFile;
1605    Complete = true;
1606 }
1607                                                                         /*}}}*/
1608 // AcqArchive::Failed - Failure handler                                 /*{{{*/
1609 // ---------------------------------------------------------------------
1610 /* Here we try other sources */
1611 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1612 {
1613    ErrorText = LookupTag(Message,"Message");
1614
1615    /* We don't really want to retry on failed media swaps, this prevents
1616       that. An interesting observation is that permanent failures are not
1617       recorded. */
1618    if (Cnf->Removable == true &&
1619        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1620    {
1621       // Vf = Version.FileList();
1622       while (Vf.end() == false) Vf++;
1623       StoreFilename = string();
1624       Item::Failed(Message,Cnf);
1625       return;
1626    }
1627
1628    if (QueueNext() == false)
1629    {
1630       // This is the retry counter
1631       if (Retries != 0 &&
1632           Cnf->LocalOnly == false &&
1633           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1634       {
1635          Retries--;
1636          Vf = Version.FileList();
1637          if (QueueNext() == true)
1638             return;
1639       }
1640
1641       StoreFilename = string();
1642       Item::Failed(Message,Cnf);
1643    }
1644 }
1645                                                                         /*}}}*/
1646 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
1647 // ---------------------------------------------------------------------
1648 bool pkgAcqArchive::IsTrusted()
1649 {
1650    return Trusted;
1651 }
1652                                                                         /*}}}*/
1653 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
1654 // ---------------------------------------------------------------------
1655 /* */
1656 void pkgAcqArchive::Finished()
1657 {
1658    if (Status == pkgAcquire::Item::StatDone &&
1659        Complete == true)
1660       return;
1661    StoreFilename = string();
1662 }
1663                                                                         /*}}}*/
1664 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
1665 // ---------------------------------------------------------------------
1666 /* The file is added to the queue */
1667 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
1668                        unsigned long Size,string Dsc,string ShortDesc,
1669                        const string &DestDir, const string &DestFilename,
1670                        bool IsIndexFile) :
1671                        Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile)
1672 {
1673    Retries = _config->FindI("Acquire::Retries",0);
1674
1675    if(!DestFilename.empty())
1676       DestFile = DestFilename;
1677    else if(!DestDir.empty())
1678       DestFile = DestDir + "/" + flNotDir(URI);
1679    else
1680       DestFile = flNotDir(URI);
1681
1682    // Create the item
1683    Desc.URI = URI;
1684    Desc.Description = Dsc;
1685    Desc.Owner = this;
1686
1687    // Set the short description to the archive component
1688    Desc.ShortDesc = ShortDesc;
1689
1690    // Get the transfer sizes
1691    FileSize = Size;
1692    struct stat Buf;
1693    if (stat(DestFile.c_str(),&Buf) == 0)
1694    {
1695       // Hmm, the partial file is too big, erase it
1696       if ((unsigned)Buf.st_size > Size)
1697          unlink(DestFile.c_str());
1698       else
1699          PartialSize = Buf.st_size;
1700    }
1701
1702    QueueURI(Desc);
1703 }
1704                                                                         /*}}}*/
1705 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
1706 // ---------------------------------------------------------------------
1707 /* */
1708 void pkgAcqFile::Done(string Message,unsigned long Size,string CalcHash,
1709                       pkgAcquire::MethodConfig *Cnf)
1710 {
1711    Item::Done(Message,Size,CalcHash,Cnf);
1712
1713    // Check the hash
1714    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
1715    {
1716       Status = StatError;
1717       ErrorText = _("Hash Sum mismatch");
1718       Rename(DestFile,DestFile + ".FAILED");
1719       return;
1720    }
1721
1722    string FileName = LookupTag(Message,"Filename");
1723    if (FileName.empty() == true)
1724    {
1725       Status = StatError;
1726       ErrorText = "Method gave a blank filename";
1727       return;
1728    }
1729
1730    Complete = true;
1731
1732    // The files timestamp matches
1733    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1734       return;
1735
1736    // We have to copy it into place
1737    if (FileName != DestFile)
1738    {
1739       Local = true;
1740       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
1741           Cnf->Removable == true)
1742       {
1743          Desc.URI = "copy:" + FileName;
1744          QueueURI(Desc);
1745          return;
1746       }
1747
1748       // Erase the file if it is a symlink so we can overwrite it
1749       struct stat St;
1750       if (lstat(DestFile.c_str(),&St) == 0)
1751       {
1752          if (S_ISLNK(St.st_mode) != 0)
1753             unlink(DestFile.c_str());
1754       }
1755
1756       // Symlink the file
1757       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
1758       {
1759          ErrorText = "Link to " + DestFile + " failure ";
1760          Status = StatError;
1761          Complete = false;
1762       }
1763    }
1764 }
1765                                                                         /*}}}*/
1766 // AcqFile::Failed - Failure handler                                    /*{{{*/
1767 // ---------------------------------------------------------------------
1768 /* Here we try other sources */
1769 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1770 {
1771    ErrorText = LookupTag(Message,"Message");
1772
1773    // This is the retry counter
1774    if (Retries != 0 &&
1775        Cnf->LocalOnly == false &&
1776        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1777    {
1778       Retries--;
1779       QueueURI(Desc);
1780       return;
1781    }
1782
1783    Item::Failed(Message,Cnf);
1784 }
1785                                                                         /*}}}*/
1786 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
1787 // ---------------------------------------------------------------------
1788 /* The only header we use is the last-modified header. */
1789 string pkgAcqFile::Custom600Headers()
1790 {
1791    if (IsIndexFile)
1792       return "\nIndex-File: true";
1793    return "";
1794 }
1795                                                                         /*}}}*/