apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <apt-pkg/acquire-item.h>
  17 #include <apt-pkg/configuration.h>
  18 #include <apt-pkg/aptconfiguration.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/vendorlist.h>
  21 #include <apt-pkg/error.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <apt-pkg/fileutl.h>
  24 #include <apt-pkg/md5.h>
  25 #include <apt-pkg/sha1.h>
  26 #include <apt-pkg/tagfile.h>
  27
  28 #include <apti18n.h>
  29
  30 #include <sys/stat.h>
  31 #include <unistd.h>
  32 #include <errno.h>
  33 #include <string>
  34 #include <sstream>
  35 #include <stdio.h>
  36 #include <ctime>
  37                                                                         /*}}}*/
  38
  39 using namespace std;
  40
  41 // Acquire::Item::Item - Constructor                                    /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* */
  44 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  45                        PartialSize(0), Mode(0), ID(0), Complete(false),
  46                        Local(false), QueueCounter(0)
  47 {
  48    Owner->Add(this);
  49    Status = StatIdle;
  50 }
  51                                                                         /*}}}*/
  52 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  53 // ---------------------------------------------------------------------
  54 /* */
  55 pkgAcquire::Item::~Item()
  56 {
  57    Owner->Remove(this);
  58 }
  59                                                                         /*}}}*/
  60 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  61 // ---------------------------------------------------------------------
  62 /* We return to an idle state if there are still other queues that could
  63    fetch this object */
  64 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  65 {
  66    Status = StatIdle;
  67    ErrorText = LookupTag(Message,"Message");
  68    UsedMirror =  LookupTag(Message,"UsedMirror");
  69    if (QueueCounter <= 1)
  70    {
  71       /* This indicates that the file is not available right now but might
  72          be sometime later. If we do a retry cycle then this should be
  73          retried [CDROMs] */
  74       if (Cnf->LocalOnly == true &&
  75           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  76       {
  77          Status = StatIdle;
  78          Dequeue();
  79          return;
  80       }
  81
  82       Status = StatError;
  83       Dequeue();
  84    }
  85
  86    // report mirror failure back to LP if we actually use a mirror
  87    string FailReason = LookupTag(Message, "FailReason");
  88    if(FailReason.size() != 0)
  89       ReportMirrorFailure(FailReason);
  90    else
  91       ReportMirrorFailure(ErrorText);
  92 }
  93                                                                         /*}}}*/
  94 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
  95 // ---------------------------------------------------------------------
  96 /* Stash status and the file size. Note that setting Complete means
  97    sub-phases of the acquire process such as decompresion are operating */
  98 void pkgAcquire::Item::Start(string /*Message*/,unsigned long Size)
  99 {
 100    Status = StatFetching;
 101    if (FileSize == 0 && Complete == false)
 102       FileSize = Size;
 103 }
 104                                                                         /*}}}*/
 105 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
 106 // ---------------------------------------------------------------------
 107 /* */
 108 void pkgAcquire::Item::Done(string Message,unsigned long Size,string Hash,
 109                             pkgAcquire::MethodConfig *Cnf)
 110 {
 111    // We just downloaded something..
 112    string FileName = LookupTag(Message,"Filename");
 113    UsedMirror =  LookupTag(Message,"UsedMirror");
 114    if (Complete == false && !Local && FileName == DestFile)
 115    {
 116       if (Owner->Log != 0)
 117          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 118    }
 119
 120    if (FileSize == 0)
 121       FileSize= Size;
 122    Status = StatDone;
 123    ErrorText = string();
 124    Owner->Dequeue(this);
 125 }
 126                                                                         /*}}}*/
 127 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 128 // ---------------------------------------------------------------------
 129 /* This helper function is used by alot of item methods as thier final
 130    step */
 131 void pkgAcquire::Item::Rename(string From,string To)
 132 {
 133    if (rename(From.c_str(),To.c_str()) != 0)
 134    {
 135       char S[300];
 136       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 137               From.c_str(),To.c_str());
 138       Status = StatError;
 139       ErrorText = S;
 140    }
 141 }
 142                                                                         /*}}}*/
 143 // Acquire::Item::ReportMirrorFailure                                   /*{{{*/
 144 // ---------------------------------------------------------------------
 145 void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
 146 {
 147    // we only act if a mirror was used at all
 148    if(UsedMirror.empty())
 149       return;
 150 #if 0
 151    std::cerr << "\nReportMirrorFailure: "
 152              << UsedMirror
 153              << " Uri: " << DescURI()
 154              << " FailCode: "
 155              << FailCode << std::endl;
 156 #endif
 157    const char *Args[40];
 158    unsigned int i = 0;
 159    string report = _config->Find("Methods::Mirror::ProblemReporting",
 160                                  "/usr/lib/apt/apt-report-mirror-failure");
 161    if(!FileExists(report))
 162       return;
 163    Args[i++] = report.c_str();
 164    Args[i++] = UsedMirror.c_str();
 165    Args[i++] = DescURI().c_str();
 166    Args[i++] = FailCode.c_str();
 167    Args[i++] = NULL;
 168    pid_t pid = ExecFork();
 169    if(pid < 0)
 170    {
 171       _error->Error("ReportMirrorFailure Fork failed");
 172       return;
 173    }
 174    else if(pid == 0)
 175    {
 176       execvp(Args[0], (char**)Args);
 177       std::cerr << "Could not exec " << Args[0] << std::endl;
 178       _exit(100);
 179    }
 180    if(!ExecWait(pid, "report-mirror-failure"))
 181    {
 182       _error->Warning("Couldn't report problem to '%s'",
 183                       _config->Find("Methods::Mirror::ProblemReporting").c_str());
 184    }
 185 }
 186                                                                         /*}}}*/
 187 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 188 // ---------------------------------------------------------------------
 189 /* Get the DiffIndex file first and see if there are patches availabe
 190  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 191  * patches. If anything goes wrong in that process, it will fall back to
 192  * the original packages file
 193  */
 194 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 195                                  string URI,string URIDesc,string ShortDesc,
 196                                  HashString ExpectedHash)
 197    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 198      Description(URIDesc)
 199 {
 200
 201    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 202
 203    Desc.Description = URIDesc + "/DiffIndex";
 204    Desc.Owner = this;
 205    Desc.ShortDesc = ShortDesc;
 206    Desc.URI = URI + ".diff/Index";
 207
 208    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 209    DestFile += URItoFileName(URI) + string(".DiffIndex");
 210
 211    if(Debug)
 212       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 213
 214    // look for the current package file
 215    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 216    CurrentPackagesFile += URItoFileName(RealURI);
 217
 218    // FIXME: this file:/ check is a hack to prevent fetching
 219    //        from local sources. this is really silly, and
 220    //        should be fixed cleanly as soon as possible
 221    if(!FileExists(CurrentPackagesFile) ||
 222       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 223    {
 224       // we don't have a pkg file or we don't want to queue
 225       if(Debug)
 226          std::clog << "No index file, local or canceld by user" << std::endl;
 227       Failed("", NULL);
 228       return;
 229    }
 230
 231    if(Debug)
 232       std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): "
 233                 << CurrentPackagesFile << std::endl;
 234
 235    QueueURI(Desc);
 236
 237 }
 238                                                                         /*}}}*/
 239 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 240 // ---------------------------------------------------------------------
 241 /* The only header we use is the last-modified header. */
 242 string pkgAcqDiffIndex::Custom600Headers()
 243 {
 244    string Final = _config->FindDir("Dir::State::lists");
 245    Final += URItoFileName(RealURI) + string(".IndexDiff");
 246
 247    if(Debug)
 248       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 249
 250    struct stat Buf;
 251    if (stat(Final.c_str(),&Buf) != 0)
 252       return "\nIndex-File: true";
 253
 254    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 255 }
 256                                                                         /*}}}*/
 257 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 258 {
 259    if(Debug)
 260       std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile
 261                 << std::endl;
 262
 263    pkgTagSection Tags;
 264    string ServerSha1;
 265    vector<DiffInfo> available_patches;
 266
 267    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 268    pkgTagFile TF(&Fd);
 269    if (_error->PendingError() == true)
 270       return false;
 271
 272    if(TF.Step(Tags) == true)
 273    {
 274       bool found = false;
 275       DiffInfo d;
 276       string size;
 277
 278       string const tmp = Tags.FindS("SHA1-Current");
 279       std::stringstream ss(tmp);
 280       ss >> ServerSha1 >> size;
 281       unsigned long const ServerSize = atol(size.c_str());
 282
 283       FileFd fd(CurrentPackagesFile, FileFd::ReadOnlyGzip);
 284       SHA1Summation SHA1;
 285       SHA1.AddFD(fd.Fd(), fd.Size());
 286       string const local_sha1 = SHA1.Result();
 287
 288       if(local_sha1 == ServerSha1)
 289       {
 290          // we have the same sha1 as the server
 291          if(Debug)
 292             std::clog << "Package file is up-to-date" << std::endl;
 293          // set found to true, this will queue a pkgAcqIndexDiffs with
 294          // a empty availabe_patches
 295          found = true;
 296       }
 297       else
 298       {
 299          if(Debug)
 300             std::clog << "SHA1-Current: " << ServerSha1 << std::endl;
 301
 302          // check the historie and see what patches we need
 303          string const history = Tags.FindS("SHA1-History");
 304          std::stringstream hist(history);
 305          while(hist >> d.sha1 >> size >> d.file)
 306          {
 307             // read until the first match is found
 308             // from that point on, we probably need all diffs
 309             if(d.sha1 == local_sha1)
 310                found=true;
 311             else if (found == false)
 312                continue;
 313
 314             if(Debug)
 315                std::clog << "Need to get diff: " << d.file << std::endl;
 316             available_patches.push_back(d);
 317          }
 318
 319          if (available_patches.empty() == false)
 320          {
 321             // patching with too many files is rather slow compared to a fast download
 322             unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 323             if (fileLimit != 0 && fileLimit < available_patches.size())
 324             {
 325                if (Debug)
 326                   std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 327                         << ") so fallback to complete download" << std::endl;
 328                return false;
 329             }
 330
 331             // see if the patches are too big
 332             found = false; // it was true and it will be true again at the end
 333             d = *available_patches.begin();
 334             string const firstPatch = d.file;
 335             unsigned long patchesSize = 0;
 336             std::stringstream patches(Tags.FindS("SHA1-Patches"));
 337             while(patches >> d.sha1 >> size >> d.file)
 338             {
 339                if (firstPatch == d.file)
 340                   found = true;
 341                else if (found == false)
 342                   continue;
 343
 344                patchesSize += atol(size.c_str());
 345             }
 346             unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 347             if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 348             {
 349                if (Debug)
 350                   std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 351                         << ") so fallback to complete download" << std::endl;
 352                return false;
 353             }
 354          }
 355       }
 356
 357       // we have something, queue the next diff
 358       if(found)
 359       {
 360          // queue the diffs
 361          string::size_type const last_space = Description.rfind(" ");
 362          if(last_space != string::npos)
 363             Description.erase(last_space, Description.size()-last_space);
 364          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 365                               ExpectedHash, ServerSha1, available_patches);
 366          Complete = false;
 367          Status = StatDone;
 368          Dequeue();
 369          return true;
 370       }
 371    }
 372
 373    // Nothing found, report and return false
 374    // Failing here is ok, if we return false later, the full
 375    // IndexFile is queued
 376    if(Debug)
 377       std::clog << "Can't find a patch in the index file" << std::endl;
 378    return false;
 379 }
 380                                                                         /*}}}*/
 381 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)      /*{{{*/
 382 {
 383    if(Debug)
 384       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << std::endl
 385                 << "Falling back to normal index file aquire" << std::endl;
 386
 387    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 388                    ExpectedHash);
 389
 390    Complete = false;
 391    Status = StatDone;
 392    Dequeue();
 393 }
 394                                                                         /*}}}*/
 395 void pkgAcqDiffIndex::Done(string Message,unsigned long Size,string Md5Hash,    /*{{{*/
 396                            pkgAcquire::MethodConfig *Cnf)
 397 {
 398    if(Debug)
 399       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 400
 401    Item::Done(Message,Size,Md5Hash,Cnf);
 402
 403    string FinalFile;
 404    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 405
 406    // sucess in downloading the index
 407    // rename the index
 408    FinalFile += string(".IndexDiff");
 409    if(Debug)
 410       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 411                 << std::endl;
 412    Rename(DestFile,FinalFile);
 413    chmod(FinalFile.c_str(),0644);
 414    DestFile = FinalFile;
 415
 416    if(!ParseDiffIndex(DestFile))
 417       return Failed("", NULL);
 418
 419    Complete = true;
 420    Status = StatDone;
 421    Dequeue();
 422    return;
 423 }
 424                                                                         /*}}}*/
 425 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 426 // ---------------------------------------------------------------------
 427 /* The package diff is added to the queue. one object is constructed
 428  * for each diff and the index
 429  */
 430 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 431                                    string URI,string URIDesc,string ShortDesc,
 432                                    HashString ExpectedHash,
 433                                    string ServerSha1,
 434                                    vector<DiffInfo> diffs)
 435    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 436      available_patches(diffs), ServerSha1(ServerSha1)
 437 {
 438
 439    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 440    DestFile += URItoFileName(URI);
 441
 442    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 443
 444    Description = URIDesc;
 445    Desc.Owner = this;
 446    Desc.ShortDesc = ShortDesc;
 447
 448    if(available_patches.size() == 0)
 449    {
 450       // we are done (yeah!)
 451       Finish(true);
 452    }
 453    else
 454    {
 455       // get the next diff
 456       State = StateFetchDiff;
 457       QueueNextDiff();
 458    }
 459 }
 460                                                                         /*}}}*/
 461 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)     /*{{{*/
 462 {
 463    if(Debug)
 464       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl
 465                 << "Falling back to normal index file aquire" << std::endl;
 466    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 467                    ExpectedHash);
 468    Finish();
 469 }
 470                                                                         /*}}}*/
 471 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 472 void pkgAcqIndexDiffs::Finish(bool allDone)
 473 {
 474    // we restore the original name, this is required, otherwise
 475    // the file will be cleaned
 476    if(allDone)
 477    {
 478       DestFile = _config->FindDir("Dir::State::lists");
 479       DestFile += URItoFileName(RealURI);
 480
 481       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 482       {
 483          Status = StatAuthError;
 484          ErrorText = _("MD5Sum mismatch");
 485          Rename(DestFile,DestFile + ".FAILED");
 486          Dequeue();
 487          return;
 488       }
 489
 490       // this is for the "real" finish
 491       Complete = true;
 492       Status = StatDone;
 493       Dequeue();
 494       if(Debug)
 495          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 496       return;
 497    }
 498
 499    if(Debug)
 500       std::clog << "Finishing: " << Desc.URI << std::endl;
 501    Complete = false;
 502    Status = StatDone;
 503    Dequeue();
 504    return;
 505 }
 506                                                                         /*}}}*/
 507 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 508 {
 509
 510    // calc sha1 of the just patched file
 511    string FinalFile = _config->FindDir("Dir::State::lists");
 512    FinalFile += URItoFileName(RealURI);
 513
 514    FileFd fd(FinalFile, FileFd::ReadOnlyGzip);
 515    SHA1Summation SHA1;
 516    SHA1.AddFD(fd.Fd(), fd.Size());
 517    string local_sha1 = string(SHA1.Result());
 518    if(Debug)
 519       std::clog << "QueueNextDiff: "
 520                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 521
 522    // final file reached before all patches are applied
 523    if(local_sha1 == ServerSha1)
 524    {
 525       Finish(true);
 526       return true;
 527    }
 528
 529    // remove all patches until the next matching patch is found
 530    // this requires the Index file to be ordered
 531    for(vector<DiffInfo>::iterator I=available_patches.begin();
 532        available_patches.size() > 0 &&
 533           I != available_patches.end() &&
 534           (*I).sha1 != local_sha1;
 535        I++)
 536    {
 537       available_patches.erase(I);
 538    }
 539
 540    // error checking and falling back if no patch was found
 541    if(available_patches.size() == 0)
 542    {
 543       Failed("", NULL);
 544       return false;
 545    }
 546
 547    // queue the right diff
 548    Desc.URI = string(RealURI) + ".diff/" + available_patches[0].file + ".gz";
 549    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 550    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 551    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 552
 553    if(Debug)
 554       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 555
 556    QueueURI(Desc);
 557
 558    return true;
 559 }
 560                                                                         /*}}}*/
 561 void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash,   /*{{{*/
 562                             pkgAcquire::MethodConfig *Cnf)
 563 {
 564    if(Debug)
 565       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 566
 567    Item::Done(Message,Size,Md5Hash,Cnf);
 568
 569    string FinalFile;
 570    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 571
 572    // sucess in downloading a diff, enter ApplyDiff state
 573    if(State == StateFetchDiff)
 574    {
 575
 576       if(Debug)
 577          std::clog << "Sending to gzip method: " << FinalFile << std::endl;
 578
 579       string FileName = LookupTag(Message,"Filename");
 580       State = StateUnzipDiff;
 581       Local = true;
 582       Desc.URI = "gzip:" + FileName;
 583       DestFile += ".decomp";
 584       QueueURI(Desc);
 585       Mode = "gzip";
 586       return;
 587    }
 588
 589    // sucess in downloading a diff, enter ApplyDiff state
 590    if(State == StateUnzipDiff)
 591    {
 592
 593       // rred excepts the patch as $FinalFile.ed
 594       Rename(DestFile,FinalFile+".ed");
 595
 596       if(Debug)
 597          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 598
 599       State = StateApplyDiff;
 600       Local = true;
 601       Desc.URI = "rred:" + FinalFile;
 602       QueueURI(Desc);
 603       Mode = "rred";
 604       return;
 605    }
 606
 607
 608    // success in download/apply a diff, queue next (if needed)
 609    if(State == StateApplyDiff)
 610    {
 611       // remove the just applied patch
 612       available_patches.erase(available_patches.begin());
 613
 614       // move into place
 615       if(Debug)
 616       {
 617          std::clog << "Moving patched file in place: " << std::endl
 618                    << DestFile << " -> " << FinalFile << std::endl;
 619       }
 620       Rename(DestFile,FinalFile);
 621       chmod(FinalFile.c_str(),0644);
 622
 623       // see if there is more to download
 624       if(available_patches.size() > 0) {
 625          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 626                               ExpectedHash, ServerSha1, available_patches);
 627          return Finish();
 628       } else
 629          return Finish(true);
 630    }
 631 }
 632                                                                         /*}}}*/
 633 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 634 // ---------------------------------------------------------------------
 635 /* The package file is added to the queue and a second class is
 636    instantiated to fetch the revision file */
 637 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 638                          string URI,string URIDesc,string ShortDesc,
 639                          HashString ExpectedHash, string comprExt)
 640    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 641 {
 642    Decompression = false;
 643    Erase = false;
 644
 645    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 646    DestFile += URItoFileName(URI);
 647
 648    if(comprExt.empty())
 649    {
 650       // autoselect the compression method
 651       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 652       if (types.empty() == true)
 653          comprExt = "plain";
 654       else
 655          comprExt = "." + types[0];
 656    }
 657    CompressionExtension = ((comprExt == "plain" || comprExt == ".") ? "" : comprExt);
 658
 659    Desc.URI = URI + CompressionExtension;
 660
 661    Desc.Description = URIDesc;
 662    Desc.Owner = this;
 663    Desc.ShortDesc = ShortDesc;
 664
 665    QueueURI(Desc);
 666 }
 667                                                                         /*}}}*/
 668 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 669 // ---------------------------------------------------------------------
 670 /* The only header we use is the last-modified header. */
 671 string pkgAcqIndex::Custom600Headers()
 672 {
 673    string Final = _config->FindDir("Dir::State::lists");
 674    Final += URItoFileName(RealURI);
 675    if (_config->FindB("Acquire::GzipIndexes",false))
 676       Final += ".gz";
 677
 678    struct stat Buf;
 679    if (stat(Final.c_str(),&Buf) != 0)
 680       return "\nIndex-File: true";
 681    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 682 }
 683                                                                         /*}}}*/
 684 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
 685 {
 686    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 687
 688    for (std::vector<std::string>::const_iterator t = types.begin();
 689         t != types.end(); t++)
 690    {
 691       // jump over all already tried compression types
 692       const unsigned int nameLen = Desc.URI.size() - (*t).size();
 693       if(Desc.URI.substr(nameLen) != *t)
 694          continue;
 695
 696       // we want to try it with the next extension (and make sure to
 697       // not skip over the end)
 698       t++;
 699       if (t == types.end())
 700          break;
 701
 702       // queue new download
 703       Desc.URI = Desc.URI.substr(0, nameLen) + *t;
 704       new pkgAcqIndex(Owner, RealURI, Desc.Description, Desc.ShortDesc,
 705       ExpectedHash, string(".").append(*t));
 706
 707       Status = StatDone;
 708       Complete = false;
 709       Dequeue();
 710       return;
 711    }
 712
 713    // on decompression failure, remove bad versions in partial/
 714    if(Decompression && Erase) {
 715       string s = _config->FindDir("Dir::State::lists") + "partial/";
 716       s += URItoFileName(RealURI);
 717       unlink(s.c_str());
 718    }
 719
 720    Item::Failed(Message,Cnf);
 721 }
 722                                                                         /*}}}*/
 723 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
 724 // ---------------------------------------------------------------------
 725 /* This goes through a number of states.. On the initial fetch the
 726    method could possibly return an alternate filename which points
 727    to the uncompressed version of the file. If this is so the file
 728    is copied into the partial directory. In all other cases the file
 729    is decompressed with a gzip uri. */
 730 void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash,
 731                        pkgAcquire::MethodConfig *Cfg)
 732 {
 733    Item::Done(Message,Size,Hash,Cfg);
 734
 735    if (Decompression == true)
 736    {
 737       if (_config->FindB("Debug::pkgAcquire::Auth", false))
 738       {
 739          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
 740          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
 741       }
 742
 743       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
 744       {
 745          Status = StatAuthError;
 746          ErrorText = _("Hash Sum mismatch");
 747          Rename(DestFile,DestFile + ".FAILED");
 748          ReportMirrorFailure("HashChecksumFailure");
 749          return;
 750       }
 751       // Done, move it into position
 752       string FinalFile = _config->FindDir("Dir::State::lists");
 753       FinalFile += URItoFileName(RealURI);
 754       Rename(DestFile,FinalFile);
 755       chmod(FinalFile.c_str(),0644);
 756
 757       /* We restore the original name to DestFile so that the clean operation
 758          will work OK */
 759       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 760       DestFile += URItoFileName(RealURI);
 761
 762       // Remove the compressed version.
 763       if (Erase == true)
 764          unlink(DestFile.c_str());
 765       return;
 766    }
 767
 768    Erase = false;
 769    Complete = true;
 770
 771    // Handle the unzipd case
 772    string FileName = LookupTag(Message,"Alt-Filename");
 773    if (FileName.empty() == false)
 774    {
 775       // The files timestamp matches
 776       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
 777          return;
 778       Decompression = true;
 779       Local = true;
 780       DestFile += ".decomp";
 781       Desc.URI = "copy:" + FileName;
 782       QueueURI(Desc);
 783       Mode = "copy";
 784       return;
 785    }
 786
 787    FileName = LookupTag(Message,"Filename");
 788    if (FileName.empty() == true)
 789    {
 790       Status = StatError;
 791       ErrorText = "Method gave a blank filename";
 792    }
 793
 794    string compExt = flExtension(flNotDir(URI(Desc.URI).Path));
 795
 796    // The files timestamp matches
 797    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) {
 798        if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
 799           // Update DestFile for .gz suffix so that the clean operation keeps it
 800           DestFile += ".gz";
 801       return;
 802     }
 803
 804    if (FileName == DestFile)
 805       Erase = true;
 806    else
 807       Local = true;
 808
 809    string decompProg;
 810
 811    // If we enable compressed indexes and already have gzip, keep it
 812    if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) {
 813       string FinalFile = _config->FindDir("Dir::State::lists");
 814       FinalFile += URItoFileName(RealURI) + ".gz";
 815       Rename(DestFile,FinalFile);
 816       chmod(FinalFile.c_str(),0644);
 817
 818       // Update DestFile for .gz suffix so that the clean operation keeps it
 819       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 820       DestFile += URItoFileName(RealURI) + ".gz";
 821       return;
 822     }
 823
 824    // get the binary name for your used compression type
 825    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
 826    if(decompProg.empty() == false);
 827    // flExtensions returns the full name if no extension is found
 828    // this is why we have this complicated compare operation here
 829    // FIMXE: add a new flJustExtension() that return "" if no
 830    //        extension is found and use that above so that it can
 831    //        be tested against ""
 832    else if(compExt == flNotDir(URI(Desc.URI).Path))
 833       decompProg = "copy";
 834    else {
 835       _error->Error("Unsupported extension: %s", compExt.c_str());
 836       return;
 837    }
 838
 839    Decompression = true;
 840    DestFile += ".decomp";
 841    Desc.URI = decompProg + ":" + FileName;
 842    QueueURI(Desc);
 843    Mode = decompProg.c_str();
 844 }
 845                                                                         /*}}}*/
 846 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
 847 // ---------------------------------------------------------------------
 848 /* The Translation file is added to the queue */
 849 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
 850                             string URI,string URIDesc,string ShortDesc)
 851   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
 852 {
 853 }
 854                                                                         /*}}}*/
 855 // AcqIndexTrans::Custom600Headers - Insert custom request headers      /*{{{*/
 856 // ---------------------------------------------------------------------
 857 string pkgAcqIndexTrans::Custom600Headers()
 858 {
 859    string Final = _config->FindDir("Dir::State::lists");
 860    Final += URItoFileName(RealURI);
 861
 862    struct stat Buf;
 863    if (stat(Final.c_str(),&Buf) != 0)
 864       return "\nFail-Ignore: true";
 865    return "\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 866 }
 867                                                                         /*}}}*/
 868 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
 869 // ---------------------------------------------------------------------
 870 /* */
 871 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 872 {
 873    if (Cnf->LocalOnly == true ||
 874        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 875    {
 876       // Ignore this
 877       Status = StatDone;
 878       Complete = false;
 879       Dequeue();
 880       return;
 881    }
 882
 883    Item::Failed(Message,Cnf);
 884 }
 885                                                                         /*}}}*/
 886 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
 887                              string URI,string URIDesc,string ShortDesc,
 888                              string MetaIndexURI, string MetaIndexURIDesc,
 889                              string MetaIndexShortDesc,
 890                              const vector<IndexTarget*>* IndexTargets,
 891                              indexRecords* MetaIndexParser) :
 892    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
 893    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
 894    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
 895 {
 896    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 897    DestFile += URItoFileName(URI);
 898
 899    // remove any partial downloaded sig-file in partial/.
 900    // it may confuse proxies and is too small to warrant a
 901    // partial download anyway
 902    unlink(DestFile.c_str());
 903
 904    // Create the item
 905    Desc.Description = URIDesc;
 906    Desc.Owner = this;
 907    Desc.ShortDesc = ShortDesc;
 908    Desc.URI = URI;
 909
 910    string Final = _config->FindDir("Dir::State::lists");
 911    Final += URItoFileName(RealURI);
 912    struct stat Buf;
 913    if (stat(Final.c_str(),&Buf) == 0)
 914    {
 915       // File was already in place.  It needs to be re-downloaded/verified
 916       // because Release might have changed, we do give it a differnt
 917       // name than DestFile because otherwise the http method will
 918       // send If-Range requests and there are too many broken servers
 919       // out there that do not understand them
 920       LastGoodSig = DestFile+".reverify";
 921       Rename(Final,LastGoodSig);
 922    }
 923
 924    QueueURI(Desc);
 925 }
 926                                                                         /*}}}*/
 927 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
 928 // ---------------------------------------------------------------------
 929 /* The only header we use is the last-modified header. */
 930 string pkgAcqMetaSig::Custom600Headers()
 931 {
 932    struct stat Buf;
 933    if (stat(LastGoodSig.c_str(),&Buf) != 0)
 934       return "\nIndex-File: true";
 935
 936    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 937 }
 938
 939 void pkgAcqMetaSig::Done(string Message,unsigned long Size,string MD5,
 940                          pkgAcquire::MethodConfig *Cfg)
 941 {
 942    Item::Done(Message,Size,MD5,Cfg);
 943
 944    string FileName = LookupTag(Message,"Filename");
 945    if (FileName.empty() == true)
 946    {
 947       Status = StatError;
 948       ErrorText = "Method gave a blank filename";
 949       return;
 950    }
 951
 952    if (FileName != DestFile)
 953    {
 954       // We have to copy it into place
 955       Local = true;
 956       Desc.URI = "copy:" + FileName;
 957       QueueURI(Desc);
 958       return;
 959    }
 960
 961    Complete = true;
 962
 963    // put the last known good file back on i-m-s hit (it will
 964    // be re-verified again)
 965    // Else do nothing, we have the new file in DestFile then
 966    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 967       Rename(LastGoodSig, DestFile);
 968
 969    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
 970    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc,
 971                        MetaIndexShortDesc,  DestFile, IndexTargets,
 972                        MetaIndexParser);
 973
 974 }
 975                                                                         /*}}}*/
 976 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
 977 {
 978    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 979
 980    // if we get a network error we fail gracefully
 981    if(Status == StatTransientNetworkError)
 982    {
 983       Item::Failed(Message,Cnf);
 984       // move the sigfile back on transient network failures
 985       if(FileExists(LastGoodSig))
 986          Rename(LastGoodSig,Final);
 987
 988       // set the status back to , Item::Failed likes to reset it
 989       Status = pkgAcquire::Item::StatTransientNetworkError;
 990       return;
 991    }
 992
 993    // Delete any existing sigfile when the acquire failed
 994    unlink(Final.c_str());
 995
 996    // queue a pkgAcqMetaIndex with no sigfile
 997    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
 998                        "", IndexTargets, MetaIndexParser);
 999
1000    if (Cnf->LocalOnly == true ||
1001        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1002    {
1003       // Ignore this
1004       Status = StatDone;
1005       Complete = false;
1006       Dequeue();
1007       return;
1008    }
1009
1010    Item::Failed(Message,Cnf);
1011 }
1012                                                                         /*}}}*/
1013 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
1014                                  string URI,string URIDesc,string ShortDesc,
1015                                  string SigFile,
1016                                  const vector<struct IndexTarget*>* IndexTargets,
1017                                  indexRecords* MetaIndexParser) :
1018    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
1019    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
1020 {
1021    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1022    DestFile += URItoFileName(URI);
1023
1024    // Create the item
1025    Desc.Description = URIDesc;
1026    Desc.Owner = this;
1027    Desc.ShortDesc = ShortDesc;
1028    Desc.URI = URI;
1029
1030    QueueURI(Desc);
1031 }
1032                                                                         /*}}}*/
1033 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
1034 // ---------------------------------------------------------------------
1035 /* The only header we use is the last-modified header. */
1036 string pkgAcqMetaIndex::Custom600Headers()
1037 {
1038    string Final = _config->FindDir("Dir::State::lists");
1039    Final += URItoFileName(RealURI);
1040
1041    struct stat Buf;
1042    if (stat(Final.c_str(),&Buf) != 0)
1043       return "\nIndex-File: true";
1044
1045    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1046 }
1047                                                                         /*}}}*/
1048 void pkgAcqMetaIndex::Done(string Message,unsigned long Size,string Hash,       /*{{{*/
1049                            pkgAcquire::MethodConfig *Cfg)
1050 {
1051    Item::Done(Message,Size,Hash,Cfg);
1052
1053    // MetaIndexes are done in two passes: one to download the
1054    // metaindex with an appropriate method, and a second to verify it
1055    // with the gpgv method
1056
1057    if (AuthPass == true)
1058    {
1059       AuthDone(Message);
1060
1061       // all cool, move Release file into place
1062       Complete = true;
1063
1064       string FinalFile = _config->FindDir("Dir::State::lists");
1065       FinalFile += URItoFileName(RealURI);
1066       Rename(DestFile,FinalFile);
1067       chmod(FinalFile.c_str(),0644);
1068       DestFile = FinalFile;
1069    }
1070    else
1071    {
1072       RetrievalDone(Message);
1073       if (!Complete)
1074          // Still more retrieving to do
1075          return;
1076
1077       if (SigFile == "")
1078       {
1079          // There was no signature file, so we are finished.  Download
1080          // the indexes without verification.
1081          QueueIndexes(false);
1082       }
1083       else
1084       {
1085          // There was a signature file, so pass it to gpgv for
1086          // verification
1087
1088          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1089             std::cerr << "Metaindex acquired, queueing gpg verification ("
1090                       << SigFile << "," << DestFile << ")\n";
1091          AuthPass = true;
1092          Desc.URI = "gpgv:" + SigFile;
1093          QueueURI(Desc);
1094          Mode = "gpgv";
1095       }
1096    }
1097 }
1098                                                                         /*}}}*/
1099 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
1100 {
1101    // We have just finished downloading a Release file (it is not
1102    // verified yet)
1103
1104    string FileName = LookupTag(Message,"Filename");
1105    if (FileName.empty() == true)
1106    {
1107       Status = StatError;
1108       ErrorText = "Method gave a blank filename";
1109       return;
1110    }
1111
1112    if (FileName != DestFile)
1113    {
1114       Local = true;
1115       Desc.URI = "copy:" + FileName;
1116       QueueURI(Desc);
1117       return;
1118    }
1119
1120    // make sure to verify against the right file on I-M-S hit
1121    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1122    if(IMSHit)
1123    {
1124       string FinalFile = _config->FindDir("Dir::State::lists");
1125       FinalFile += URItoFileName(RealURI);
1126       DestFile = FinalFile;
1127    }
1128    Complete = true;
1129 }
1130                                                                         /*}}}*/
1131 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1132 {
1133    // At this point, the gpgv method has succeeded, so there is a
1134    // valid signature from a key in the trusted keyring.  We
1135    // perform additional verification of its contents, and use them
1136    // to verify the indexes we are about to download
1137
1138    if (!MetaIndexParser->Load(DestFile))
1139    {
1140       Status = StatAuthError;
1141       ErrorText = MetaIndexParser->ErrorText;
1142       return;
1143    }
1144
1145    if (!VerifyVendor(Message))
1146    {
1147       return;
1148    }
1149
1150    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1151       std::cerr << "Signature verification succeeded: "
1152                 << DestFile << std::endl;
1153
1154    // Download further indexes with verification
1155    QueueIndexes(true);
1156
1157    // Done, move signature file into position
1158    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1159       URItoFileName(RealURI) + ".gpg";
1160    Rename(SigFile,VerifiedSigFile);
1161    chmod(VerifiedSigFile.c_str(),0644);
1162 }
1163                                                                         /*}}}*/
1164 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1165 {
1166    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1167         Target != IndexTargets->end();
1168         Target++)
1169    {
1170       HashString ExpectedIndexHash;
1171       if (verify)
1172       {
1173          const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1174          if (!Record)
1175          {
1176             Status = StatAuthError;
1177             ErrorText = "Unable to find expected entry  "
1178                + (*Target)->MetaKey + " in Meta-index file (malformed Release file?)";
1179             return;
1180          }
1181          ExpectedIndexHash = Record->Hash;
1182          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1183          {
1184             std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1185             std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1186          }
1187          if (ExpectedIndexHash.empty())
1188          {
1189             Status = StatAuthError;
1190             ErrorText = "Unable to find hash sum for "
1191                + (*Target)->MetaKey + " in Meta-index file";
1192             return;
1193          }
1194       }
1195
1196       /* Queue Packages file (either diff or full packages files, depending
1197          on the users option) - we also check if the PDiff Index file is listed
1198          in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
1199          instead, but passing the required info to it is to much hassle */
1200       if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
1201           MetaIndexParser->Exists(string((*Target)->MetaKey).append(".diff/Index")) == true))
1202          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1203                              (*Target)->ShortDesc, ExpectedIndexHash);
1204       else
1205          new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description,
1206                             (*Target)->ShortDesc, ExpectedIndexHash);
1207    }
1208 }
1209                                                                         /*}}}*/
1210 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1211 {
1212 //    // Maybe this should be made available from above so we don't have
1213 //    // to read and parse it every time?
1214 //    pkgVendorList List;
1215 //    List.ReadMainList();
1216
1217 //    const Vendor* Vndr = NULL;
1218 //    for (std::vector<string>::const_iterator I = GPGVOutput.begin(); I != GPGVOutput.end(); I++)
1219 //    {
1220 //       string::size_type pos = (*I).find("VALIDSIG ");
1221 //       if (_config->FindB("Debug::Vendor", false))
1222 //          std::cerr << "Looking for VALIDSIG in \"" << (*I) << "\": pos " << pos
1223 //                    << std::endl;
1224 //       if (pos != std::string::npos)
1225 //       {
1226 //          string Fingerprint = (*I).substr(pos+sizeof("VALIDSIG"));
1227 //          if (_config->FindB("Debug::Vendor", false))
1228 //             std::cerr << "Looking for \"" << Fingerprint << "\" in vendor..." <<
1229 //                std::endl;
1230 //          Vndr = List.FindVendor(Fingerprint) != "";
1231 //          if (Vndr != NULL);
1232 //          break;
1233 //       }
1234 //    }
1235    string::size_type pos;
1236
1237    // check for missing sigs (that where not fatal because otherwise we had
1238    // bombed earlier)
1239    string missingkeys;
1240    string msg = _("There is no public key available for the "
1241                   "following key IDs:\n");
1242    pos = Message.find("NO_PUBKEY ");
1243    if (pos != std::string::npos)
1244    {
1245       string::size_type start = pos+strlen("NO_PUBKEY ");
1246       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1247       missingkeys += (Fingerprint);
1248    }
1249    if(!missingkeys.empty())
1250       _error->Warning("%s", string(msg+missingkeys).c_str());
1251
1252    string Transformed = MetaIndexParser->GetExpectedDist();
1253
1254    if (Transformed == "../project/experimental")
1255    {
1256       Transformed = "experimental";
1257    }
1258
1259    pos = Transformed.rfind('/');
1260    if (pos != string::npos)
1261    {
1262       Transformed = Transformed.substr(0, pos);
1263    }
1264
1265    if (Transformed == ".")
1266    {
1267       Transformed = "";
1268    }
1269
1270    if (_config->FindB("Acquire::Check-Valid-Until", true) == true &&
1271        MetaIndexParser->GetValidUntil() > 0) {
1272       time_t const invalid_since = time(NULL) - MetaIndexParser->GetValidUntil();
1273       if (invalid_since > 0)
1274          // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1275          // the time since then the file is invalid - formated in the same way as in
1276          // the download progress display (e.g. 7d 3h 42min 1s)
1277          return _error->Error(_("Release file expired, ignoring %s (invalid since %s)"),
1278                               RealURI.c_str(), TimeToStr(invalid_since).c_str());
1279    }
1280
1281    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1282    {
1283       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1284       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1285       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1286    }
1287
1288    if (MetaIndexParser->CheckDist(Transformed) == false)
1289    {
1290       // This might become fatal one day
1291 //       Status = StatAuthError;
1292 //       ErrorText = "Conflicting distribution; expected "
1293 //          + MetaIndexParser->GetExpectedDist() + " but got "
1294 //          + MetaIndexParser->GetDist();
1295 //       return false;
1296       if (!Transformed.empty())
1297       {
1298          _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1299                          Desc.Description.c_str(),
1300                          Transformed.c_str(),
1301                          MetaIndexParser->GetDist().c_str());
1302       }
1303    }
1304
1305    return true;
1306 }
1307                                                                         /*}}}*/
1308 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1309 // ---------------------------------------------------------------------
1310 /* */
1311 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1312 {
1313    if (AuthPass == true)
1314    {
1315       // gpgv method failed, if we have a good signature
1316       string LastGoodSigFile = _config->FindDir("Dir::State::lists") +
1317          "partial/" + URItoFileName(RealURI) + ".gpg.reverify";
1318       if(FileExists(LastGoodSigFile))
1319       {
1320          string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1321             URItoFileName(RealURI) + ".gpg";
1322          Rename(LastGoodSigFile,VerifiedSigFile);
1323          Status = StatTransientNetworkError;
1324          _error->Warning(_("A error occurred during the signature "
1325                            "verification. The repository is not updated "
1326                            "and the previous index files will be used."
1327                            "GPG error: %s: %s\n"),
1328                          Desc.Description.c_str(),
1329                          LookupTag(Message,"Message").c_str());
1330          RunScripts("APT::Update::Auth-Failure");
1331          return;
1332       } else {
1333          _error->Warning(_("GPG error: %s: %s"),
1334                          Desc.Description.c_str(),
1335                          LookupTag(Message,"Message").c_str());
1336       }
1337       // gpgv method failed
1338       ReportMirrorFailure("GPGFailure");
1339    }
1340
1341    // No Release file was present, or verification failed, so fall
1342    // back to queueing Packages files without verification
1343    QueueIndexes(false);
1344 }
1345                                                                         /*}}}*/
1346 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1347 // ---------------------------------------------------------------------
1348 /* This just sets up the initial fetch environment and queues the first
1349    possibilitiy */
1350 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1351                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1352                              string &StoreFilename) :
1353                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1354                StoreFilename(StoreFilename), Vf(Version.FileList()),
1355                Trusted(false)
1356 {
1357    Retries = _config->FindI("Acquire::Retries",0);
1358
1359    if (Version.Arch() == 0)
1360    {
1361       _error->Error(_("I wasn't able to locate a file for the %s package. "
1362                       "This might mean you need to manually fix this package. "
1363                       "(due to missing arch)"),
1364                     Version.ParentPkg().Name());
1365       return;
1366    }
1367
1368    /* We need to find a filename to determine the extension. We make the
1369       assumption here that all the available sources for this version share
1370       the same extension.. */
1371    // Skip not source sources, they do not have file fields.
1372    for (; Vf.end() == false; Vf++)
1373    {
1374       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1375          continue;
1376       break;
1377    }
1378
1379    // Does not really matter here.. we are going to fail out below
1380    if (Vf.end() != true)
1381    {
1382       // If this fails to get a file name we will bomb out below.
1383       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1384       if (_error->PendingError() == true)
1385          return;
1386
1387       // Generate the final file name as: package_version_arch.foo
1388       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1389                       QuoteString(Version.VerStr(),"_:") + '_' +
1390                       QuoteString(Version.Arch(),"_:.") +
1391                       "." + flExtension(Parse.FileName());
1392    }
1393
1394    // check if we have one trusted source for the package. if so, switch
1395    // to "TrustedOnly" mode
1396    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; i++)
1397    {
1398       pkgIndexFile *Index;
1399       if (Sources->FindIndex(i.File(),Index) == false)
1400          continue;
1401       if (_config->FindB("Debug::pkgAcquire::Auth", false))
1402       {
1403          std::cerr << "Checking index: " << Index->Describe()
1404                    << "(Trusted=" << Index->IsTrusted() << ")\n";
1405       }
1406       if (Index->IsTrusted()) {
1407          Trusted = true;
1408          break;
1409       }
1410    }
1411
1412    // "allow-unauthenticated" restores apts old fetching behaviour
1413    // that means that e.g. unauthenticated file:// uris are higher
1414    // priority than authenticated http:// uris
1415    if (_config->FindB("APT::Get::AllowUnauthenticated",false) == true)
1416       Trusted = false;
1417
1418    // Select a source
1419    if (QueueNext() == false && _error->PendingError() == false)
1420       _error->Error(_("I wasn't able to locate file for the %s package. "
1421                     "This might mean you need to manually fix this package."),
1422                     Version.ParentPkg().Name());
1423 }
1424                                                                         /*}}}*/
1425 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1426 // ---------------------------------------------------------------------
1427 /* This queues the next available file version for download. It checks if
1428    the archive is already available in the cache and stashs the MD5 for
1429    checking later. */
1430 bool pkgAcqArchive::QueueNext()
1431 {
1432    string const ForceHash = _config->Find("Acquire::ForceHash");
1433    for (; Vf.end() == false; Vf++)
1434    {
1435       // Ignore not source sources
1436       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1437          continue;
1438
1439       // Try to cross match against the source list
1440       pkgIndexFile *Index;
1441       if (Sources->FindIndex(Vf.File(),Index) == false)
1442             continue;
1443
1444       // only try to get a trusted package from another source if that source
1445       // is also trusted
1446       if(Trusted && !Index->IsTrusted())
1447          continue;
1448
1449       // Grab the text package record
1450       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1451       if (_error->PendingError() == true)
1452          return false;
1453
1454       string PkgFile = Parse.FileName();
1455       if (ForceHash.empty() == false)
1456       {
1457          if(stringcasecmp(ForceHash, "sha256") == 0)
1458             ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
1459          else if (stringcasecmp(ForceHash, "sha1") == 0)
1460             ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
1461          else
1462             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1463       }
1464       else
1465       {
1466          string Hash;
1467          if ((Hash = Parse.SHA256Hash()).empty() == false)
1468             ExpectedHash = HashString("SHA256", Hash);
1469          else if ((Hash = Parse.SHA1Hash()).empty() == false)
1470             ExpectedHash = HashString("SHA1", Hash);
1471          else
1472             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1473       }
1474       if (PkgFile.empty() == true)
1475          return _error->Error(_("The package index files are corrupted. No Filename: "
1476                               "field for package %s."),
1477                               Version.ParentPkg().Name());
1478
1479       Desc.URI = Index->ArchiveURI(PkgFile);
1480       Desc.Description = Index->ArchiveInfo(Version);
1481       Desc.Owner = this;
1482       Desc.ShortDesc = Version.ParentPkg().Name();
1483
1484       // See if we already have the file. (Legacy filenames)
1485       FileSize = Version->Size;
1486       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
1487       struct stat Buf;
1488       if (stat(FinalFile.c_str(),&Buf) == 0)
1489       {
1490          // Make sure the size matches
1491          if ((unsigned)Buf.st_size == Version->Size)
1492          {
1493             Complete = true;
1494             Local = true;
1495             Status = StatDone;
1496             StoreFilename = DestFile = FinalFile;
1497             return true;
1498          }
1499
1500          /* Hmm, we have a file and its size does not match, this means it is
1501             an old style mismatched arch */
1502          unlink(FinalFile.c_str());
1503       }
1504
1505       // Check it again using the new style output filenames
1506       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
1507       if (stat(FinalFile.c_str(),&Buf) == 0)
1508       {
1509          // Make sure the size matches
1510          if ((unsigned)Buf.st_size == Version->Size)
1511          {
1512             Complete = true;
1513             Local = true;
1514             Status = StatDone;
1515             StoreFilename = DestFile = FinalFile;
1516             return true;
1517          }
1518
1519          /* Hmm, we have a file and its size does not match, this shouldnt
1520             happen.. */
1521          unlink(FinalFile.c_str());
1522       }
1523
1524       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
1525
1526       // Check the destination file
1527       if (stat(DestFile.c_str(),&Buf) == 0)
1528       {
1529          // Hmm, the partial file is too big, erase it
1530          if ((unsigned)Buf.st_size > Version->Size)
1531             unlink(DestFile.c_str());
1532          else
1533             PartialSize = Buf.st_size;
1534       }
1535
1536       // Create the item
1537       Local = false;
1538       Desc.URI = Index->ArchiveURI(PkgFile);
1539       Desc.Description = Index->ArchiveInfo(Version);
1540       Desc.Owner = this;
1541       Desc.ShortDesc = Version.ParentPkg().Name();
1542       QueueURI(Desc);
1543
1544       Vf++;
1545       return true;
1546    }
1547    return false;
1548 }
1549                                                                         /*}}}*/
1550 // AcqArchive::Done - Finished fetching                                 /*{{{*/
1551 // ---------------------------------------------------------------------
1552 /* */
1553 void pkgAcqArchive::Done(string Message,unsigned long Size,string CalcHash,
1554                          pkgAcquire::MethodConfig *Cfg)
1555 {
1556    Item::Done(Message,Size,CalcHash,Cfg);
1557
1558    // Check the size
1559    if (Size != Version->Size)
1560    {
1561       Status = StatError;
1562       ErrorText = _("Size mismatch");
1563       return;
1564    }
1565
1566    // Check the hash
1567    if(ExpectedHash.toStr() != CalcHash)
1568    {
1569       Status = StatError;
1570       ErrorText = _("Hash Sum mismatch");
1571       if(FileExists(DestFile))
1572          Rename(DestFile,DestFile + ".FAILED");
1573       return;
1574    }
1575
1576    // Grab the output filename
1577    string FileName = LookupTag(Message,"Filename");
1578    if (FileName.empty() == true)
1579    {
1580       Status = StatError;
1581       ErrorText = "Method gave a blank filename";
1582       return;
1583    }
1584
1585    Complete = true;
1586
1587    // Reference filename
1588    if (FileName != DestFile)
1589    {
1590       StoreFilename = DestFile = FileName;
1591       Local = true;
1592       return;
1593    }
1594
1595    // Done, move it into position
1596    string FinalFile = _config->FindDir("Dir::Cache::Archives");
1597    FinalFile += flNotDir(StoreFilename);
1598    Rename(DestFile,FinalFile);
1599
1600    StoreFilename = DestFile = FinalFile;
1601    Complete = true;
1602 }
1603                                                                         /*}}}*/
1604 // AcqArchive::Failed - Failure handler                                 /*{{{*/
1605 // ---------------------------------------------------------------------
1606 /* Here we try other sources */
1607 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1608 {
1609    ErrorText = LookupTag(Message,"Message");
1610
1611    /* We don't really want to retry on failed media swaps, this prevents
1612       that. An interesting observation is that permanent failures are not
1613       recorded. */
1614    if (Cnf->Removable == true &&
1615        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1616    {
1617       // Vf = Version.FileList();
1618       while (Vf.end() == false) Vf++;
1619       StoreFilename = string();
1620       Item::Failed(Message,Cnf);
1621       return;
1622    }
1623
1624    if (QueueNext() == false)
1625    {
1626       // This is the retry counter
1627       if (Retries != 0 &&
1628           Cnf->LocalOnly == false &&
1629           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1630       {
1631          Retries--;
1632          Vf = Version.FileList();
1633          if (QueueNext() == true)
1634             return;
1635       }
1636
1637       StoreFilename = string();
1638       Item::Failed(Message,Cnf);
1639    }
1640 }
1641                                                                         /*}}}*/
1642 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
1643 // ---------------------------------------------------------------------
1644 bool pkgAcqArchive::IsTrusted()
1645 {
1646    return Trusted;
1647 }
1648                                                                         /*}}}*/
1649 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
1650 // ---------------------------------------------------------------------
1651 /* */
1652 void pkgAcqArchive::Finished()
1653 {
1654    if (Status == pkgAcquire::Item::StatDone &&
1655        Complete == true)
1656       return;
1657    StoreFilename = string();
1658 }
1659                                                                         /*}}}*/
1660 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
1661 // ---------------------------------------------------------------------
1662 /* The file is added to the queue */
1663 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
1664                        unsigned long Size,string Dsc,string ShortDesc,
1665                        const string &DestDir, const string &DestFilename,
1666                        bool IsIndexFile) :
1667                        Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile)
1668 {
1669    Retries = _config->FindI("Acquire::Retries",0);
1670
1671    if(!DestFilename.empty())
1672       DestFile = DestFilename;
1673    else if(!DestDir.empty())
1674       DestFile = DestDir + "/" + flNotDir(URI);
1675    else
1676       DestFile = flNotDir(URI);
1677
1678    // Create the item
1679    Desc.URI = URI;
1680    Desc.Description = Dsc;
1681    Desc.Owner = this;
1682
1683    // Set the short description to the archive component
1684    Desc.ShortDesc = ShortDesc;
1685
1686    // Get the transfer sizes
1687    FileSize = Size;
1688    struct stat Buf;
1689    if (stat(DestFile.c_str(),&Buf) == 0)
1690    {
1691       // Hmm, the partial file is too big, erase it
1692       if ((unsigned)Buf.st_size > Size)
1693          unlink(DestFile.c_str());
1694       else
1695          PartialSize = Buf.st_size;
1696    }
1697
1698    QueueURI(Desc);
1699 }
1700                                                                         /*}}}*/
1701 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
1702 // ---------------------------------------------------------------------
1703 /* */
1704 void pkgAcqFile::Done(string Message,unsigned long Size,string CalcHash,
1705                       pkgAcquire::MethodConfig *Cnf)
1706 {
1707    Item::Done(Message,Size,CalcHash,Cnf);
1708
1709    // Check the hash
1710    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
1711    {
1712       Status = StatError;
1713       ErrorText = _("Hash Sum mismatch");
1714       Rename(DestFile,DestFile + ".FAILED");
1715       return;
1716    }
1717
1718    string FileName = LookupTag(Message,"Filename");
1719    if (FileName.empty() == true)
1720    {
1721       Status = StatError;
1722       ErrorText = "Method gave a blank filename";
1723       return;
1724    }
1725
1726    Complete = true;
1727
1728    // The files timestamp matches
1729    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1730       return;
1731
1732    // We have to copy it into place
1733    if (FileName != DestFile)
1734    {
1735       Local = true;
1736       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
1737           Cnf->Removable == true)
1738       {
1739          Desc.URI = "copy:" + FileName;
1740          QueueURI(Desc);
1741          return;
1742       }
1743
1744       // Erase the file if it is a symlink so we can overwrite it
1745       struct stat St;
1746       if (lstat(DestFile.c_str(),&St) == 0)
1747       {
1748          if (S_ISLNK(St.st_mode) != 0)
1749             unlink(DestFile.c_str());
1750       }
1751
1752       // Symlink the file
1753       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
1754       {
1755          ErrorText = "Link to " + DestFile + " failure ";
1756          Status = StatError;
1757          Complete = false;
1758       }
1759    }
1760 }
1761                                                                         /*}}}*/
1762 // AcqFile::Failed - Failure handler                                    /*{{{*/
1763 // ---------------------------------------------------------------------
1764 /* Here we try other sources */
1765 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1766 {
1767    ErrorText = LookupTag(Message,"Message");
1768
1769    // This is the retry counter
1770    if (Retries != 0 &&
1771        Cnf->LocalOnly == false &&
1772        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1773    {
1774       Retries--;
1775       QueueURI(Desc);
1776       return;
1777    }
1778
1779    Item::Failed(Message,Cnf);
1780 }
1781                                                                         /*}}}*/
1782 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
1783 // ---------------------------------------------------------------------
1784 /* The only header we use is the last-modified header. */
1785 string pkgAcqFile::Custom600Headers()
1786 {
1787    if (IsIndexFile)
1788       return "\nIndex-File: true";
1789    return "";
1790 }
1791                                                                         /*}}}*/