apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <apt-pkg/acquire-item.h>
  17 #include <apt-pkg/configuration.h>
  18 #include <apt-pkg/aptconfiguration.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/vendorlist.h>
  21 #include <apt-pkg/error.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <apt-pkg/fileutl.h>
  24 #include <apt-pkg/md5.h>
  25 #include <apt-pkg/sha1.h>
  26 #include <apt-pkg/tagfile.h>
  27
  28 #include <apti18n.h>
  29
  30 #include <sys/stat.h>
  31 #include <unistd.h>
  32 #include <errno.h>
  33 #include <string>
  34 #include <sstream>
  35 #include <stdio.h>
  36 #include <ctime>
  37                                                                         /*}}}*/
  38
  39 using namespace std;
  40
  41 // Acquire::Item::Item - Constructor                                    /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* */
  44 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  45                        PartialSize(0), Mode(0), ID(0), Complete(false),
  46                        Local(false), QueueCounter(0)
  47 {
  48    Owner->Add(this);
  49    Status = StatIdle;
  50 }
  51                                                                         /*}}}*/
  52 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  53 // ---------------------------------------------------------------------
  54 /* */
  55 pkgAcquire::Item::~Item()
  56 {
  57    Owner->Remove(this);
  58 }
  59                                                                         /*}}}*/
  60 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  61 // ---------------------------------------------------------------------
  62 /* We return to an idle state if there are still other queues that could
  63    fetch this object */
  64 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  65 {
  66    Status = StatIdle;
  67    ErrorText = LookupTag(Message,"Message");
  68    UsedMirror =  LookupTag(Message,"UsedMirror");
  69    if (QueueCounter <= 1)
  70    {
  71       /* This indicates that the file is not available right now but might
  72          be sometime later. If we do a retry cycle then this should be
  73          retried [CDROMs] */
  74       if (Cnf->LocalOnly == true &&
  75           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  76       {
  77          Status = StatIdle;
  78          Dequeue();
  79          return;
  80       }
  81
  82       Status = StatError;
  83       Dequeue();
  84    }
  85
  86    // report mirror failure back to LP if we actually use a mirror
  87    string FailReason = LookupTag(Message, "FailReason");
  88    if(FailReason.size() != 0)
  89       ReportMirrorFailure(FailReason);
  90    else
  91       ReportMirrorFailure(ErrorText);
  92 }
  93                                                                         /*}}}*/
  94 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
  95 // ---------------------------------------------------------------------
  96 /* Stash status and the file size. Note that setting Complete means
  97    sub-phases of the acquire process such as decompresion are operating */
  98 void pkgAcquire::Item::Start(string /*Message*/,unsigned long Size)
  99 {
 100    Status = StatFetching;
 101    if (FileSize == 0 && Complete == false)
 102       FileSize = Size;
 103 }
 104                                                                         /*}}}*/
 105 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
 106 // ---------------------------------------------------------------------
 107 /* */
 108 void pkgAcquire::Item::Done(string Message,unsigned long Size,string Hash,
 109                             pkgAcquire::MethodConfig *Cnf)
 110 {
 111    // We just downloaded something..
 112    string FileName = LookupTag(Message,"Filename");
 113    UsedMirror =  LookupTag(Message,"UsedMirror");
 114    if (Complete == false && !Local && FileName == DestFile)
 115    {
 116       if (Owner->Log != 0)
 117          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 118    }
 119
 120    if (FileSize == 0)
 121       FileSize= Size;
 122    Status = StatDone;
 123    ErrorText = string();
 124    Owner->Dequeue(this);
 125 }
 126                                                                         /*}}}*/
 127 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 128 // ---------------------------------------------------------------------
 129 /* This helper function is used by alot of item methods as thier final
 130    step */
 131 void pkgAcquire::Item::Rename(string From,string To)
 132 {
 133    if (rename(From.c_str(),To.c_str()) != 0)
 134    {
 135       char S[300];
 136       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 137               From.c_str(),To.c_str());
 138       Status = StatError;
 139       ErrorText = S;
 140    }
 141 }
 142                                                                         /*}}}*/
 143 // Acquire::Item::ReportMirrorFailure                                   /*{{{*/
 144 // ---------------------------------------------------------------------
 145 void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
 146 {
 147    // we only act if a mirror was used at all
 148    if(UsedMirror.empty())
 149       return;
 150 #if 0
 151    std::cerr << "\nReportMirrorFailure: "
 152              << UsedMirror
 153              << " Uri: " << DescURI()
 154              << " FailCode: "
 155              << FailCode << std::endl;
 156 #endif
 157    const char *Args[40];
 158    unsigned int i = 0;
 159    string report = _config->Find("Methods::Mirror::ProblemReporting",
 160                                  "/usr/lib/apt/apt-report-mirror-failure");
 161    if(!FileExists(report))
 162       return;
 163    Args[i++] = report.c_str();
 164    Args[i++] = UsedMirror.c_str();
 165    Args[i++] = DescURI().c_str();
 166    Args[i++] = FailCode.c_str();
 167    Args[i++] = NULL;
 168    pid_t pid = ExecFork();
 169    if(pid < 0)
 170    {
 171       _error->Error("ReportMirrorFailure Fork failed");
 172       return;
 173    }
 174    else if(pid == 0)
 175    {
 176       execvp(Args[0], (char**)Args);
 177       std::cerr << "Could not exec " << Args[0] << std::endl;
 178       _exit(100);
 179    }
 180    if(!ExecWait(pid, "report-mirror-failure"))
 181    {
 182       _error->Warning("Couldn't report problem to '%s'",
 183                       _config->Find("Methods::Mirror::ProblemReporting").c_str());
 184    }
 185 }
 186                                                                         /*}}}*/
 187 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 188 // ---------------------------------------------------------------------
 189 /* Get the DiffIndex file first and see if there are patches availabe
 190  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 191  * patches. If anything goes wrong in that process, it will fall back to
 192  * the original packages file
 193  */
 194 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 195                                  string URI,string URIDesc,string ShortDesc,
 196                                  HashString ExpectedHash)
 197    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 198      Description(URIDesc)
 199 {
 200
 201    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 202
 203    Desc.Description = URIDesc + "/DiffIndex";
 204    Desc.Owner = this;
 205    Desc.ShortDesc = ShortDesc;
 206    Desc.URI = URI + ".diff/Index";
 207
 208    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 209    DestFile += URItoFileName(URI) + string(".DiffIndex");
 210
 211    if(Debug)
 212       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 213
 214    // look for the current package file
 215    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 216    CurrentPackagesFile += URItoFileName(RealURI);
 217
 218    // FIXME: this file:/ check is a hack to prevent fetching
 219    //        from local sources. this is really silly, and
 220    //        should be fixed cleanly as soon as possible
 221    if(!FileExists(CurrentPackagesFile) ||
 222       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 223    {
 224       // we don't have a pkg file or we don't want to queue
 225       if(Debug)
 226          std::clog << "No index file, local or canceld by user" << std::endl;
 227       Failed("", NULL);
 228       return;
 229    }
 230
 231    if(Debug)
 232       std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): "
 233                 << CurrentPackagesFile << std::endl;
 234
 235    QueueURI(Desc);
 236
 237 }
 238                                                                         /*}}}*/
 239 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 240 // ---------------------------------------------------------------------
 241 /* The only header we use is the last-modified header. */
 242 string pkgAcqDiffIndex::Custom600Headers()
 243 {
 244    string Final = _config->FindDir("Dir::State::lists");
 245    Final += URItoFileName(RealURI) + string(".IndexDiff");
 246
 247    if(Debug)
 248       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 249
 250    struct stat Buf;
 251    if (stat(Final.c_str(),&Buf) != 0)
 252       return "\nIndex-File: true";
 253
 254    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 255 }
 256                                                                         /*}}}*/
 257 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 258 {
 259    if(Debug)
 260       std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile
 261                 << std::endl;
 262
 263    pkgTagSection Tags;
 264    string ServerSha1;
 265    vector<DiffInfo> available_patches;
 266
 267    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 268    pkgTagFile TF(&Fd);
 269    if (_error->PendingError() == true)
 270       return false;
 271
 272    if(TF.Step(Tags) == true)
 273    {
 274       bool found = false;
 275       DiffInfo d;
 276       string size;
 277
 278       string const tmp = Tags.FindS("SHA1-Current");
 279       std::stringstream ss(tmp);
 280       ss >> ServerSha1 >> size;
 281       unsigned long const ServerSize = atol(size.c_str());
 282
 283       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 284       SHA1Summation SHA1;
 285       SHA1.AddFD(fd.Fd(), fd.Size());
 286       string const local_sha1 = SHA1.Result();
 287
 288       if(local_sha1 == ServerSha1)
 289       {
 290          // we have the same sha1 as the server
 291          if(Debug)
 292             std::clog << "Package file is up-to-date" << std::endl;
 293          // set found to true, this will queue a pkgAcqIndexDiffs with
 294          // a empty availabe_patches
 295          found = true;
 296       }
 297       else
 298       {
 299          if(Debug)
 300             std::clog << "SHA1-Current: " << ServerSha1 << std::endl;
 301
 302          // check the historie and see what patches we need
 303          string const history = Tags.FindS("SHA1-History");
 304          std::stringstream hist(history);
 305          while(hist >> d.sha1 >> size >> d.file)
 306          {
 307             // read until the first match is found
 308             // from that point on, we probably need all diffs
 309             if(d.sha1 == local_sha1)
 310                found=true;
 311             else if (found == false)
 312                continue;
 313
 314             if(Debug)
 315                std::clog << "Need to get diff: " << d.file << std::endl;
 316             available_patches.push_back(d);
 317          }
 318
 319          if (available_patches.empty() == false)
 320          {
 321             // patching with too many files is rather slow compared to a fast download
 322             unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 323             if (fileLimit != 0 && fileLimit < available_patches.size())
 324             {
 325                if (Debug)
 326                   std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 327                         << ") so fallback to complete download" << std::endl;
 328                return false;
 329             }
 330
 331             // see if the patches are too big
 332             found = false; // it was true and it will be true again at the end
 333             d = *available_patches.begin();
 334             string const firstPatch = d.file;
 335             unsigned long patchesSize = 0;
 336             std::stringstream patches(Tags.FindS("SHA1-Patches"));
 337             while(patches >> d.sha1 >> size >> d.file)
 338             {
 339                if (firstPatch == d.file)
 340                   found = true;
 341                else if (found == false)
 342                   continue;
 343
 344                patchesSize += atol(size.c_str());
 345             }
 346             unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 347             if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 348             {
 349                if (Debug)
 350                   std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 351                         << ") so fallback to complete download" << std::endl;
 352                return false;
 353             }
 354          }
 355       }
 356
 357       // we have something, queue the next diff
 358       if(found)
 359       {
 360          // queue the diffs
 361          string::size_type const last_space = Description.rfind(" ");
 362          if(last_space != string::npos)
 363             Description.erase(last_space, Description.size()-last_space);
 364          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 365                               ExpectedHash, ServerSha1, available_patches);
 366          Complete = false;
 367          Status = StatDone;
 368          Dequeue();
 369          return true;
 370       }
 371    }
 372
 373    // Nothing found, report and return false
 374    // Failing here is ok, if we return false later, the full
 375    // IndexFile is queued
 376    if(Debug)
 377       std::clog << "Can't find a patch in the index file" << std::endl;
 378    return false;
 379 }
 380                                                                         /*}}}*/
 381 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)      /*{{{*/
 382 {
 383    if(Debug)
 384       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << std::endl
 385                 << "Falling back to normal index file aquire" << std::endl;
 386
 387    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 388                    ExpectedHash);
 389
 390    Complete = false;
 391    Status = StatDone;
 392    Dequeue();
 393 }
 394                                                                         /*}}}*/
 395 void pkgAcqDiffIndex::Done(string Message,unsigned long Size,string Md5Hash,    /*{{{*/
 396                            pkgAcquire::MethodConfig *Cnf)
 397 {
 398    if(Debug)
 399       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 400
 401    Item::Done(Message,Size,Md5Hash,Cnf);
 402
 403    string FinalFile;
 404    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 405
 406    // sucess in downloading the index
 407    // rename the index
 408    FinalFile += string(".IndexDiff");
 409    if(Debug)
 410       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 411                 << std::endl;
 412    Rename(DestFile,FinalFile);
 413    chmod(FinalFile.c_str(),0644);
 414    DestFile = FinalFile;
 415
 416    if(!ParseDiffIndex(DestFile))
 417       return Failed("", NULL);
 418
 419    Complete = true;
 420    Status = StatDone;
 421    Dequeue();
 422    return;
 423 }
 424                                                                         /*}}}*/
 425 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 426 // ---------------------------------------------------------------------
 427 /* The package diff is added to the queue. one object is constructed
 428  * for each diff and the index
 429  */
 430 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 431                                    string URI,string URIDesc,string ShortDesc,
 432                                    HashString ExpectedHash,
 433                                    string ServerSha1,
 434                                    vector<DiffInfo> diffs)
 435    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 436      available_patches(diffs), ServerSha1(ServerSha1)
 437 {
 438
 439    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 440    DestFile += URItoFileName(URI);
 441
 442    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 443
 444    Description = URIDesc;
 445    Desc.Owner = this;
 446    Desc.ShortDesc = ShortDesc;
 447
 448    if(available_patches.size() == 0)
 449    {
 450       // we are done (yeah!)
 451       Finish(true);
 452    }
 453    else
 454    {
 455       // get the next diff
 456       State = StateFetchDiff;
 457       QueueNextDiff();
 458    }
 459 }
 460                                                                         /*}}}*/
 461 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)     /*{{{*/
 462 {
 463    if(Debug)
 464       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl
 465                 << "Falling back to normal index file aquire" << std::endl;
 466    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 467                    ExpectedHash);
 468    Finish();
 469 }
 470                                                                         /*}}}*/
 471 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 472 void pkgAcqIndexDiffs::Finish(bool allDone)
 473 {
 474    // we restore the original name, this is required, otherwise
 475    // the file will be cleaned
 476    if(allDone)
 477    {
 478       DestFile = _config->FindDir("Dir::State::lists");
 479       DestFile += URItoFileName(RealURI);
 480
 481       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 482       {
 483          Status = StatAuthError;
 484          ErrorText = _("MD5Sum mismatch");
 485          Rename(DestFile,DestFile + ".FAILED");
 486          Dequeue();
 487          return;
 488       }
 489
 490       // this is for the "real" finish
 491       Complete = true;
 492       Status = StatDone;
 493       Dequeue();
 494       if(Debug)
 495          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 496       return;
 497    }
 498
 499    if(Debug)
 500       std::clog << "Finishing: " << Desc.URI << std::endl;
 501    Complete = false;
 502    Status = StatDone;
 503    Dequeue();
 504    return;
 505 }
 506                                                                         /*}}}*/
 507 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 508 {
 509
 510    // calc sha1 of the just patched file
 511    string FinalFile = _config->FindDir("Dir::State::lists");
 512    FinalFile += URItoFileName(RealURI);
 513
 514    FileFd fd(FinalFile, FileFd::ReadOnly);
 515    SHA1Summation SHA1;
 516    SHA1.AddFD(fd.Fd(), fd.Size());
 517    string local_sha1 = string(SHA1.Result());
 518    if(Debug)
 519       std::clog << "QueueNextDiff: "
 520                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 521
 522    // final file reached before all patches are applied
 523    if(local_sha1 == ServerSha1)
 524    {
 525       Finish(true);
 526       return true;
 527    }
 528
 529    // remove all patches until the next matching patch is found
 530    // this requires the Index file to be ordered
 531    for(vector<DiffInfo>::iterator I=available_patches.begin();
 532        available_patches.size() > 0 &&
 533           I != available_patches.end() &&
 534           (*I).sha1 != local_sha1;
 535        I++)
 536    {
 537       available_patches.erase(I);
 538    }
 539
 540    // error checking and falling back if no patch was found
 541    if(available_patches.size() == 0)
 542    {
 543       Failed("", NULL);
 544       return false;
 545    }
 546
 547    // queue the right diff
 548    Desc.URI = string(RealURI) + ".diff/" + available_patches[0].file + ".gz";
 549    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 550    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 551    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 552
 553    if(Debug)
 554       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 555
 556    QueueURI(Desc);
 557
 558    return true;
 559 }
 560                                                                         /*}}}*/
 561 void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash,   /*{{{*/
 562                             pkgAcquire::MethodConfig *Cnf)
 563 {
 564    if(Debug)
 565       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 566
 567    Item::Done(Message,Size,Md5Hash,Cnf);
 568
 569    string FinalFile;
 570    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 571
 572    // sucess in downloading a diff, enter ApplyDiff state
 573    if(State == StateFetchDiff)
 574    {
 575
 576       if(Debug)
 577          std::clog << "Sending to gzip method: " << FinalFile << std::endl;
 578
 579       string FileName = LookupTag(Message,"Filename");
 580       State = StateUnzipDiff;
 581       Local = true;
 582       Desc.URI = "gzip:" + FileName;
 583       DestFile += ".decomp";
 584       QueueURI(Desc);
 585       Mode = "gzip";
 586       return;
 587    }
 588
 589    // sucess in downloading a diff, enter ApplyDiff state
 590    if(State == StateUnzipDiff)
 591    {
 592
 593       // rred excepts the patch as $FinalFile.ed
 594       Rename(DestFile,FinalFile+".ed");
 595
 596       if(Debug)
 597          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 598
 599       State = StateApplyDiff;
 600       Local = true;
 601       Desc.URI = "rred:" + FinalFile;
 602       QueueURI(Desc);
 603       Mode = "rred";
 604       return;
 605    }
 606
 607
 608    // success in download/apply a diff, queue next (if needed)
 609    if(State == StateApplyDiff)
 610    {
 611       // remove the just applied patch
 612       available_patches.erase(available_patches.begin());
 613
 614       // move into place
 615       if(Debug)
 616       {
 617          std::clog << "Moving patched file in place: " << std::endl
 618                    << DestFile << " -> " << FinalFile << std::endl;
 619       }
 620       Rename(DestFile,FinalFile);
 621       chmod(FinalFile.c_str(),0644);
 622
 623       // see if there is more to download
 624       if(available_patches.size() > 0) {
 625          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 626                               ExpectedHash, ServerSha1, available_patches);
 627          return Finish();
 628       } else
 629          return Finish(true);
 630    }
 631 }
 632                                                                         /*}}}*/
 633 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 634 // ---------------------------------------------------------------------
 635 /* The package file is added to the queue and a second class is
 636    instantiated to fetch the revision file */
 637 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 638                          string URI,string URIDesc,string ShortDesc,
 639                          HashString ExpectedHash, string comprExt)
 640    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 641 {
 642    Decompression = false;
 643    Erase = false;
 644
 645    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 646    DestFile += URItoFileName(URI);
 647
 648    if(comprExt.empty())
 649    {
 650       // autoselect the compression method
 651       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 652       if (types.empty() == true)
 653          comprExt = "plain";
 654       else
 655          comprExt = "." + types[0];
 656    }
 657    CompressionExtension = ((comprExt == "plain" || comprExt == ".") ? "" : comprExt);
 658
 659    Desc.URI = URI + CompressionExtension;
 660
 661    Desc.Description = URIDesc;
 662    Desc.Owner = this;
 663    Desc.ShortDesc = ShortDesc;
 664
 665    QueueURI(Desc);
 666 }
 667                                                                         /*}}}*/
 668 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 669 // ---------------------------------------------------------------------
 670 /* The only header we use is the last-modified header. */
 671 string pkgAcqIndex::Custom600Headers()
 672 {
 673    string Final = _config->FindDir("Dir::State::lists");
 674    Final += URItoFileName(RealURI);
 675
 676    struct stat Buf;
 677    if (stat(Final.c_str(),&Buf) != 0)
 678       return "\nIndex-File: true";
 679    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 680 }
 681                                                                         /*}}}*/
 682 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
 683 {
 684    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 685
 686    for (std::vector<std::string>::const_iterator t = types.begin();
 687         t != types.end(); t++)
 688    {
 689       // jump over all already tried compression types
 690       const unsigned int nameLen = Desc.URI.size() - (*t).size();
 691       if(Desc.URI.substr(nameLen) != *t)
 692          continue;
 693
 694       // we want to try it with the next extension (and make sure to
 695       // not skip over the end)
 696       t++;
 697       if (t == types.end())
 698          break;
 699
 700       // queue new download
 701       Desc.URI = Desc.URI.substr(0, nameLen) + *t;
 702       new pkgAcqIndex(Owner, RealURI, Desc.Description, Desc.ShortDesc,
 703       ExpectedHash, string(".").append(*t));
 704
 705       Status = StatDone;
 706       Complete = false;
 707       Dequeue();
 708       return;
 709    }
 710
 711    // on decompression failure, remove bad versions in partial/
 712    if(Decompression && Erase) {
 713       string s = _config->FindDir("Dir::State::lists") + "partial/";
 714       s += URItoFileName(RealURI);
 715       unlink(s.c_str());
 716    }
 717
 718    Item::Failed(Message,Cnf);
 719 }
 720                                                                         /*}}}*/
 721 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
 722 // ---------------------------------------------------------------------
 723 /* This goes through a number of states.. On the initial fetch the
 724    method could possibly return an alternate filename which points
 725    to the uncompressed version of the file. If this is so the file
 726    is copied into the partial directory. In all other cases the file
 727    is decompressed with a gzip uri. */
 728 void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash,
 729                        pkgAcquire::MethodConfig *Cfg)
 730 {
 731    Item::Done(Message,Size,Hash,Cfg);
 732
 733    if (Decompression == true)
 734    {
 735       if (_config->FindB("Debug::pkgAcquire::Auth", false))
 736       {
 737          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
 738          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
 739       }
 740
 741       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
 742       {
 743          Status = StatAuthError;
 744          ErrorText = _("Hash Sum mismatch");
 745          Rename(DestFile,DestFile + ".FAILED");
 746          ReportMirrorFailure("HashChecksumFailure");
 747          return;
 748       }
 749       // Done, move it into position
 750       string FinalFile = _config->FindDir("Dir::State::lists");
 751       FinalFile += URItoFileName(RealURI);
 752       Rename(DestFile,FinalFile);
 753       chmod(FinalFile.c_str(),0644);
 754
 755       /* We restore the original name to DestFile so that the clean operation
 756          will work OK */
 757       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 758       DestFile += URItoFileName(RealURI);
 759
 760       // Remove the compressed version.
 761       if (Erase == true)
 762          unlink(DestFile.c_str());
 763       return;
 764    }
 765
 766    Erase = false;
 767    Complete = true;
 768
 769    // Handle the unzipd case
 770    string FileName = LookupTag(Message,"Alt-Filename");
 771    if (FileName.empty() == false)
 772    {
 773       // The files timestamp matches
 774       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
 775          return;
 776       Decompression = true;
 777       Local = true;
 778       DestFile += ".decomp";
 779       Desc.URI = "copy:" + FileName;
 780       QueueURI(Desc);
 781       Mode = "copy";
 782       return;
 783    }
 784
 785    FileName = LookupTag(Message,"Filename");
 786    if (FileName.empty() == true)
 787    {
 788       Status = StatError;
 789       ErrorText = "Method gave a blank filename";
 790    }
 791
 792    // The files timestamp matches
 793    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 794       return;
 795
 796    if (FileName == DestFile)
 797       Erase = true;
 798    else
 799       Local = true;
 800
 801    string compExt = flExtension(flNotDir(URI(Desc.URI).Path));
 802    string decompProg;
 803
 804    // get the binary name for your used compression type
 805    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
 806    if(decompProg.empty() == false);
 807    // flExtensions returns the full name if no extension is found
 808    // this is why we have this complicated compare operation here
 809    // FIMXE: add a new flJustExtension() that return "" if no
 810    //        extension is found and use that above so that it can
 811    //        be tested against ""
 812    else if(compExt == flNotDir(URI(Desc.URI).Path))
 813       decompProg = "copy";
 814    else {
 815       _error->Error("Unsupported extension: %s", compExt.c_str());
 816       return;
 817    }
 818
 819    Decompression = true;
 820    DestFile += ".decomp";
 821    Desc.URI = decompProg + ":" + FileName;
 822    QueueURI(Desc);
 823    Mode = decompProg.c_str();
 824 }
 825                                                                         /*}}}*/
 826 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
 827 // ---------------------------------------------------------------------
 828 /* The Translation file is added to the queue */
 829 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
 830                             string URI,string URIDesc,string ShortDesc)
 831   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
 832 {
 833 }
 834                                                                         /*}}}*/
 835 // AcqIndexTrans::Custom600Headers - Insert custom request headers      /*{{{*/
 836 // ---------------------------------------------------------------------
 837 string pkgAcqIndexTrans::Custom600Headers()
 838 {
 839    string Final = _config->FindDir("Dir::State::lists");
 840    Final += URItoFileName(RealURI);
 841
 842    struct stat Buf;
 843    if (stat(Final.c_str(),&Buf) != 0)
 844       return "\nFail-Ignore: true";
 845    return "\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 846 }
 847                                                                         /*}}}*/
 848 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
 849 // ---------------------------------------------------------------------
 850 /* */
 851 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 852 {
 853    if (Cnf->LocalOnly == true ||
 854        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 855    {
 856       // Ignore this
 857       Status = StatDone;
 858       Complete = false;
 859       Dequeue();
 860       return;
 861    }
 862
 863    Item::Failed(Message,Cnf);
 864 }
 865                                                                         /*}}}*/
 866 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
 867                              string URI,string URIDesc,string ShortDesc,
 868                              string MetaIndexURI, string MetaIndexURIDesc,
 869                              string MetaIndexShortDesc,
 870                              const vector<IndexTarget*>* IndexTargets,
 871                              indexRecords* MetaIndexParser) :
 872    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
 873    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
 874    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
 875 {
 876    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 877    DestFile += URItoFileName(URI);
 878
 879    // remove any partial downloaded sig-file in partial/.
 880    // it may confuse proxies and is too small to warrant a
 881    // partial download anyway
 882    unlink(DestFile.c_str());
 883
 884    // Create the item
 885    Desc.Description = URIDesc;
 886    Desc.Owner = this;
 887    Desc.ShortDesc = ShortDesc;
 888    Desc.URI = URI;
 889
 890    string Final = _config->FindDir("Dir::State::lists");
 891    Final += URItoFileName(RealURI);
 892    struct stat Buf;
 893    if (stat(Final.c_str(),&Buf) == 0)
 894    {
 895       // File was already in place.  It needs to be re-downloaded/verified
 896       // because Release might have changed, we do give it a differnt
 897       // name than DestFile because otherwise the http method will
 898       // send If-Range requests and there are too many broken servers
 899       // out there that do not understand them
 900       LastGoodSig = DestFile+".reverify";
 901       Rename(Final,LastGoodSig);
 902    }
 903
 904    QueueURI(Desc);
 905 }
 906                                                                         /*}}}*/
 907 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
 908 // ---------------------------------------------------------------------
 909 /* The only header we use is the last-modified header. */
 910 string pkgAcqMetaSig::Custom600Headers()
 911 {
 912    struct stat Buf;
 913    if (stat(LastGoodSig.c_str(),&Buf) != 0)
 914       return "\nIndex-File: true";
 915
 916    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 917 }
 918
 919 void pkgAcqMetaSig::Done(string Message,unsigned long Size,string MD5,
 920                          pkgAcquire::MethodConfig *Cfg)
 921 {
 922    Item::Done(Message,Size,MD5,Cfg);
 923
 924    string FileName = LookupTag(Message,"Filename");
 925    if (FileName.empty() == true)
 926    {
 927       Status = StatError;
 928       ErrorText = "Method gave a blank filename";
 929       return;
 930    }
 931
 932    if (FileName != DestFile)
 933    {
 934       // We have to copy it into place
 935       Local = true;
 936       Desc.URI = "copy:" + FileName;
 937       QueueURI(Desc);
 938       return;
 939    }
 940
 941    Complete = true;
 942
 943    // put the last known good file back on i-m-s hit (it will
 944    // be re-verified again)
 945    // Else do nothing, we have the new file in DestFile then
 946    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 947       Rename(LastGoodSig, DestFile);
 948
 949    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
 950    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc,
 951                        MetaIndexShortDesc,  DestFile, IndexTargets,
 952                        MetaIndexParser);
 953
 954 }
 955                                                                         /*}}}*/
 956 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
 957 {
 958    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 959
 960    // if we get a network error we fail gracefully
 961    if(Status == StatTransientNetworkError)
 962    {
 963       Item::Failed(Message,Cnf);
 964       // move the sigfile back on transient network failures
 965       if(FileExists(LastGoodSig))
 966          Rename(LastGoodSig,Final);
 967
 968       // set the status back to , Item::Failed likes to reset it
 969       Status = pkgAcquire::Item::StatTransientNetworkError;
 970       return;
 971    }
 972
 973    // Delete any existing sigfile when the acquire failed
 974    unlink(Final.c_str());
 975
 976    // queue a pkgAcqMetaIndex with no sigfile
 977    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
 978                        "", IndexTargets, MetaIndexParser);
 979
 980    if (Cnf->LocalOnly == true ||
 981        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 982    {
 983       // Ignore this
 984       Status = StatDone;
 985       Complete = false;
 986       Dequeue();
 987       return;
 988    }
 989
 990    Item::Failed(Message,Cnf);
 991 }
 992                                                                         /*}}}*/
 993 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
 994                                  string URI,string URIDesc,string ShortDesc,
 995                                  string SigFile,
 996                                  const vector<struct IndexTarget*>* IndexTargets,
 997                                  indexRecords* MetaIndexParser) :
 998    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
 999    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
1000 {
1001    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1002    DestFile += URItoFileName(URI);
1003
1004    // Create the item
1005    Desc.Description = URIDesc;
1006    Desc.Owner = this;
1007    Desc.ShortDesc = ShortDesc;
1008    Desc.URI = URI;
1009
1010    QueueURI(Desc);
1011 }
1012                                                                         /*}}}*/
1013 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
1014 // ---------------------------------------------------------------------
1015 /* The only header we use is the last-modified header. */
1016 string pkgAcqMetaIndex::Custom600Headers()
1017 {
1018    string Final = _config->FindDir("Dir::State::lists");
1019    Final += URItoFileName(RealURI);
1020
1021    struct stat Buf;
1022    if (stat(Final.c_str(),&Buf) != 0)
1023       return "\nIndex-File: true";
1024
1025    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1026 }
1027                                                                         /*}}}*/
1028 void pkgAcqMetaIndex::Done(string Message,unsigned long Size,string Hash,       /*{{{*/
1029                            pkgAcquire::MethodConfig *Cfg)
1030 {
1031    Item::Done(Message,Size,Hash,Cfg);
1032
1033    // MetaIndexes are done in two passes: one to download the
1034    // metaindex with an appropriate method, and a second to verify it
1035    // with the gpgv method
1036
1037    if (AuthPass == true)
1038    {
1039       AuthDone(Message);
1040
1041       // all cool, move Release file into place
1042       Complete = true;
1043
1044       string FinalFile = _config->FindDir("Dir::State::lists");
1045       FinalFile += URItoFileName(RealURI);
1046       Rename(DestFile,FinalFile);
1047       chmod(FinalFile.c_str(),0644);
1048       DestFile = FinalFile;
1049    }
1050    else
1051    {
1052       RetrievalDone(Message);
1053       if (!Complete)
1054          // Still more retrieving to do
1055          return;
1056
1057       if (SigFile == "")
1058       {
1059          // There was no signature file, so we are finished.  Download
1060          // the indexes without verification.
1061          QueueIndexes(false);
1062       }
1063       else
1064       {
1065          // There was a signature file, so pass it to gpgv for
1066          // verification
1067
1068          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1069             std::cerr << "Metaindex acquired, queueing gpg verification ("
1070                       << SigFile << "," << DestFile << ")\n";
1071          AuthPass = true;
1072          Desc.URI = "gpgv:" + SigFile;
1073          QueueURI(Desc);
1074          Mode = "gpgv";
1075       }
1076    }
1077 }
1078                                                                         /*}}}*/
1079 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
1080 {
1081    // We have just finished downloading a Release file (it is not
1082    // verified yet)
1083
1084    string FileName = LookupTag(Message,"Filename");
1085    if (FileName.empty() == true)
1086    {
1087       Status = StatError;
1088       ErrorText = "Method gave a blank filename";
1089       return;
1090    }
1091
1092    if (FileName != DestFile)
1093    {
1094       Local = true;
1095       Desc.URI = "copy:" + FileName;
1096       QueueURI(Desc);
1097       return;
1098    }
1099
1100    // make sure to verify against the right file on I-M-S hit
1101    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1102    if(IMSHit)
1103    {
1104       string FinalFile = _config->FindDir("Dir::State::lists");
1105       FinalFile += URItoFileName(RealURI);
1106       DestFile = FinalFile;
1107    }
1108    Complete = true;
1109 }
1110                                                                         /*}}}*/
1111 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1112 {
1113    // At this point, the gpgv method has succeeded, so there is a
1114    // valid signature from a key in the trusted keyring.  We
1115    // perform additional verification of its contents, and use them
1116    // to verify the indexes we are about to download
1117
1118    if (!MetaIndexParser->Load(DestFile))
1119    {
1120       Status = StatAuthError;
1121       ErrorText = MetaIndexParser->ErrorText;
1122       return;
1123    }
1124
1125    if (!VerifyVendor(Message))
1126    {
1127       return;
1128    }
1129
1130    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1131       std::cerr << "Signature verification succeeded: "
1132                 << DestFile << std::endl;
1133
1134    // Download further indexes with verification
1135    QueueIndexes(true);
1136
1137    // Done, move signature file into position
1138    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1139       URItoFileName(RealURI) + ".gpg";
1140    Rename(SigFile,VerifiedSigFile);
1141    chmod(VerifiedSigFile.c_str(),0644);
1142 }
1143                                                                         /*}}}*/
1144 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1145 {
1146    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1147         Target != IndexTargets->end();
1148         Target++)
1149    {
1150       HashString ExpectedIndexHash;
1151       if (verify)
1152       {
1153          const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1154          if (!Record)
1155          {
1156             Status = StatAuthError;
1157             ErrorText = "Unable to find expected entry  "
1158                + (*Target)->MetaKey + " in Meta-index file (malformed Release file?)";
1159             return;
1160          }
1161          ExpectedIndexHash = Record->Hash;
1162          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1163          {
1164             std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1165             std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1166          }
1167          if (ExpectedIndexHash.empty())
1168          {
1169             Status = StatAuthError;
1170             ErrorText = "Unable to find hash sum for "
1171                + (*Target)->MetaKey + " in Meta-index file";
1172             return;
1173          }
1174       }
1175
1176       /* Queue Packages file (either diff or full packages files, depending
1177          on the users option) - we also check if the PDiff Index file is listed
1178          in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
1179          instead, but passing the required info to it is to much hassle */
1180       if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
1181           MetaIndexParser->Exists(string((*Target)->MetaKey).append(".diff/Index")) == true))
1182          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1183                              (*Target)->ShortDesc, ExpectedIndexHash);
1184       else
1185          new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description,
1186                             (*Target)->ShortDesc, ExpectedIndexHash);
1187    }
1188 }
1189                                                                         /*}}}*/
1190 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1191 {
1192 //    // Maybe this should be made available from above so we don't have
1193 //    // to read and parse it every time?
1194 //    pkgVendorList List;
1195 //    List.ReadMainList();
1196
1197 //    const Vendor* Vndr = NULL;
1198 //    for (std::vector<string>::const_iterator I = GPGVOutput.begin(); I != GPGVOutput.end(); I++)
1199 //    {
1200 //       string::size_type pos = (*I).find("VALIDSIG ");
1201 //       if (_config->FindB("Debug::Vendor", false))
1202 //          std::cerr << "Looking for VALIDSIG in \"" << (*I) << "\": pos " << pos
1203 //                    << std::endl;
1204 //       if (pos != std::string::npos)
1205 //       {
1206 //          string Fingerprint = (*I).substr(pos+sizeof("VALIDSIG"));
1207 //          if (_config->FindB("Debug::Vendor", false))
1208 //             std::cerr << "Looking for \"" << Fingerprint << "\" in vendor..." <<
1209 //                std::endl;
1210 //          Vndr = List.FindVendor(Fingerprint) != "";
1211 //          if (Vndr != NULL);
1212 //          break;
1213 //       }
1214 //    }
1215    string::size_type pos;
1216
1217    // check for missing sigs (that where not fatal because otherwise we had
1218    // bombed earlier)
1219    string missingkeys;
1220    string msg = _("There is no public key available for the "
1221                   "following key IDs:\n");
1222    pos = Message.find("NO_PUBKEY ");
1223    if (pos != std::string::npos)
1224    {
1225       string::size_type start = pos+strlen("NO_PUBKEY ");
1226       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1227       missingkeys += (Fingerprint);
1228    }
1229    if(!missingkeys.empty())
1230       _error->Warning("%s", string(msg+missingkeys).c_str());
1231
1232    string Transformed = MetaIndexParser->GetExpectedDist();
1233
1234    if (Transformed == "../project/experimental")
1235    {
1236       Transformed = "experimental";
1237    }
1238
1239    pos = Transformed.rfind('/');
1240    if (pos != string::npos)
1241    {
1242       Transformed = Transformed.substr(0, pos);
1243    }
1244
1245    if (Transformed == ".")
1246    {
1247       Transformed = "";
1248    }
1249
1250    if (_config->FindB("Acquire::Check-Valid-Until", true) == true &&
1251        MetaIndexParser->GetValidUntil() > 0) {
1252       time_t const invalid_since = time(NULL) - MetaIndexParser->GetValidUntil();
1253       if (invalid_since > 0)
1254          // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1255          // the time since then the file is invalid - formated in the same way as in
1256          // the download progress display (e.g. 7d 3h 42min 1s)
1257          return _error->Error(_("Release file expired, ignoring %s (invalid since %s)"),
1258                               RealURI.c_str(), TimeToStr(invalid_since).c_str());
1259    }
1260
1261    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1262    {
1263       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1264       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1265       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1266    }
1267
1268    if (MetaIndexParser->CheckDist(Transformed) == false)
1269    {
1270       // This might become fatal one day
1271 //       Status = StatAuthError;
1272 //       ErrorText = "Conflicting distribution; expected "
1273 //          + MetaIndexParser->GetExpectedDist() + " but got "
1274 //          + MetaIndexParser->GetDist();
1275 //       return false;
1276       if (!Transformed.empty())
1277       {
1278          _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1279                          Desc.Description.c_str(),
1280                          Transformed.c_str(),
1281                          MetaIndexParser->GetDist().c_str());
1282       }
1283    }
1284
1285    return true;
1286 }
1287                                                                         /*}}}*/
1288 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1289 // ---------------------------------------------------------------------
1290 /* */
1291 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1292 {
1293    if (AuthPass == true)
1294    {
1295       // gpgv method failed, if we have a good signature
1296       string LastGoodSigFile = _config->FindDir("Dir::State::lists") +
1297          "partial/" + URItoFileName(RealURI) + ".gpg.reverify";
1298       if(FileExists(LastGoodSigFile))
1299       {
1300          string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1301             URItoFileName(RealURI) + ".gpg";
1302          Rename(LastGoodSigFile,VerifiedSigFile);
1303          Status = StatTransientNetworkError;
1304          _error->Warning(_("A error occurred during the signature "
1305                            "verification. The repository is not updated "
1306                            "and the previous index files will be used."
1307                            "GPG error: %s: %s\n"),
1308                          Desc.Description.c_str(),
1309                          LookupTag(Message,"Message").c_str());
1310          RunScripts("APT::Update::Auth-Failure");
1311          return;
1312       } else {
1313          _error->Warning(_("GPG error: %s: %s"),
1314                          Desc.Description.c_str(),
1315                          LookupTag(Message,"Message").c_str());
1316       }
1317       // gpgv method failed
1318       ReportMirrorFailure("GPGFailure");
1319    }
1320
1321    // No Release file was present, or verification failed, so fall
1322    // back to queueing Packages files without verification
1323    QueueIndexes(false);
1324 }
1325                                                                         /*}}}*/
1326 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1327 // ---------------------------------------------------------------------
1328 /* This just sets up the initial fetch environment and queues the first
1329    possibilitiy */
1330 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1331                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1332                              string &StoreFilename) :
1333                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1334                StoreFilename(StoreFilename), Vf(Version.FileList()),
1335                Trusted(false)
1336 {
1337    Retries = _config->FindI("Acquire::Retries",0);
1338
1339    if (Version.Arch() == 0)
1340    {
1341       _error->Error(_("I wasn't able to locate a file for the %s package. "
1342                       "This might mean you need to manually fix this package. "
1343                       "(due to missing arch)"),
1344                     Version.ParentPkg().Name());
1345       return;
1346    }
1347
1348    /* We need to find a filename to determine the extension. We make the
1349       assumption here that all the available sources for this version share
1350       the same extension.. */
1351    // Skip not source sources, they do not have file fields.
1352    for (; Vf.end() == false; Vf++)
1353    {
1354       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1355          continue;
1356       break;
1357    }
1358
1359    // Does not really matter here.. we are going to fail out below
1360    if (Vf.end() != true)
1361    {
1362       // If this fails to get a file name we will bomb out below.
1363       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1364       if (_error->PendingError() == true)
1365          return;
1366
1367       // Generate the final file name as: package_version_arch.foo
1368       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1369                       QuoteString(Version.VerStr(),"_:") + '_' +
1370                       QuoteString(Version.Arch(),"_:.") +
1371                       "." + flExtension(Parse.FileName());
1372    }
1373
1374    // check if we have one trusted source for the package. if so, switch
1375    // to "TrustedOnly" mode
1376    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; i++)
1377    {
1378       pkgIndexFile *Index;
1379       if (Sources->FindIndex(i.File(),Index) == false)
1380          continue;
1381       if (_config->FindB("Debug::pkgAcquire::Auth", false))
1382       {
1383          std::cerr << "Checking index: " << Index->Describe()
1384                    << "(Trusted=" << Index->IsTrusted() << ")\n";
1385       }
1386       if (Index->IsTrusted()) {
1387          Trusted = true;
1388          break;
1389       }
1390    }
1391
1392    // "allow-unauthenticated" restores apts old fetching behaviour
1393    // that means that e.g. unauthenticated file:// uris are higher
1394    // priority than authenticated http:// uris
1395    if (_config->FindB("APT::Get::AllowUnauthenticated",false) == true)
1396       Trusted = false;
1397
1398    // Select a source
1399    if (QueueNext() == false && _error->PendingError() == false)
1400       _error->Error(_("I wasn't able to locate file for the %s package. "
1401                     "This might mean you need to manually fix this package."),
1402                     Version.ParentPkg().Name());
1403 }
1404                                                                         /*}}}*/
1405 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1406 // ---------------------------------------------------------------------
1407 /* This queues the next available file version for download. It checks if
1408    the archive is already available in the cache and stashs the MD5 for
1409    checking later. */
1410 bool pkgAcqArchive::QueueNext()
1411 {
1412    string const ForceHash = _config->Find("Acquire::ForceHash");
1413    for (; Vf.end() == false; Vf++)
1414    {
1415       // Ignore not source sources
1416       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1417          continue;
1418
1419       // Try to cross match against the source list
1420       pkgIndexFile *Index;
1421       if (Sources->FindIndex(Vf.File(),Index) == false)
1422             continue;
1423
1424       // only try to get a trusted package from another source if that source
1425       // is also trusted
1426       if(Trusted && !Index->IsTrusted())
1427          continue;
1428
1429       // Grab the text package record
1430       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1431       if (_error->PendingError() == true)
1432          return false;
1433
1434       string PkgFile = Parse.FileName();
1435       if (ForceHash.empty() == false)
1436       {
1437          if(stringcasecmp(ForceHash, "sha256") == 0)
1438             ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
1439          else if (stringcasecmp(ForceHash, "sha1") == 0)
1440             ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
1441          else
1442             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1443       }
1444       else
1445       {
1446          string Hash;
1447          if ((Hash = Parse.SHA256Hash()).empty() == false)
1448             ExpectedHash = HashString("SHA256", Hash);
1449          else if ((Hash = Parse.SHA1Hash()).empty() == false)
1450             ExpectedHash = HashString("SHA1", Hash);
1451          else
1452             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1453       }
1454       if (PkgFile.empty() == true)
1455          return _error->Error(_("The package index files are corrupted. No Filename: "
1456                               "field for package %s."),
1457                               Version.ParentPkg().Name());
1458
1459       Desc.URI = Index->ArchiveURI(PkgFile);
1460       Desc.Description = Index->ArchiveInfo(Version);
1461       Desc.Owner = this;
1462       Desc.ShortDesc = Version.ParentPkg().Name();
1463
1464       // See if we already have the file. (Legacy filenames)
1465       FileSize = Version->Size;
1466       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
1467       struct stat Buf;
1468       if (stat(FinalFile.c_str(),&Buf) == 0)
1469       {
1470          // Make sure the size matches
1471          if ((unsigned)Buf.st_size == Version->Size)
1472          {
1473             Complete = true;
1474             Local = true;
1475             Status = StatDone;
1476             StoreFilename = DestFile = FinalFile;
1477             return true;
1478          }
1479
1480          /* Hmm, we have a file and its size does not match, this means it is
1481             an old style mismatched arch */
1482          unlink(FinalFile.c_str());
1483       }
1484
1485       // Check it again using the new style output filenames
1486       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
1487       if (stat(FinalFile.c_str(),&Buf) == 0)
1488       {
1489          // Make sure the size matches
1490          if ((unsigned)Buf.st_size == Version->Size)
1491          {
1492             Complete = true;
1493             Local = true;
1494             Status = StatDone;
1495             StoreFilename = DestFile = FinalFile;
1496             return true;
1497          }
1498
1499          /* Hmm, we have a file and its size does not match, this shouldnt
1500             happen.. */
1501          unlink(FinalFile.c_str());
1502       }
1503
1504       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
1505
1506       // Check the destination file
1507       if (stat(DestFile.c_str(),&Buf) == 0)
1508       {
1509          // Hmm, the partial file is too big, erase it
1510          if ((unsigned)Buf.st_size > Version->Size)
1511             unlink(DestFile.c_str());
1512          else
1513             PartialSize = Buf.st_size;
1514       }
1515
1516       // Create the item
1517       Local = false;
1518       Desc.URI = Index->ArchiveURI(PkgFile);
1519       Desc.Description = Index->ArchiveInfo(Version);
1520       Desc.Owner = this;
1521       Desc.ShortDesc = Version.ParentPkg().Name();
1522       QueueURI(Desc);
1523
1524       Vf++;
1525       return true;
1526    }
1527    return false;
1528 }
1529                                                                         /*}}}*/
1530 // AcqArchive::Done - Finished fetching                                 /*{{{*/
1531 // ---------------------------------------------------------------------
1532 /* */
1533 void pkgAcqArchive::Done(string Message,unsigned long Size,string CalcHash,
1534                          pkgAcquire::MethodConfig *Cfg)
1535 {
1536    Item::Done(Message,Size,CalcHash,Cfg);
1537
1538    // Check the size
1539    if (Size != Version->Size)
1540    {
1541       Status = StatError;
1542       ErrorText = _("Size mismatch");
1543       return;
1544    }
1545
1546    // Check the hash
1547    if(ExpectedHash.toStr() != CalcHash)
1548    {
1549       Status = StatError;
1550       ErrorText = _("Hash Sum mismatch");
1551       if(FileExists(DestFile))
1552          Rename(DestFile,DestFile + ".FAILED");
1553       return;
1554    }
1555
1556    // Grab the output filename
1557    string FileName = LookupTag(Message,"Filename");
1558    if (FileName.empty() == true)
1559    {
1560       Status = StatError;
1561       ErrorText = "Method gave a blank filename";
1562       return;
1563    }
1564
1565    Complete = true;
1566
1567    // Reference filename
1568    if (FileName != DestFile)
1569    {
1570       StoreFilename = DestFile = FileName;
1571       Local = true;
1572       return;
1573    }
1574
1575    // Done, move it into position
1576    string FinalFile = _config->FindDir("Dir::Cache::Archives");
1577    FinalFile += flNotDir(StoreFilename);
1578    Rename(DestFile,FinalFile);
1579
1580    StoreFilename = DestFile = FinalFile;
1581    Complete = true;
1582 }
1583                                                                         /*}}}*/
1584 // AcqArchive::Failed - Failure handler                                 /*{{{*/
1585 // ---------------------------------------------------------------------
1586 /* Here we try other sources */
1587 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1588 {
1589    ErrorText = LookupTag(Message,"Message");
1590
1591    /* We don't really want to retry on failed media swaps, this prevents
1592       that. An interesting observation is that permanent failures are not
1593       recorded. */
1594    if (Cnf->Removable == true &&
1595        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1596    {
1597       // Vf = Version.FileList();
1598       while (Vf.end() == false) Vf++;
1599       StoreFilename = string();
1600       Item::Failed(Message,Cnf);
1601       return;
1602    }
1603
1604    if (QueueNext() == false)
1605    {
1606       // This is the retry counter
1607       if (Retries != 0 &&
1608           Cnf->LocalOnly == false &&
1609           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1610       {
1611          Retries--;
1612          Vf = Version.FileList();
1613          if (QueueNext() == true)
1614             return;
1615       }
1616
1617       StoreFilename = string();
1618       Item::Failed(Message,Cnf);
1619    }
1620 }
1621                                                                         /*}}}*/
1622 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
1623 // ---------------------------------------------------------------------
1624 bool pkgAcqArchive::IsTrusted()
1625 {
1626    return Trusted;
1627 }
1628                                                                         /*}}}*/
1629 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
1630 // ---------------------------------------------------------------------
1631 /* */
1632 void pkgAcqArchive::Finished()
1633 {
1634    if (Status == pkgAcquire::Item::StatDone &&
1635        Complete == true)
1636       return;
1637    StoreFilename = string();
1638 }
1639                                                                         /*}}}*/
1640 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
1641 // ---------------------------------------------------------------------
1642 /* The file is added to the queue */
1643 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
1644                        unsigned long Size,string Dsc,string ShortDesc,
1645                        const string &DestDir, const string &DestFilename,
1646                        bool IsIndexFile) :
1647                        Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile)
1648 {
1649    Retries = _config->FindI("Acquire::Retries",0);
1650
1651    if(!DestFilename.empty())
1652       DestFile = DestFilename;
1653    else if(!DestDir.empty())
1654       DestFile = DestDir + "/" + flNotDir(URI);
1655    else
1656       DestFile = flNotDir(URI);
1657
1658    // Create the item
1659    Desc.URI = URI;
1660    Desc.Description = Dsc;
1661    Desc.Owner = this;
1662
1663    // Set the short description to the archive component
1664    Desc.ShortDesc = ShortDesc;
1665
1666    // Get the transfer sizes
1667    FileSize = Size;
1668    struct stat Buf;
1669    if (stat(DestFile.c_str(),&Buf) == 0)
1670    {
1671       // Hmm, the partial file is too big, erase it
1672       if ((unsigned)Buf.st_size > Size)
1673          unlink(DestFile.c_str());
1674       else
1675          PartialSize = Buf.st_size;
1676    }
1677
1678    QueueURI(Desc);
1679 }
1680                                                                         /*}}}*/
1681 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
1682 // ---------------------------------------------------------------------
1683 /* */
1684 void pkgAcqFile::Done(string Message,unsigned long Size,string CalcHash,
1685                       pkgAcquire::MethodConfig *Cnf)
1686 {
1687    Item::Done(Message,Size,CalcHash,Cnf);
1688
1689    // Check the hash
1690    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
1691    {
1692       Status = StatError;
1693       ErrorText = "Hash Sum mismatch";
1694       Rename(DestFile,DestFile + ".FAILED");
1695       return;
1696    }
1697
1698    string FileName = LookupTag(Message,"Filename");
1699    if (FileName.empty() == true)
1700    {
1701       Status = StatError;
1702       ErrorText = "Method gave a blank filename";
1703       return;
1704    }
1705
1706    Complete = true;
1707
1708    // The files timestamp matches
1709    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1710       return;
1711
1712    // We have to copy it into place
1713    if (FileName != DestFile)
1714    {
1715       Local = true;
1716       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
1717           Cnf->Removable == true)
1718       {
1719          Desc.URI = "copy:" + FileName;
1720          QueueURI(Desc);
1721          return;
1722       }
1723
1724       // Erase the file if it is a symlink so we can overwrite it
1725       struct stat St;
1726       if (lstat(DestFile.c_str(),&St) == 0)
1727       {
1728          if (S_ISLNK(St.st_mode) != 0)
1729             unlink(DestFile.c_str());
1730       }
1731
1732       // Symlink the file
1733       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
1734       {
1735          ErrorText = "Link to " + DestFile + " failure ";
1736          Status = StatError;
1737          Complete = false;
1738       }
1739    }
1740 }
1741                                                                         /*}}}*/
1742 // AcqFile::Failed - Failure handler                                    /*{{{*/
1743 // ---------------------------------------------------------------------
1744 /* Here we try other sources */
1745 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1746 {
1747    ErrorText = LookupTag(Message,"Message");
1748
1749    // This is the retry counter
1750    if (Retries != 0 &&
1751        Cnf->LocalOnly == false &&
1752        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1753    {
1754       Retries--;
1755       QueueURI(Desc);
1756       return;
1757    }
1758
1759    Item::Failed(Message,Cnf);
1760 }
1761                                                                         /*}}}*/
1762 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
1763 // ---------------------------------------------------------------------
1764 /* The only header we use is the last-modified header. */
1765 string pkgAcqFile::Custom600Headers()
1766 {
1767    if (IsIndexFile)
1768       return "\nIndex-File: true";
1769    return "";
1770 }
1771                                                                         /*}}}*/