apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <config.h>
  17
  18 #include <apt-pkg/acquire-item.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/aptconfiguration.h>
  21 #include <apt-pkg/sourcelist.h>
  22 #include <apt-pkg/error.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/sha1.h>
  26 #include <apt-pkg/tagfile.h>
  27 #include <apt-pkg/indexrecords.h>
  28 #include <apt-pkg/acquire.h>
  29 #include <apt-pkg/hashes.h>
  30 #include <apt-pkg/indexfile.h>
  31 #include <apt-pkg/pkgcache.h>
  32 #include <apt-pkg/cacheiterators.h>
  33 #include <apt-pkg/pkgrecords.h>
  34
  35 #include <stddef.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38 #include <iostream>
  39 #include <vector>
  40 #include <sys/stat.h>
  41 #include <unistd.h>
  42 #include <errno.h>
  43 #include <string>
  44 #include <sstream>
  45 #include <stdio.h>
  46 #include <ctime>
  47
  48 #include <apti18n.h>
  49                                                                         /*}}}*/
  50
  51 using namespace std;
  52
  53 // Acquire::Item::Item - Constructor                                    /*{{{*/
  54 // ---------------------------------------------------------------------
  55 /* */
  56 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  57                        PartialSize(0), Mode(0), ID(0), Complete(false),
  58                        Local(false), QueueCounter(0)
  59 {
  60    Owner->Add(this);
  61    Status = StatIdle;
  62 }
  63                                                                         /*}}}*/
  64 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  65 // ---------------------------------------------------------------------
  66 /* */
  67 pkgAcquire::Item::~Item()
  68 {
  69    Owner->Remove(this);
  70 }
  71                                                                         /*}}}*/
  72 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  73 // ---------------------------------------------------------------------
  74 /* We return to an idle state if there are still other queues that could
  75    fetch this object */
  76 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  77 {
  78    Status = StatIdle;
  79    ErrorText = LookupTag(Message,"Message");
  80    UsedMirror =  LookupTag(Message,"UsedMirror");
  81    if (QueueCounter <= 1)
  82    {
  83       /* This indicates that the file is not available right now but might
  84          be sometime later. If we do a retry cycle then this should be
  85          retried [CDROMs] */
  86       if (Cnf->LocalOnly == true &&
  87           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  88       {
  89          Status = StatIdle;
  90          Dequeue();
  91          return;
  92       }
  93
  94       Status = StatError;
  95       Dequeue();
  96    }
  97
  98    // report mirror failure back to LP if we actually use a mirror
  99    string FailReason = LookupTag(Message, "FailReason");
 100    if(FailReason.size() != 0)
 101       ReportMirrorFailure(FailReason);
 102    else
 103       ReportMirrorFailure(ErrorText);
 104 }
 105                                                                         /*}}}*/
 106 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
 107 // ---------------------------------------------------------------------
 108 /* Stash status and the file size. Note that setting Complete means
 109    sub-phases of the acquire process such as decompresion are operating */
 110 void pkgAcquire::Item::Start(string /*Message*/,unsigned long long Size)
 111 {
 112    Status = StatFetching;
 113    if (FileSize == 0 && Complete == false)
 114       FileSize = Size;
 115 }
 116                                                                         /*}}}*/
 117 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
 118 // ---------------------------------------------------------------------
 119 /* */
 120 void pkgAcquire::Item::Done(string Message,unsigned long long Size,string /*Hash*/,
 121                             pkgAcquire::MethodConfig * /*Cnf*/)
 122 {
 123    // We just downloaded something..
 124    string FileName = LookupTag(Message,"Filename");
 125    UsedMirror =  LookupTag(Message,"UsedMirror");
 126    if (Complete == false && !Local && FileName == DestFile)
 127    {
 128       if (Owner->Log != 0)
 129          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 130    }
 131
 132    if (FileSize == 0)
 133       FileSize= Size;
 134    Status = StatDone;
 135    ErrorText = string();
 136    Owner->Dequeue(this);
 137 }
 138                                                                         /*}}}*/
 139 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 140 // ---------------------------------------------------------------------
 141 /* This helper function is used by a lot of item methods as their final
 142    step */
 143 void pkgAcquire::Item::Rename(string From,string To)
 144 {
 145    if (rename(From.c_str(),To.c_str()) != 0)
 146    {
 147       char S[300];
 148       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 149               From.c_str(),To.c_str());
 150       Status = StatError;
 151       ErrorText = S;
 152    }
 153 }
 154                                                                         /*}}}*/
 155 bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const error)/*{{{*/
 156 {
 157    if(FileExists(DestFile))
 158       Rename(DestFile, DestFile + ".FAILED");
 159
 160    switch (error)
 161    {
 162       case HashSumMismatch:
 163          ErrorText = _("Hash Sum mismatch");
 164          Status = StatAuthError;
 165          ReportMirrorFailure("HashChecksumFailure");
 166          break;
 167       case SizeMismatch:
 168          ErrorText = _("Size mismatch");
 169          Status = StatAuthError;
 170          ReportMirrorFailure("SizeFailure");
 171          break;
 172       case InvalidFormat:
 173          ErrorText = _("Invalid file format");
 174          Status = StatError;
 175          // do not report as usually its not the mirrors fault, but Portal/Proxy
 176          break;
 177    }
 178    return false;
 179 }
 180                                                                         /*}}}*/
 181 // Acquire::Item::ReportMirrorFailure                                   /*{{{*/
 182 // ---------------------------------------------------------------------
 183 void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
 184 {
 185    // we only act if a mirror was used at all
 186    if(UsedMirror.empty())
 187       return;
 188 #if 0
 189    std::cerr << "\nReportMirrorFailure: "
 190              << UsedMirror
 191              << " Uri: " << DescURI()
 192              << " FailCode: "
 193              << FailCode << std::endl;
 194 #endif
 195    const char *Args[40];
 196    unsigned int i = 0;
 197    string report = _config->Find("Methods::Mirror::ProblemReporting",
 198                                  "/usr/lib/apt/apt-report-mirror-failure");
 199    if(!FileExists(report))
 200       return;
 201    Args[i++] = report.c_str();
 202    Args[i++] = UsedMirror.c_str();
 203    Args[i++] = DescURI().c_str();
 204    Args[i++] = FailCode.c_str();
 205    Args[i++] = NULL;
 206    pid_t pid = ExecFork();
 207    if(pid < 0)
 208    {
 209       _error->Error("ReportMirrorFailure Fork failed");
 210       return;
 211    }
 212    else if(pid == 0)
 213    {
 214       execvp(Args[0], (char**)Args);
 215       std::cerr << "Could not exec " << Args[0] << std::endl;
 216       _exit(100);
 217    }
 218    if(!ExecWait(pid, "report-mirror-failure"))
 219    {
 220       _error->Warning("Couldn't report problem to '%s'",
 221                       _config->Find("Methods::Mirror::ProblemReporting").c_str());
 222    }
 223 }
 224                                                                         /*}}}*/
 225 // AcqSubIndex::AcqSubIndex - Constructor                               /*{{{*/
 226 // ---------------------------------------------------------------------
 227 /* Get a sub-index file based on checksums from a 'master' file and
 228    possibly query additional files */
 229 pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, string const &URI,
 230                                  string const &URIDesc, string const &ShortDesc,
 231                                  HashString const &ExpectedHash)
 232    : Item(Owner), ExpectedHash(ExpectedHash)
 233 {
 234    /* XXX: Beware: Currently this class does nothing (of value) anymore ! */
 235    Debug = _config->FindB("Debug::pkgAcquire::SubIndex",false);
 236
 237    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 238    DestFile += URItoFileName(URI);
 239
 240    Desc.URI = URI;
 241    Desc.Description = URIDesc;
 242    Desc.Owner = this;
 243    Desc.ShortDesc = ShortDesc;
 244
 245    QueueURI(Desc);
 246
 247    if(Debug)
 248       std::clog << "pkgAcqSubIndex: " << Desc.URI << std::endl;
 249 }
 250                                                                         /*}}}*/
 251 // AcqSubIndex::Custom600Headers - Insert custom request headers        /*{{{*/
 252 // ---------------------------------------------------------------------
 253 /* The only header we use is the last-modified header. */
 254 string pkgAcqSubIndex::Custom600Headers()
 255 {
 256    string Final = _config->FindDir("Dir::State::lists");
 257    Final += URItoFileName(Desc.URI);
 258
 259    struct stat Buf;
 260    if (stat(Final.c_str(),&Buf) != 0)
 261       return "\nIndex-File: true\nFail-Ignore: true\n";
 262    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 263 }
 264                                                                         /*}}}*/
 265 void pkgAcqSubIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 266 {
 267    if(Debug)
 268       std::clog << "pkgAcqSubIndex failed: " << Desc.URI << " with " << Message << std::endl;
 269
 270    Complete = false;
 271    Status = StatDone;
 272    Dequeue();
 273
 274    // No good Index is provided
 275 }
 276                                                                         /*}}}*/
 277 void pkgAcqSubIndex::Done(string Message,unsigned long long Size,string Md5Hash,        /*{{{*/
 278                            pkgAcquire::MethodConfig *Cnf)
 279 {
 280    if(Debug)
 281       std::clog << "pkgAcqSubIndex::Done(): " << Desc.URI << std::endl;
 282
 283    string FileName = LookupTag(Message,"Filename");
 284    if (FileName.empty() == true)
 285    {
 286       Status = StatError;
 287       ErrorText = "Method gave a blank filename";
 288       return;
 289    }
 290
 291    if (FileName != DestFile)
 292    {
 293       Local = true;
 294       Desc.URI = "copy:" + FileName;
 295       QueueURI(Desc);
 296       return;
 297    }
 298
 299    Item::Done(Message,Size,Md5Hash,Cnf);
 300
 301    string FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(Desc.URI);
 302
 303    /* Downloaded invalid transindex => Error (LP: #346386) (Closes: #627642) */
 304    indexRecords SubIndexParser;
 305    if (FileExists(DestFile) == true && !SubIndexParser.Load(DestFile)) {
 306       Status = StatError;
 307       ErrorText = SubIndexParser.ErrorText;
 308       return;
 309    }
 310
 311    // success in downloading the index
 312    // rename the index
 313    if(Debug)
 314       std::clog << "Renaming: " << DestFile << " -> " << FinalFile << std::endl;
 315    Rename(DestFile,FinalFile);
 316    chmod(FinalFile.c_str(),0644);
 317    DestFile = FinalFile;
 318
 319    if(ParseIndex(DestFile) == false)
 320       return Failed("", NULL);
 321
 322    Complete = true;
 323    Status = StatDone;
 324    Dequeue();
 325    return;
 326 }
 327                                                                         /*}}}*/
 328 bool pkgAcqSubIndex::ParseIndex(string const &IndexFile)                /*{{{*/
 329 {
 330    indexRecords SubIndexParser;
 331    if (FileExists(IndexFile) == false || SubIndexParser.Load(IndexFile) == false)
 332       return false;
 333    // so something with the downloaded index
 334    return true;
 335 }
 336                                                                         /*}}}*/
 337 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 338 // ---------------------------------------------------------------------
 339 /* Get the DiffIndex file first and see if there are patches available
 340  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 341  * patches. If anything goes wrong in that process, it will fall back to
 342  * the original packages file
 343  */
 344 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 345                                  string URI,string URIDesc,string ShortDesc,
 346                                  HashString ExpectedHash)
 347    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 348      Description(URIDesc)
 349 {
 350
 351    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 352
 353    Desc.Description = URIDesc + "/DiffIndex";
 354    Desc.Owner = this;
 355    Desc.ShortDesc = ShortDesc;
 356    Desc.URI = URI + ".diff/Index";
 357
 358    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 359    DestFile += URItoFileName(Desc.URI);
 360
 361    if(Debug)
 362       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 363
 364    // look for the current package file
 365    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 366    CurrentPackagesFile += URItoFileName(RealURI);
 367
 368    // FIXME: this file:/ check is a hack to prevent fetching
 369    //        from local sources. this is really silly, and
 370    //        should be fixed cleanly as soon as possible
 371    if(!FileExists(CurrentPackagesFile) ||
 372       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 373    {
 374       // we don't have a pkg file or we don't want to queue
 375       if(Debug)
 376          std::clog << "No index file, local or canceld by user" << std::endl;
 377       Failed("", NULL);
 378       return;
 379    }
 380
 381    if(Debug)
 382       std::clog << "pkgAcqDiffIndex::pkgAcqDiffIndex(): "
 383          << CurrentPackagesFile << std::endl;
 384
 385    QueueURI(Desc);
 386
 387 }
 388                                                                         /*}}}*/
 389 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 390 // ---------------------------------------------------------------------
 391 /* The only header we use is the last-modified header. */
 392 string pkgAcqDiffIndex::Custom600Headers()
 393 {
 394    string Final = _config->FindDir("Dir::State::lists");
 395    Final += URItoFileName(Desc.URI);
 396
 397    if(Debug)
 398       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 399
 400    struct stat Buf;
 401    if (stat(Final.c_str(),&Buf) != 0)
 402       return "\nIndex-File: true";
 403
 404    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 405 }
 406                                                                         /*}}}*/
 407 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 408 {
 409    if(Debug)
 410       std::clog << "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile
 411          << std::endl;
 412
 413    pkgTagSection Tags;
 414    string ServerSha1;
 415    vector<DiffInfo> available_patches;
 416
 417    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 418    pkgTagFile TF(&Fd);
 419    if (_error->PendingError() == true)
 420       return false;
 421
 422    if(TF.Step(Tags) == true)
 423    {
 424       bool found = false;
 425       DiffInfo d;
 426       string size;
 427
 428       string const tmp = Tags.FindS("SHA1-Current");
 429       std::stringstream ss(tmp);
 430       ss >> ServerSha1 >> size;
 431       unsigned long const ServerSize = atol(size.c_str());
 432
 433       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 434       SHA1Summation SHA1;
 435       SHA1.AddFD(fd);
 436       string const local_sha1 = SHA1.Result();
 437
 438       if(local_sha1 == ServerSha1)
 439       {
 440          // we have the same sha1 as the server so we are done here
 441          if(Debug)
 442             std::clog << "Package file is up-to-date" << std::endl;
 443          // list cleanup needs to know that this file as well as the already
 444          // present index is ours, so we create an empty diff to save it for us
 445          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 446                ExpectedHash, ServerSha1, available_patches);
 447          return true;
 448       }
 449       else
 450       {
 451          if(Debug)
 452             std::clog << "SHA1-Current: " << ServerSha1 << " and we start at "<< fd.Name() << " " << fd.Size() << " " << local_sha1 << std::endl;
 453
 454          // check the historie and see what patches we need
 455          string const history = Tags.FindS("SHA1-History");
 456          std::stringstream hist(history);
 457          while(hist >> d.sha1 >> size >> d.file)
 458          {
 459             // read until the first match is found
 460             // from that point on, we probably need all diffs
 461             if(d.sha1 == local_sha1)
 462                found=true;
 463             else if (found == false)
 464                continue;
 465
 466             if(Debug)
 467                std::clog << "Need to get diff: " << d.file << std::endl;
 468             available_patches.push_back(d);
 469          }
 470
 471          if (available_patches.empty() == false)
 472          {
 473             // patching with too many files is rather slow compared to a fast download
 474             unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 475             if (fileLimit != 0 && fileLimit < available_patches.size())
 476             {
 477                if (Debug)
 478                   std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 479                         << ") so fallback to complete download" << std::endl;
 480                return false;
 481             }
 482
 483             // see if the patches are too big
 484             found = false; // it was true and it will be true again at the end
 485             d = *available_patches.begin();
 486             string const firstPatch = d.file;
 487             unsigned long patchesSize = 0;
 488             std::stringstream patches(Tags.FindS("SHA1-Patches"));
 489             while(patches >> d.sha1 >> size >> d.file)
 490             {
 491                if (firstPatch == d.file)
 492                   found = true;
 493                else if (found == false)
 494                   continue;
 495
 496                patchesSize += atol(size.c_str());
 497             }
 498             unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 499             if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 500             {
 501                if (Debug)
 502                   std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 503                         << ") so fallback to complete download" << std::endl;
 504                return false;
 505             }
 506          }
 507       }
 508
 509       // we have something, queue the next diff
 510       if(found)
 511       {
 512          // queue the diffs
 513          string::size_type const last_space = Description.rfind(" ");
 514          if(last_space != string::npos)
 515             Description.erase(last_space, Description.size()-last_space);
 516
 517          /* decide if we should download patches one by one or in one go:
 518             The first is good if the server merges patches, but many don't so client
 519             based merging can be attempt in which case the second is better.
 520             "bad things" will happen if patches are merged on the server,
 521             but client side merging is attempt as well */
 522          bool pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true);
 523          if (pdiff_merge == true)
 524          {
 525             // reprepro adds this flag if it has merged patches on the server
 526             std::string const precedence = Tags.FindS("X-Patch-Precedence");
 527             pdiff_merge = (precedence != "merged");
 528          }
 529
 530          if (pdiff_merge == false)
 531             new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 532                   ExpectedHash, ServerSha1, available_patches);
 533          else
 534          {
 535             std::vector<pkgAcqIndexMergeDiffs*> *diffs = new std::vector<pkgAcqIndexMergeDiffs*>(available_patches.size());
 536             for(size_t i = 0; i < available_patches.size(); ++i)
 537                (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, RealURI, Description, Desc.ShortDesc, ExpectedHash,
 538                      available_patches[i], diffs);
 539          }
 540
 541          Complete = false;
 542          Status = StatDone;
 543          Dequeue();
 544          return true;
 545       }
 546    }
 547
 548    // Nothing found, report and return false
 549    // Failing here is ok, if we return false later, the full
 550    // IndexFile is queued
 551    if(Debug)
 552       std::clog << "Can't find a patch in the index file" << std::endl;
 553    return false;
 554 }
 555                                                                         /*}}}*/
 556 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 557 {
 558    if(Debug)
 559       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl
 560                 << "Falling back to normal index file acquire" << std::endl;
 561
 562    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 563                    ExpectedHash);
 564
 565    Complete = false;
 566    Status = StatDone;
 567    Dequeue();
 568 }
 569                                                                         /*}}}*/
 570 void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,string Md5Hash,       /*{{{*/
 571                            pkgAcquire::MethodConfig *Cnf)
 572 {
 573    if(Debug)
 574       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 575
 576    Item::Done(Message,Size,Md5Hash,Cnf);
 577
 578    string FinalFile;
 579    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 580
 581    // success in downloading the index
 582    // rename the index
 583    FinalFile += string(".IndexDiff");
 584    if(Debug)
 585       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 586                 << std::endl;
 587    Rename(DestFile,FinalFile);
 588    chmod(FinalFile.c_str(),0644);
 589    DestFile = FinalFile;
 590
 591    if(!ParseDiffIndex(DestFile))
 592       return Failed("", NULL);
 593
 594    Complete = true;
 595    Status = StatDone;
 596    Dequeue();
 597    return;
 598 }
 599                                                                         /*}}}*/
 600 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 601 // ---------------------------------------------------------------------
 602 /* The package diff is added to the queue. one object is constructed
 603  * for each diff and the index
 604  */
 605 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 606                                    string URI,string URIDesc,string ShortDesc,
 607                                    HashString ExpectedHash,
 608                                    string ServerSha1,
 609                                    vector<DiffInfo> diffs)
 610    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 611      available_patches(diffs), ServerSha1(ServerSha1)
 612 {
 613
 614    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 615    DestFile += URItoFileName(URI);
 616
 617    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 618
 619    Description = URIDesc;
 620    Desc.Owner = this;
 621    Desc.ShortDesc = ShortDesc;
 622
 623    if(available_patches.empty() == true)
 624    {
 625       // we are done (yeah!)
 626       Finish(true);
 627    }
 628    else
 629    {
 630       // get the next diff
 631       State = StateFetchDiff;
 632       QueueNextDiff();
 633    }
 634 }
 635                                                                         /*}}}*/
 636 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 637 {
 638    if(Debug)
 639       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl
 640                 << "Falling back to normal index file acquire" << std::endl;
 641    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 642                    ExpectedHash);
 643    Finish();
 644 }
 645                                                                         /*}}}*/
 646 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 647 void pkgAcqIndexDiffs::Finish(bool allDone)
 648 {
 649    // we restore the original name, this is required, otherwise
 650    // the file will be cleaned
 651    if(allDone)
 652    {
 653       DestFile = _config->FindDir("Dir::State::lists");
 654       DestFile += URItoFileName(RealURI);
 655
 656       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 657       {
 658          RenameOnError(HashSumMismatch);
 659          Dequeue();
 660          return;
 661       }
 662
 663       // this is for the "real" finish
 664       Complete = true;
 665       Status = StatDone;
 666       Dequeue();
 667       if(Debug)
 668          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 669       return;
 670    }
 671
 672    if(Debug)
 673       std::clog << "Finishing: " << Desc.URI << std::endl;
 674    Complete = false;
 675    Status = StatDone;
 676    Dequeue();
 677    return;
 678 }
 679                                                                         /*}}}*/
 680 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 681 {
 682
 683    // calc sha1 of the just patched file
 684    string FinalFile = _config->FindDir("Dir::State::lists");
 685    FinalFile += URItoFileName(RealURI);
 686
 687    FileFd fd(FinalFile, FileFd::ReadOnly);
 688    SHA1Summation SHA1;
 689    SHA1.AddFD(fd);
 690    string local_sha1 = string(SHA1.Result());
 691    if(Debug)
 692       std::clog << "QueueNextDiff: "
 693                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 694
 695    // final file reached before all patches are applied
 696    if(local_sha1 == ServerSha1)
 697    {
 698       Finish(true);
 699       return true;
 700    }
 701
 702    // remove all patches until the next matching patch is found
 703    // this requires the Index file to be ordered
 704    for(vector<DiffInfo>::iterator I=available_patches.begin();
 705        available_patches.empty() == false &&
 706           I != available_patches.end() &&
 707           I->sha1 != local_sha1;
 708        ++I)
 709    {
 710       available_patches.erase(I);
 711    }
 712
 713    // error checking and falling back if no patch was found
 714    if(available_patches.empty() == true)
 715    {
 716       Failed("", NULL);
 717       return false;
 718    }
 719
 720    // queue the right diff
 721    Desc.URI = RealURI + ".diff/" + available_patches[0].file + ".gz";
 722    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 723    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 724    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 725
 726    if(Debug)
 727       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 728
 729    QueueURI(Desc);
 730
 731    return true;
 732 }
 733                                                                         /*}}}*/
 734 void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size,string Md5Hash,      /*{{{*/
 735                             pkgAcquire::MethodConfig *Cnf)
 736 {
 737    if(Debug)
 738       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 739
 740    Item::Done(Message,Size,Md5Hash,Cnf);
 741
 742    string FinalFile;
 743    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 744
 745    // success in downloading a diff, enter ApplyDiff state
 746    if(State == StateFetchDiff)
 747    {
 748
 749       // rred excepts the patch as $FinalFile.ed
 750       Rename(DestFile,FinalFile+".ed");
 751
 752       if(Debug)
 753          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 754
 755       State = StateApplyDiff;
 756       Local = true;
 757       Desc.URI = "rred:" + FinalFile;
 758       QueueURI(Desc);
 759       Mode = "rred";
 760       return;
 761    }
 762
 763
 764    // success in download/apply a diff, queue next (if needed)
 765    if(State == StateApplyDiff)
 766    {
 767       // remove the just applied patch
 768       available_patches.erase(available_patches.begin());
 769       unlink((FinalFile + ".ed").c_str());
 770
 771       // move into place
 772       if(Debug)
 773       {
 774          std::clog << "Moving patched file in place: " << std::endl
 775                    << DestFile << " -> " << FinalFile << std::endl;
 776       }
 777       Rename(DestFile,FinalFile);
 778       chmod(FinalFile.c_str(),0644);
 779
 780       // see if there is more to download
 781       if(available_patches.empty() == false) {
 782          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 783                               ExpectedHash, ServerSha1, available_patches);
 784          return Finish();
 785       } else
 786          return Finish(true);
 787    }
 788 }
 789                                                                         /*}}}*/
 790 // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor                 /*{{{*/
 791 pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner,
 792                                    string const &URI, string const &URIDesc,
 793                                    string const &ShortDesc, HashString const &ExpectedHash,
 794                                    DiffInfo const &patch,
 795                                    std::vector<pkgAcqIndexMergeDiffs*> const * const allPatches)
 796    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 797      patch(patch),allPatches(allPatches), State(StateFetchDiff)
 798 {
 799
 800    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 801    DestFile += URItoFileName(URI);
 802
 803    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 804
 805    Description = URIDesc;
 806    Desc.Owner = this;
 807    Desc.ShortDesc = ShortDesc;
 808
 809    Desc.URI = RealURI + ".diff/" + patch.file + ".gz";
 810    Desc.Description = Description + " " + patch.file + string(".pdiff");
 811    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 812    DestFile += URItoFileName(RealURI + ".diff/" + patch.file);
 813
 814    if(Debug)
 815       std::clog << "pkgAcqIndexMergeDiffs: " << Desc.URI << std::endl;
 816
 817    QueueURI(Desc);
 818 }
 819                                                                         /*}}}*/
 820 void pkgAcqIndexMergeDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 821 {
 822    if(Debug)
 823       std::clog << "pkgAcqIndexMergeDiffs failed: " << Desc.URI << " with " << Message << std::endl;
 824    Complete = false;
 825    Status = StatDone;
 826    Dequeue();
 827
 828    // check if we are the first to fail, otherwise we are done here
 829    State = StateDoneDiff;
 830    for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 831          I != allPatches->end(); ++I)
 832       if ((*I)->State == StateErrorDiff)
 833          return;
 834
 835    // first failure means we should fallback
 836    State = StateErrorDiff;
 837    std::clog << "Falling back to normal index file acquire" << std::endl;
 838    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 839                    ExpectedHash);
 840 }
 841                                                                         /*}}}*/
 842 void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,string Md5Hash, /*{{{*/
 843                             pkgAcquire::MethodConfig *Cnf)
 844 {
 845    if(Debug)
 846       std::clog << "pkgAcqIndexMergeDiffs::Done(): " << Desc.URI << std::endl;
 847
 848    Item::Done(Message,Size,Md5Hash,Cnf);
 849
 850    string const FinalFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 851
 852    if (State == StateFetchDiff)
 853    {
 854       // rred expects the patch as $FinalFile.ed.$patchname.gz
 855       Rename(DestFile, FinalFile + ".ed." + patch.file + ".gz");
 856
 857       // check if this is the last completed diff
 858       State = StateDoneDiff;
 859       for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 860             I != allPatches->end(); ++I)
 861          if ((*I)->State != StateDoneDiff)
 862          {
 863             if(Debug)
 864                std::clog << "Not the last done diff in the batch: " << Desc.URI << std::endl;
 865             return;
 866          }
 867
 868       // this is the last completed diff, so we are ready to apply now
 869       State = StateApplyDiff;
 870
 871       if(Debug)
 872          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 873
 874       Local = true;
 875       Desc.URI = "rred:" + FinalFile;
 876       QueueURI(Desc);
 877       Mode = "rred";
 878       return;
 879    }
 880    // success in download/apply all diffs, clean up
 881    else if (State == StateApplyDiff)
 882    {
 883       // see if we really got the expected file
 884       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 885       {
 886          RenameOnError(HashSumMismatch);
 887          return;
 888       }
 889
 890       // move the result into place
 891       if(Debug)
 892          std::clog << "Moving patched file in place: " << std::endl
 893                    << DestFile << " -> " << FinalFile << std::endl;
 894       Rename(DestFile, FinalFile);
 895       chmod(FinalFile.c_str(), 0644);
 896
 897       // otherwise lists cleanup will eat the file
 898       DestFile = FinalFile;
 899
 900       // ensure the ed's are gone regardless of list-cleanup
 901       for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 902             I != allPatches->end(); ++I)
 903       {
 904             std::string patch = FinalFile + ".ed." + (*I)->patch.file + ".gz";
 905             unlink(patch.c_str());
 906       }
 907
 908       // all set and done
 909       Complete = true;
 910       if(Debug)
 911          std::clog << "allDone: " << DestFile << "\n" << std::endl;
 912    }
 913 }
 914                                                                         /*}}}*/
 915 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 916 // ---------------------------------------------------------------------
 917 /* The package file is added to the queue and a second class is
 918    instantiated to fetch the revision file */
 919 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 920                          string URI,string URIDesc,string ShortDesc,
 921                          HashString ExpectedHash, string comprExt)
 922    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 923 {
 924    if(comprExt.empty() == true)
 925    {
 926       // autoselect the compression method
 927       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 928       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 929          comprExt.append(*t).append(" ");
 930       if (comprExt.empty() == false)
 931          comprExt.erase(comprExt.end()-1);
 932    }
 933    CompressionExtension = comprExt;
 934
 935    Init(URI, URIDesc, ShortDesc);
 936 }
 937 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target,
 938                          HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
 939    : Item(Owner), RealURI(Target->URI), ExpectedHash(ExpectedHash)
 940 {
 941    // autoselect the compression method
 942    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 943    CompressionExtension = "";
 944    if (ExpectedHash.empty() == false)
 945    {
 946       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 947          if (*t == "uncompressed" || MetaIndexParser->Exists(string(Target->MetaKey).append(".").append(*t)) == true)
 948             CompressionExtension.append(*t).append(" ");
 949    }
 950    else
 951    {
 952       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 953          CompressionExtension.append(*t).append(" ");
 954    }
 955    if (CompressionExtension.empty() == false)
 956       CompressionExtension.erase(CompressionExtension.end()-1);
 957
 958    Init(Target->URI, Target->Description, Target->ShortDesc);
 959 }
 960                                                                         /*}}}*/
 961 // AcqIndex::Init - defered Constructor                                 /*{{{*/
 962 void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &ShortDesc) {
 963    Decompression = false;
 964    Erase = false;
 965
 966    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 967    DestFile += URItoFileName(URI);
 968
 969    std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
 970    if (comprExt == "uncompressed")
 971       Desc.URI = URI;
 972    else
 973       Desc.URI = URI + '.' + comprExt;
 974
 975    Desc.Description = URIDesc;
 976    Desc.Owner = this;
 977    Desc.ShortDesc = ShortDesc;
 978
 979    QueueURI(Desc);
 980 }
 981                                                                         /*}}}*/
 982 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 983 // ---------------------------------------------------------------------
 984 /* The only header we use is the last-modified header. */
 985 string pkgAcqIndex::Custom600Headers()
 986 {
 987    string Final = _config->FindDir("Dir::State::lists");
 988    Final += URItoFileName(RealURI);
 989    if (_config->FindB("Acquire::GzipIndexes",false))
 990       Final += ".gz";
 991
 992    string msg = "\nIndex-File: true";
 993    // FIXME: this really should use "IndexTarget::IsOptional()" but that
 994    //        seems to be difficult without breaking ABI
 995    if (ShortDesc().find("Translation") != 0)
 996       msg += "\nFail-Ignore: true";
 997    struct stat Buf;
 998    if (stat(Final.c_str(),&Buf) == 0)
 999       msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1000
1001    return msg;
1002 }
1003                                                                         /*}}}*/
1004 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
1005 {
1006    size_t const nextExt = CompressionExtension.find(' ');
1007    if (nextExt != std::string::npos)
1008    {
1009       CompressionExtension = CompressionExtension.substr(nextExt+1);
1010       Init(RealURI, Desc.Description, Desc.ShortDesc);
1011       return;
1012    }
1013
1014    // on decompression failure, remove bad versions in partial/
1015    if (Decompression && Erase) {
1016       string s = _config->FindDir("Dir::State::lists") + "partial/";
1017       s.append(URItoFileName(RealURI));
1018       unlink(s.c_str());
1019    }
1020
1021    Item::Failed(Message,Cnf);
1022 }
1023                                                                         /*}}}*/
1024 // pkgAcqIndex::GetFinalFilename - Return the full final file path      /*{{{*/
1025 std::string pkgAcqIndex::GetFinalFilename(std::string const &URI,
1026                                           std::string const &compExt)
1027 {
1028    std::string FinalFile = _config->FindDir("Dir::State::lists");
1029    FinalFile += URItoFileName(URI);
1030    if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
1031       FinalFile += ".gz";
1032    return FinalFile;
1033 }
1034                                                                         /*}}}*/
1035 // AcqIndex::ReverifyAfterIMS - Reverify index after an ims-hit         /*{{{*/
1036 void pkgAcqIndex::ReverifyAfterIMS(std::string const &FileName)
1037 {
1038    std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
1039    if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
1040       DestFile += ".gz";
1041
1042    string FinalFile = GetFinalFilename(RealURI, compExt);
1043    Rename(FinalFile, FileName);
1044    Decompression = true;
1045    Desc.URI = "copy:" + FileName;
1046    QueueURI(Desc);
1047 }
1048                                                                         /*}}}*/
1049 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
1050 // ---------------------------------------------------------------------
1051 /* This goes through a number of states.. On the initial fetch the
1052    method could possibly return an alternate filename which points
1053    to the uncompressed version of the file. If this is so the file
1054    is copied into the partial directory. In all other cases the file
1055    is decompressed with a gzip uri. */
1056 void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash,
1057                        pkgAcquire::MethodConfig *Cfg)
1058 {
1059    Item::Done(Message,Size,Hash,Cfg);
1060    std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
1061
1062    if (Decompression == true)
1063    {
1064       if (_config->FindB("Debug::pkgAcquire::Auth", false))
1065       {
1066          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
1067          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
1068       }
1069
1070       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
1071       {
1072          Desc.URI = RealURI;
1073          RenameOnError(HashSumMismatch);
1074          return;
1075       }
1076
1077       // FIXME: this can go away once we only ever download stuff that
1078       //        has a valid hash and we never do GET based probing
1079       //
1080       /* Always verify the index file for correctness (all indexes must
1081        * have a Package field) (LP: #346386) (Closes: #627642)
1082        */
1083       FileFd fd(DestFile, FileFd::ReadOnlyGzip);
1084       // Only test for correctness if the file is not empty (empty is ok)
1085       if (fd.Size() > 0)
1086       {
1087          pkgTagSection sec;
1088          pkgTagFile tag(&fd);
1089
1090          // all our current indexes have a field 'Package' in each section
1091          if (_error->PendingError() == true || tag.Step(sec) == false || sec.Exists("Package") == false)
1092          {
1093             RenameOnError(InvalidFormat);
1094             return;
1095          }
1096       }
1097
1098       // Done, move it into position
1099       string FinalFile = GetFinalFilename(RealURI, compExt);
1100       Rename(DestFile,FinalFile);
1101       chmod(FinalFile.c_str(),0644);
1102
1103       /* We restore the original name to DestFile so that the clean operation
1104          will work OK */
1105       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1106       DestFile += URItoFileName(RealURI);
1107       if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
1108          DestFile += ".gz";
1109
1110       // Remove the compressed version.
1111       if (Erase == true)
1112          unlink(DestFile.c_str());
1113       return;
1114    }
1115
1116    Erase = false;
1117    Complete = true;
1118
1119    // Handle the unzipd case
1120    string FileName = LookupTag(Message,"Alt-Filename");
1121    if (FileName.empty() == false)
1122    {
1123       Decompression = true;
1124       Local = true;
1125       DestFile += ".decomp";
1126       Desc.URI = "copy:" + FileName;
1127       QueueURI(Desc);
1128       Mode = "copy";
1129       return;
1130    }
1131
1132    FileName = LookupTag(Message,"Filename");
1133    if (FileName.empty() == true)
1134    {
1135       Status = StatError;
1136       ErrorText = "Method gave a blank filename";
1137    }
1138
1139    if (FileName == DestFile)
1140       Erase = true;
1141    else
1142       Local = true;
1143
1144    // The files timestamp matches, for non-local URLs reverify the local
1145    // file, for local file, uncompress again to ensure the hashsum is still
1146    // matching the Release file
1147    if (!Local && StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1148    {
1149       ReverifyAfterIMS(FileName);
1150       return;
1151    }
1152    string decompProg;
1153
1154    // If we enable compressed indexes, queue for hash verification
1155    if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local)
1156    {
1157       DestFile = _config->FindDir("Dir::State::lists");
1158       DestFile += URItoFileName(RealURI) + ".gz";
1159
1160       Decompression = true;
1161       Desc.URI = "copy:" + FileName;
1162       QueueURI(Desc);
1163
1164       return;
1165     }
1166
1167    // get the binary name for your used compression type
1168    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
1169    if(decompProg.empty() == false);
1170    else if(compExt == "uncompressed")
1171       decompProg = "copy";
1172    else {
1173       _error->Error("Unsupported extension: %s", compExt.c_str());
1174       return;
1175    }
1176
1177    Decompression = true;
1178    DestFile += ".decomp";
1179    Desc.URI = decompProg + ":" + FileName;
1180    QueueURI(Desc);
1181
1182    // FIXME: this points to a c++ string that goes out of scope
1183    Mode = decompProg.c_str();
1184 }
1185                                                                         /*}}}*/
1186 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
1187 // ---------------------------------------------------------------------
1188 /* The Translation file is added to the queue */
1189 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
1190                             string URI,string URIDesc,string ShortDesc)
1191   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
1192 {
1193 }
1194 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const *Target,
1195                          HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
1196   : pkgAcqIndex(Owner, Target, ExpectedHash, MetaIndexParser)
1197 {
1198 }
1199                                                                         /*}}}*/
1200 // AcqIndexTrans::Custom600Headers - Insert custom request headers      /*{{{*/
1201 // ---------------------------------------------------------------------
1202 string pkgAcqIndexTrans::Custom600Headers()
1203 {
1204    string Final = _config->FindDir("Dir::State::lists");
1205    Final += URItoFileName(RealURI);
1206
1207    if (_config->FindB("Acquire::GzipIndexes",false))
1208       Final += ".gz";
1209
1210    struct stat Buf;
1211    if (stat(Final.c_str(),&Buf) != 0)
1212       return "\nFail-Ignore: true\nIndex-File: true";
1213    return "\nFail-Ignore: true\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1214 }
1215                                                                         /*}}}*/
1216 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
1217 // ---------------------------------------------------------------------
1218 /* */
1219 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1220 {
1221    size_t const nextExt = CompressionExtension.find(' ');
1222    if (nextExt != std::string::npos)
1223    {
1224       CompressionExtension = CompressionExtension.substr(nextExt+1);
1225       Init(RealURI, Desc.Description, Desc.ShortDesc);
1226       Status = StatIdle;
1227       return;
1228    }
1229
1230    if (Cnf->LocalOnly == true ||
1231        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1232    {
1233       // Ignore this
1234       Status = StatDone;
1235       Complete = false;
1236       Dequeue();
1237       return;
1238    }
1239
1240    Item::Failed(Message,Cnf);
1241 }
1242                                                                         /*}}}*/
1243 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
1244                              string URI,string URIDesc,string ShortDesc,
1245                              string MetaIndexURI, string MetaIndexURIDesc,
1246                              string MetaIndexShortDesc,
1247                              const vector<IndexTarget*>* IndexTargets,
1248                              indexRecords* MetaIndexParser) :
1249    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
1250    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1251    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
1252 {
1253    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1254    DestFile += URItoFileName(URI);
1255
1256    // remove any partial downloaded sig-file in partial/.
1257    // it may confuse proxies and is too small to warrant a
1258    // partial download anyway
1259    unlink(DestFile.c_str());
1260
1261    // Create the item
1262    Desc.Description = URIDesc;
1263    Desc.Owner = this;
1264    Desc.ShortDesc = ShortDesc;
1265    Desc.URI = URI;
1266
1267    string Final = _config->FindDir("Dir::State::lists");
1268    Final += URItoFileName(RealURI);
1269    if (RealFileExists(Final) == true)
1270    {
1271       // File was already in place.  It needs to be re-downloaded/verified
1272       // because Release might have changed, we do give it a different
1273       // name than DestFile because otherwise the http method will
1274       // send If-Range requests and there are too many broken servers
1275       // out there that do not understand them
1276       LastGoodSig = DestFile+".reverify";
1277       Rename(Final,LastGoodSig);
1278    }
1279
1280    QueueURI(Desc);
1281 }
1282                                                                         /*}}}*/
1283 pkgAcqMetaSig::~pkgAcqMetaSig()                                         /*{{{*/
1284 {
1285    // if the file was never queued undo file-changes done in the constructor
1286    if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false &&
1287          LastGoodSig.empty() == false)
1288    {
1289       string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1290       if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true)
1291          Rename(LastGoodSig, Final);
1292    }
1293
1294 }
1295                                                                         /*}}}*/
1296 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
1297 // ---------------------------------------------------------------------
1298 /* The only header we use is the last-modified header. */
1299 string pkgAcqMetaSig::Custom600Headers()
1300 {
1301    struct stat Buf;
1302    if (stat(LastGoodSig.c_str(),&Buf) != 0)
1303       return "\nIndex-File: true";
1304
1305    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1306 }
1307
1308 void pkgAcqMetaSig::Done(string Message,unsigned long long Size,string MD5,
1309                          pkgAcquire::MethodConfig *Cfg)
1310 {
1311    Item::Done(Message,Size,MD5,Cfg);
1312
1313    string FileName = LookupTag(Message,"Filename");
1314    if (FileName.empty() == true)
1315    {
1316       Status = StatError;
1317       ErrorText = "Method gave a blank filename";
1318       return;
1319    }
1320
1321    if (FileName != DestFile)
1322    {
1323       // We have to copy it into place
1324       Local = true;
1325       Desc.URI = "copy:" + FileName;
1326       QueueURI(Desc);
1327       return;
1328    }
1329
1330    Complete = true;
1331
1332    // put the last known good file back on i-m-s hit (it will
1333    // be re-verified again)
1334    // Else do nothing, we have the new file in DestFile then
1335    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1336       Rename(LastGoodSig, DestFile);
1337
1338    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
1339    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc,
1340                        MetaIndexShortDesc,  DestFile, IndexTargets,
1341                        MetaIndexParser);
1342
1343 }
1344                                                                         /*}}}*/
1345 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
1346 {
1347    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1348
1349    // if we get a network error we fail gracefully
1350    if(Status == StatTransientNetworkError)
1351    {
1352       Item::Failed(Message,Cnf);
1353       // move the sigfile back on transient network failures
1354       if(FileExists(LastGoodSig))
1355          Rename(LastGoodSig,Final);
1356
1357       // set the status back to , Item::Failed likes to reset it
1358       Status = pkgAcquire::Item::StatTransientNetworkError;
1359       return;
1360    }
1361
1362    // Delete any existing sigfile when the acquire failed
1363    unlink(Final.c_str());
1364
1365    // queue a pkgAcqMetaIndex with no sigfile
1366    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1367                        "", IndexTargets, MetaIndexParser);
1368
1369    if (Cnf->LocalOnly == true ||
1370        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1371    {
1372       // Ignore this
1373       Status = StatDone;
1374       Complete = false;
1375       Dequeue();
1376       return;
1377    }
1378
1379    Item::Failed(Message,Cnf);
1380 }
1381                                                                         /*}}}*/
1382 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
1383                                  string URI,string URIDesc,string ShortDesc,
1384                                  string SigFile,
1385                                  const vector<struct IndexTarget*>* IndexTargets,
1386                                  indexRecords* MetaIndexParser) :
1387    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
1388    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
1389 {
1390    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1391    DestFile += URItoFileName(URI);
1392
1393    // Create the item
1394    Desc.Description = URIDesc;
1395    Desc.Owner = this;
1396    Desc.ShortDesc = ShortDesc;
1397    Desc.URI = URI;
1398
1399    QueueURI(Desc);
1400 }
1401                                                                         /*}}}*/
1402 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
1403 // ---------------------------------------------------------------------
1404 /* The only header we use is the last-modified header. */
1405 string pkgAcqMetaIndex::Custom600Headers()
1406 {
1407    string Final = _config->FindDir("Dir::State::lists");
1408    Final += URItoFileName(RealURI);
1409
1410    struct stat Buf;
1411    if (stat(Final.c_str(),&Buf) != 0)
1412       return "\nIndex-File: true";
1413
1414    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1415 }
1416                                                                         /*}}}*/
1417 void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,string Hash,  /*{{{*/
1418                            pkgAcquire::MethodConfig *Cfg)
1419 {
1420    Item::Done(Message,Size,Hash,Cfg);
1421
1422    // MetaIndexes are done in two passes: one to download the
1423    // metaindex with an appropriate method, and a second to verify it
1424    // with the gpgv method
1425
1426    if (AuthPass == true)
1427    {
1428       AuthDone(Message);
1429
1430       // all cool, move Release file into place
1431       Complete = true;
1432    }
1433    else
1434    {
1435       RetrievalDone(Message);
1436       if (!Complete)
1437          // Still more retrieving to do
1438          return;
1439
1440       if (SigFile == "")
1441       {
1442          // There was no signature file, so we are finished.  Download
1443          // the indexes and do only hashsum verification if possible
1444          MetaIndexParser->Load(DestFile);
1445          QueueIndexes(false);
1446       }
1447       else
1448       {
1449          // There was a signature file, so pass it to gpgv for
1450          // verification
1451
1452          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1453             std::cerr << "Metaindex acquired, queueing gpg verification ("
1454                       << SigFile << "," << DestFile << ")\n";
1455          AuthPass = true;
1456          Desc.URI = "gpgv:" + SigFile;
1457          QueueURI(Desc);
1458          Mode = "gpgv";
1459          return;
1460       }
1461    }
1462
1463    if (Complete == true)
1464    {
1465       string FinalFile = _config->FindDir("Dir::State::lists");
1466       FinalFile += URItoFileName(RealURI);
1467       if (SigFile == DestFile)
1468          SigFile = FinalFile;
1469       Rename(DestFile,FinalFile);
1470       chmod(FinalFile.c_str(),0644);
1471       DestFile = FinalFile;
1472    }
1473 }
1474                                                                         /*}}}*/
1475 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
1476 {
1477    // We have just finished downloading a Release file (it is not
1478    // verified yet)
1479
1480    string FileName = LookupTag(Message,"Filename");
1481    if (FileName.empty() == true)
1482    {
1483       Status = StatError;
1484       ErrorText = "Method gave a blank filename";
1485       return;
1486    }
1487
1488    if (FileName != DestFile)
1489    {
1490       Local = true;
1491       Desc.URI = "copy:" + FileName;
1492       QueueURI(Desc);
1493       return;
1494    }
1495
1496    // make sure to verify against the right file on I-M-S hit
1497    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1498    if(IMSHit)
1499    {
1500       string FinalFile = _config->FindDir("Dir::State::lists");
1501       FinalFile += URItoFileName(RealURI);
1502       if (SigFile == DestFile)
1503       {
1504          SigFile = FinalFile;
1505          // constructor of pkgAcqMetaClearSig moved it out of the way,
1506          // now move it back in on IMS hit for the 'old' file
1507          string const OldClearSig = DestFile + ".reverify";
1508          if (RealFileExists(OldClearSig) == true)
1509             Rename(OldClearSig, FinalFile);
1510       }
1511       DestFile = FinalFile;
1512    }
1513    Complete = true;
1514 }
1515                                                                         /*}}}*/
1516 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1517 {
1518    // At this point, the gpgv method has succeeded, so there is a
1519    // valid signature from a key in the trusted keyring.  We
1520    // perform additional verification of its contents, and use them
1521    // to verify the indexes we are about to download
1522
1523    if (!MetaIndexParser->Load(DestFile))
1524    {
1525       Status = StatAuthError;
1526       ErrorText = MetaIndexParser->ErrorText;
1527       return;
1528    }
1529
1530    if (!VerifyVendor(Message))
1531    {
1532       return;
1533    }
1534
1535    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1536       std::cerr << "Signature verification succeeded: "
1537                 << DestFile << std::endl;
1538
1539    // do not trust any previously unverified content that we may have
1540    string LastGoodSigFile = _config->FindDir("Dir::State::lists").append("partial/").append(URItoFileName(RealURI));
1541    if (DestFile != SigFile)
1542       LastGoodSigFile.append(".gpg");
1543    LastGoodSigFile.append(".reverify");
1544    if(IMSHit == false && RealFileExists(LastGoodSigFile) == false)
1545    {
1546       for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1547            Target != IndexTargets->end();
1548            ++Target)
1549       {
1550          // remove old indexes
1551          std::string index = _config->FindDir("Dir::State::lists") +
1552             URItoFileName((*Target)->URI);
1553          unlink(index.c_str());
1554          // and also old gzipindexes
1555          index += ".gz";
1556          unlink(index.c_str());
1557       }
1558    }
1559
1560
1561    // Download further indexes with verification
1562    QueueIndexes(true);
1563
1564    // is it a clearsigned MetaIndex file?
1565    if (DestFile == SigFile)
1566       return;
1567
1568    // Done, move signature file into position
1569    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1570       URItoFileName(RealURI) + ".gpg";
1571    Rename(SigFile,VerifiedSigFile);
1572    chmod(VerifiedSigFile.c_str(),0644);
1573 }
1574                                                                         /*}}}*/
1575 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1576 {
1577 #if 0
1578    /* Reject invalid, existing Release files (LP: #346386) (Closes: #627642)
1579     * FIXME: Disabled; it breaks unsigned repositories without hashes */
1580    if (!verify && FileExists(DestFile) && !MetaIndexParser->Load(DestFile))
1581    {
1582       Status = StatError;
1583       ErrorText = MetaIndexParser->ErrorText;
1584       return;
1585    }
1586 #endif
1587    bool transInRelease = false;
1588    {
1589       std::vector<std::string> const keys = MetaIndexParser->MetaKeys();
1590       for (std::vector<std::string>::const_iterator k = keys.begin(); k != keys.end(); ++k)
1591          // FIXME: Feels wrong to check for hardcoded string here, but what should we do else…
1592          if (k->find("Translation-") != std::string::npos)
1593          {
1594             transInRelease = true;
1595             break;
1596          }
1597    }
1598
1599    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1600         Target != IndexTargets->end();
1601         ++Target)
1602    {
1603       HashString ExpectedIndexHash;
1604       const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1605       bool compressedAvailable = false;
1606       if (Record == NULL)
1607       {
1608          if ((*Target)->IsOptional() == true)
1609          {
1610             std::vector<std::string> types = APT::Configuration::getCompressionTypes();
1611             for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
1612                if (MetaIndexParser->Exists((*Target)->MetaKey + "." + *t) == true)
1613                {
1614                   compressedAvailable = true;
1615                   break;
1616                }
1617          }
1618          else if (verify == true)
1619          {
1620             Status = StatAuthError;
1621             strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), (*Target)->MetaKey.c_str());
1622             return;
1623          }
1624       }
1625       else
1626       {
1627          ExpectedIndexHash = Record->Hash;
1628          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1629          {
1630             std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1631             std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1632             std::cerr << "For: " << Record->MetaKeyFilename << std::endl;
1633          }
1634          if (verify == true && ExpectedIndexHash.empty() == true && (*Target)->IsOptional() == false)
1635          {
1636             Status = StatAuthError;
1637             strprintf(ErrorText, _("Unable to find hash sum for '%s' in Release file"), (*Target)->MetaKey.c_str());
1638             return;
1639          }
1640       }
1641
1642       if ((*Target)->IsOptional() == true)
1643       {
1644          if ((*Target)->IsSubIndex() == true)
1645             new pkgAcqSubIndex(Owner, (*Target)->URI, (*Target)->Description,
1646                                 (*Target)->ShortDesc, ExpectedIndexHash);
1647          else if (transInRelease == false || Record != NULL || compressedAvailable == true)
1648          {
1649             if (_config->FindB("Acquire::PDiffs",true) == true && transInRelease == true &&
1650                 MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true)
1651                new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1652                                    (*Target)->ShortDesc, ExpectedIndexHash);
1653             else
1654                new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1655          }
1656          continue;
1657       }
1658
1659       /* Queue Packages file (either diff or full packages files, depending
1660          on the users option) - we also check if the PDiff Index file is listed
1661          in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
1662          instead, but passing the required info to it is to much hassle */
1663       if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
1664           MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true))
1665          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1666                              (*Target)->ShortDesc, ExpectedIndexHash);
1667       else
1668          new pkgAcqIndex(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1669    }
1670 }
1671                                                                         /*}}}*/
1672 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1673 {
1674    string::size_type pos;
1675
1676    // check for missing sigs (that where not fatal because otherwise we had
1677    // bombed earlier)
1678    string missingkeys;
1679    string msg = _("There is no public key available for the "
1680                   "following key IDs:\n");
1681    pos = Message.find("NO_PUBKEY ");
1682    if (pos != std::string::npos)
1683    {
1684       string::size_type start = pos+strlen("NO_PUBKEY ");
1685       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1686       missingkeys += (Fingerprint);
1687    }
1688    if(!missingkeys.empty())
1689       _error->Warning("%s", (msg + missingkeys).c_str());
1690
1691    string Transformed = MetaIndexParser->GetExpectedDist();
1692
1693    if (Transformed == "../project/experimental")
1694    {
1695       Transformed = "experimental";
1696    }
1697
1698    pos = Transformed.rfind('/');
1699    if (pos != string::npos)
1700    {
1701       Transformed = Transformed.substr(0, pos);
1702    }
1703
1704    if (Transformed == ".")
1705    {
1706       Transformed = "";
1707    }
1708
1709    if (_config->FindB("Acquire::Check-Valid-Until", true) == true &&
1710        MetaIndexParser->GetValidUntil() > 0) {
1711       time_t const invalid_since = time(NULL) - MetaIndexParser->GetValidUntil();
1712       if (invalid_since > 0)
1713          // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1714          // the time since then the file is invalid - formated in the same way as in
1715          // the download progress display (e.g. 7d 3h 42min 1s)
1716          return _error->Error(
1717             _("Release file for %s is expired (invalid since %s). "
1718               "Updates for this repository will not be applied."),
1719             RealURI.c_str(), TimeToStr(invalid_since).c_str());
1720    }
1721
1722    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1723    {
1724       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1725       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1726       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1727    }
1728
1729    if (MetaIndexParser->CheckDist(Transformed) == false)
1730    {
1731       // This might become fatal one day
1732 //       Status = StatAuthError;
1733 //       ErrorText = "Conflicting distribution; expected "
1734 //          + MetaIndexParser->GetExpectedDist() + " but got "
1735 //          + MetaIndexParser->GetDist();
1736 //       return false;
1737       if (!Transformed.empty())
1738       {
1739          _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1740                          Desc.Description.c_str(),
1741                          Transformed.c_str(),
1742                          MetaIndexParser->GetDist().c_str());
1743       }
1744    }
1745
1746    return true;
1747 }
1748                                                                         /*}}}*/
1749 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1750 // ---------------------------------------------------------------------
1751 /* */
1752 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)
1753 {
1754    if (AuthPass == true)
1755    {
1756       // gpgv method failed, if we have a good signature
1757       string LastGoodSigFile = _config->FindDir("Dir::State::lists").append("partial/").append(URItoFileName(RealURI));
1758       if (DestFile != SigFile)
1759          LastGoodSigFile.append(".gpg");
1760       LastGoodSigFile.append(".reverify");
1761
1762       if(FileExists(LastGoodSigFile))
1763       {
1764          string VerifiedSigFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1765          if (DestFile != SigFile)
1766             VerifiedSigFile.append(".gpg");
1767          Rename(LastGoodSigFile, VerifiedSigFile);
1768          Status = StatTransientNetworkError;
1769          _error->Warning(_("An error occurred during the signature "
1770                            "verification. The repository is not updated "
1771                            "and the previous index files will be used. "
1772                            "GPG error: %s: %s\n"),
1773                          Desc.Description.c_str(),
1774                          LookupTag(Message,"Message").c_str());
1775          RunScripts("APT::Update::Auth-Failure");
1776          return;
1777       } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) {
1778          /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */
1779          _error->Error(_("GPG error: %s: %s"),
1780                          Desc.Description.c_str(),
1781                          LookupTag(Message,"Message").c_str());
1782          return;
1783       } else {
1784          _error->Warning(_("GPG error: %s: %s"),
1785                          Desc.Description.c_str(),
1786                          LookupTag(Message,"Message").c_str());
1787       }
1788       // gpgv method failed
1789       ReportMirrorFailure("GPGFailure");
1790    }
1791
1792    /* Always move the meta index, even if gpgv failed. This ensures
1793     * that PackageFile objects are correctly filled in */
1794    if (FileExists(DestFile)) {
1795       string FinalFile = _config->FindDir("Dir::State::lists");
1796       FinalFile += URItoFileName(RealURI);
1797       /* InRelease files become Release files, otherwise
1798        * they would be considered as trusted later on */
1799       if (SigFile == DestFile) {
1800          RealURI = RealURI.replace(RealURI.rfind("InRelease"), 9,
1801                                        "Release");
1802          FinalFile = FinalFile.replace(FinalFile.rfind("InRelease"), 9,
1803                                        "Release");
1804          SigFile = FinalFile;
1805       }
1806       Rename(DestFile,FinalFile);
1807       chmod(FinalFile.c_str(),0644);
1808
1809       DestFile = FinalFile;
1810    }
1811
1812    // No Release file was present, or verification failed, so fall
1813    // back to queueing Packages files without verification
1814    QueueIndexes(false);
1815 }
1816                                                                         /*}}}*/
1817 pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner,               /*{{{*/
1818                 string const &URI, string const &URIDesc, string const &ShortDesc,
1819                 string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc,
1820                 string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc,
1821                 const vector<struct IndexTarget*>* IndexTargets,
1822                 indexRecords* MetaIndexParser) :
1823         pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, "", IndexTargets, MetaIndexParser),
1824         MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1825         MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc)
1826 {
1827    SigFile = DestFile;
1828
1829    // keep the old InRelease around in case of transistent network errors
1830    string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1831    if (RealFileExists(Final) == true)
1832    {
1833       string const LastGoodSig = DestFile + ".reverify";
1834       Rename(Final,LastGoodSig);
1835    }
1836 }
1837                                                                         /*}}}*/
1838 pkgAcqMetaClearSig::~pkgAcqMetaClearSig()                               /*{{{*/
1839 {
1840    // if the file was never queued undo file-changes done in the constructor
1841    if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false)
1842    {
1843       string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1844       string const LastGoodSig = DestFile + ".reverify";
1845       if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true)
1846          Rename(LastGoodSig, Final);
1847    }
1848 }
1849                                                                         /*}}}*/
1850 // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/
1851 // ---------------------------------------------------------------------
1852 // FIXME: this can go away once the InRelease file is used widely
1853 string pkgAcqMetaClearSig::Custom600Headers()
1854 {
1855    string Final = _config->FindDir("Dir::State::lists");
1856    Final += URItoFileName(RealURI);
1857
1858    struct stat Buf;
1859    if (stat(Final.c_str(),&Buf) != 0)
1860    {
1861       Final = DestFile + ".reverify";
1862       if (stat(Final.c_str(),&Buf) != 0)
1863          return "\nIndex-File: true\nFail-Ignore: true\n";
1864    }
1865
1866    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1867 }
1868                                                                         /*}}}*/
1869 void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/
1870 {
1871    if (AuthPass == false)
1872    {
1873       // Remove the 'old' InRelease file if we try Release.gpg now as otherwise
1874       // the file will stay around and gives a false-auth impression (CVE-2012-0214)
1875       string FinalFile = _config->FindDir("Dir::State::lists");
1876       FinalFile.append(URItoFileName(RealURI));
1877       if (FileExists(FinalFile))
1878          unlink(FinalFile.c_str());
1879
1880       new pkgAcqMetaSig(Owner,
1881                         MetaSigURI, MetaSigURIDesc, MetaSigShortDesc,
1882                         MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1883                         IndexTargets, MetaIndexParser);
1884       if (Cnf->LocalOnly == true ||
1885           StringToBool(LookupTag(Message, "Transient-Failure"), false) == false)
1886          Dequeue();
1887    }
1888    else
1889       pkgAcqMetaIndex::Failed(Message, Cnf);
1890 }
1891                                                                         /*}}}*/
1892 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1893 // ---------------------------------------------------------------------
1894 /* This just sets up the initial fetch environment and queues the first
1895    possibilitiy */
1896 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1897                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1898                              string &StoreFilename) :
1899                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1900                StoreFilename(StoreFilename), Vf(Version.FileList()),
1901                Trusted(false)
1902 {
1903    Retries = _config->FindI("Acquire::Retries",0);
1904
1905    if (Version.Arch() == 0)
1906    {
1907       _error->Error(_("I wasn't able to locate a file for the %s package. "
1908                       "This might mean you need to manually fix this package. "
1909                       "(due to missing arch)"),
1910                     Version.ParentPkg().FullName().c_str());
1911       return;
1912    }
1913
1914    /* We need to find a filename to determine the extension. We make the
1915       assumption here that all the available sources for this version share
1916       the same extension.. */
1917    // Skip not source sources, they do not have file fields.
1918    for (; Vf.end() == false; ++Vf)
1919    {
1920       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1921          continue;
1922       break;
1923    }
1924
1925    // Does not really matter here.. we are going to fail out below
1926    if (Vf.end() != true)
1927    {
1928       // If this fails to get a file name we will bomb out below.
1929       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1930       if (_error->PendingError() == true)
1931          return;
1932
1933       // Generate the final file name as: package_version_arch.foo
1934       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1935                       QuoteString(Version.VerStr(),"_:") + '_' +
1936                       QuoteString(Version.Arch(),"_:.") +
1937                       "." + flExtension(Parse.FileName());
1938    }
1939
1940    // check if we have one trusted source for the package. if so, switch
1941    // to "TrustedOnly" mode - but only if not in AllowUnauthenticated mode
1942    bool const allowUnauth = _config->FindB("APT::Get::AllowUnauthenticated", false);
1943    bool const debugAuth = _config->FindB("Debug::pkgAcquire::Auth", false);
1944    bool seenUntrusted = false;
1945    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; ++i)
1946    {
1947       pkgIndexFile *Index;
1948       if (Sources->FindIndex(i.File(),Index) == false)
1949          continue;
1950
1951       if (debugAuth == true)
1952          std::cerr << "Checking index: " << Index->Describe()
1953                    << "(Trusted=" << Index->IsTrusted() << ")" << std::endl;
1954
1955       if (Index->IsTrusted() == true)
1956       {
1957          Trusted = true;
1958          if (allowUnauth == false)
1959             break;
1960       }
1961       else
1962          seenUntrusted = true;
1963    }
1964
1965    // "allow-unauthenticated" restores apts old fetching behaviour
1966    // that means that e.g. unauthenticated file:// uris are higher
1967    // priority than authenticated http:// uris
1968    if (allowUnauth == true && seenUntrusted == true)
1969       Trusted = false;
1970
1971    // Select a source
1972    if (QueueNext() == false && _error->PendingError() == false)
1973       _error->Error(_("Can't find a source to download version '%s' of '%s'"),
1974                     Version.VerStr(), Version.ParentPkg().FullName(false).c_str());
1975 }
1976                                                                         /*}}}*/
1977 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1978 // ---------------------------------------------------------------------
1979 /* This queues the next available file version for download. It checks if
1980    the archive is already available in the cache and stashs the MD5 for
1981    checking later. */
1982 bool pkgAcqArchive::QueueNext()
1983 {
1984    string const ForceHash = _config->Find("Acquire::ForceHash");
1985    for (; Vf.end() == false; ++Vf)
1986    {
1987       // Ignore not source sources
1988       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1989          continue;
1990
1991       // Try to cross match against the source list
1992       pkgIndexFile *Index;
1993       if (Sources->FindIndex(Vf.File(),Index) == false)
1994             continue;
1995
1996       // only try to get a trusted package from another source if that source
1997       // is also trusted
1998       if(Trusted && !Index->IsTrusted())
1999          continue;
2000
2001       // Grab the text package record
2002       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
2003       if (_error->PendingError() == true)
2004          return false;
2005
2006       string PkgFile = Parse.FileName();
2007       if (ForceHash.empty() == false)
2008       {
2009          if(stringcasecmp(ForceHash, "sha512") == 0)
2010             ExpectedHash = HashString("SHA512", Parse.SHA512Hash());
2011          else if(stringcasecmp(ForceHash, "sha256") == 0)
2012             ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
2013          else if (stringcasecmp(ForceHash, "sha1") == 0)
2014             ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
2015          else
2016             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
2017       }
2018       else
2019       {
2020          string Hash;
2021          if ((Hash = Parse.SHA512Hash()).empty() == false)
2022             ExpectedHash = HashString("SHA512", Hash);
2023          else if ((Hash = Parse.SHA256Hash()).empty() == false)
2024             ExpectedHash = HashString("SHA256", Hash);
2025          else if ((Hash = Parse.SHA1Hash()).empty() == false)
2026             ExpectedHash = HashString("SHA1", Hash);
2027          else
2028             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
2029       }
2030       if (PkgFile.empty() == true)
2031          return _error->Error(_("The package index files are corrupted. No Filename: "
2032                               "field for package %s."),
2033                               Version.ParentPkg().Name());
2034
2035       Desc.URI = Index->ArchiveURI(PkgFile);
2036       Desc.Description = Index->ArchiveInfo(Version);
2037       Desc.Owner = this;
2038       Desc.ShortDesc = Version.ParentPkg().FullName(true);
2039
2040       // See if we already have the file. (Legacy filenames)
2041       FileSize = Version->Size;
2042       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
2043       struct stat Buf;
2044       if (stat(FinalFile.c_str(),&Buf) == 0)
2045       {
2046          // Make sure the size matches
2047          if ((unsigned long long)Buf.st_size == Version->Size)
2048          {
2049             Complete = true;
2050             Local = true;
2051             Status = StatDone;
2052             StoreFilename = DestFile = FinalFile;
2053             return true;
2054          }
2055
2056          /* Hmm, we have a file and its size does not match, this means it is
2057             an old style mismatched arch */
2058          unlink(FinalFile.c_str());
2059       }
2060
2061       // Check it again using the new style output filenames
2062       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
2063       if (stat(FinalFile.c_str(),&Buf) == 0)
2064       {
2065          // Make sure the size matches
2066          if ((unsigned long long)Buf.st_size == Version->Size)
2067          {
2068             Complete = true;
2069             Local = true;
2070             Status = StatDone;
2071             StoreFilename = DestFile = FinalFile;
2072             return true;
2073          }
2074
2075          /* Hmm, we have a file and its size does not match, this shouldn't
2076             happen.. */
2077          unlink(FinalFile.c_str());
2078       }
2079
2080       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
2081
2082       // Check the destination file
2083       if (stat(DestFile.c_str(),&Buf) == 0)
2084       {
2085          // Hmm, the partial file is too big, erase it
2086          if ((unsigned long long)Buf.st_size > Version->Size)
2087             unlink(DestFile.c_str());
2088          else
2089             PartialSize = Buf.st_size;
2090       }
2091
2092       // Disables download of archives - useful if no real installation follows,
2093       // e.g. if we are just interested in proposed installation order
2094       if (_config->FindB("Debug::pkgAcqArchive::NoQueue", false) == true)
2095       {
2096          Complete = true;
2097          Local = true;
2098          Status = StatDone;
2099          StoreFilename = DestFile = FinalFile;
2100          return true;
2101       }
2102
2103       // Create the item
2104       Local = false;
2105       QueueURI(Desc);
2106
2107       ++Vf;
2108       return true;
2109    }
2110    return false;
2111 }
2112                                                                         /*}}}*/
2113 // AcqArchive::Done - Finished fetching                                 /*{{{*/
2114 // ---------------------------------------------------------------------
2115 /* */
2116 void pkgAcqArchive::Done(string Message,unsigned long long Size,string CalcHash,
2117                          pkgAcquire::MethodConfig *Cfg)
2118 {
2119    Item::Done(Message,Size,CalcHash,Cfg);
2120
2121    // Check the size
2122    if (Size != Version->Size)
2123    {
2124       RenameOnError(SizeMismatch);
2125       return;
2126    }
2127
2128    // Check the hash
2129    if(ExpectedHash.toStr() != CalcHash)
2130    {
2131       RenameOnError(HashSumMismatch);
2132       return;
2133    }
2134
2135    // Grab the output filename
2136    string FileName = LookupTag(Message,"Filename");
2137    if (FileName.empty() == true)
2138    {
2139       Status = StatError;
2140       ErrorText = "Method gave a blank filename";
2141       return;
2142    }
2143
2144    Complete = true;
2145
2146    // Reference filename
2147    if (FileName != DestFile)
2148    {
2149       StoreFilename = DestFile = FileName;
2150       Local = true;
2151       return;
2152    }
2153
2154    // Done, move it into position
2155    string FinalFile = _config->FindDir("Dir::Cache::Archives");
2156    FinalFile += flNotDir(StoreFilename);
2157    Rename(DestFile,FinalFile);
2158
2159    StoreFilename = DestFile = FinalFile;
2160    Complete = true;
2161 }
2162                                                                         /*}}}*/
2163 // AcqArchive::Failed - Failure handler                                 /*{{{*/
2164 // ---------------------------------------------------------------------
2165 /* Here we try other sources */
2166 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2167 {
2168    ErrorText = LookupTag(Message,"Message");
2169
2170    /* We don't really want to retry on failed media swaps, this prevents
2171       that. An interesting observation is that permanent failures are not
2172       recorded. */
2173    if (Cnf->Removable == true &&
2174        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2175    {
2176       // Vf = Version.FileList();
2177       while (Vf.end() == false) ++Vf;
2178       StoreFilename = string();
2179       Item::Failed(Message,Cnf);
2180       return;
2181    }
2182
2183    if (QueueNext() == false)
2184    {
2185       // This is the retry counter
2186       if (Retries != 0 &&
2187           Cnf->LocalOnly == false &&
2188           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2189       {
2190          Retries--;
2191          Vf = Version.FileList();
2192          if (QueueNext() == true)
2193             return;
2194       }
2195
2196       StoreFilename = string();
2197       Item::Failed(Message,Cnf);
2198    }
2199 }
2200                                                                         /*}}}*/
2201 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
2202 // ---------------------------------------------------------------------
2203 APT_PURE bool pkgAcqArchive::IsTrusted()
2204 {
2205    return Trusted;
2206 }
2207                                                                         /*}}}*/
2208 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
2209 // ---------------------------------------------------------------------
2210 /* */
2211 void pkgAcqArchive::Finished()
2212 {
2213    if (Status == pkgAcquire::Item::StatDone &&
2214        Complete == true)
2215       return;
2216    StoreFilename = string();
2217 }
2218                                                                         /*}}}*/
2219 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
2220 // ---------------------------------------------------------------------
2221 /* The file is added to the queue */
2222 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
2223                        unsigned long long Size,string Dsc,string ShortDesc,
2224                        const string &DestDir, const string &DestFilename,
2225                        bool IsIndexFile) :
2226                        Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile)
2227 {
2228    Retries = _config->FindI("Acquire::Retries",0);
2229
2230    if(!DestFilename.empty())
2231       DestFile = DestFilename;
2232    else if(!DestDir.empty())
2233       DestFile = DestDir + "/" + flNotDir(URI);
2234    else
2235       DestFile = flNotDir(URI);
2236
2237    // Create the item
2238    Desc.URI = URI;
2239    Desc.Description = Dsc;
2240    Desc.Owner = this;
2241
2242    // Set the short description to the archive component
2243    Desc.ShortDesc = ShortDesc;
2244
2245    // Get the transfer sizes
2246    FileSize = Size;
2247    struct stat Buf;
2248    if (stat(DestFile.c_str(),&Buf) == 0)
2249    {
2250       // Hmm, the partial file is too big, erase it
2251       if ((Size > 0) && (unsigned long long)Buf.st_size > Size)
2252          unlink(DestFile.c_str());
2253       else
2254          PartialSize = Buf.st_size;
2255    }
2256
2257    QueueURI(Desc);
2258 }
2259                                                                         /*}}}*/
2260 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
2261 // ---------------------------------------------------------------------
2262 /* */
2263 void pkgAcqFile::Done(string Message,unsigned long long Size,string CalcHash,
2264                       pkgAcquire::MethodConfig *Cnf)
2265 {
2266    Item::Done(Message,Size,CalcHash,Cnf);
2267
2268    // Check the hash
2269    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
2270    {
2271       RenameOnError(HashSumMismatch);
2272       return;
2273    }
2274
2275    string FileName = LookupTag(Message,"Filename");
2276    if (FileName.empty() == true)
2277    {
2278       Status = StatError;
2279       ErrorText = "Method gave a blank filename";
2280       return;
2281    }
2282
2283    Complete = true;
2284
2285    // The files timestamp matches
2286    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
2287       return;
2288
2289    // We have to copy it into place
2290    if (FileName != DestFile)
2291    {
2292       Local = true;
2293       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
2294           Cnf->Removable == true)
2295       {
2296          Desc.URI = "copy:" + FileName;
2297          QueueURI(Desc);
2298          return;
2299       }
2300
2301       // Erase the file if it is a symlink so we can overwrite it
2302       struct stat St;
2303       if (lstat(DestFile.c_str(),&St) == 0)
2304       {
2305          if (S_ISLNK(St.st_mode) != 0)
2306             unlink(DestFile.c_str());
2307       }
2308
2309       // Symlink the file
2310       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
2311       {
2312          ErrorText = "Link to " + DestFile + " failure ";
2313          Status = StatError;
2314          Complete = false;
2315       }
2316    }
2317 }
2318                                                                         /*}}}*/
2319 // AcqFile::Failed - Failure handler                                    /*{{{*/
2320 // ---------------------------------------------------------------------
2321 /* Here we try other sources */
2322 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2323 {
2324    ErrorText = LookupTag(Message,"Message");
2325
2326    // This is the retry counter
2327    if (Retries != 0 &&
2328        Cnf->LocalOnly == false &&
2329        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2330    {
2331       Retries--;
2332       QueueURI(Desc);
2333       return;
2334    }
2335
2336    Item::Failed(Message,Cnf);
2337 }
2338                                                                         /*}}}*/
2339 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
2340 // ---------------------------------------------------------------------
2341 /* The only header we use is the last-modified header. */
2342 string pkgAcqFile::Custom600Headers()
2343 {
2344    if (IsIndexFile)
2345       return "\nIndex-File: true";
2346    return "";
2347 }
2348                                                                         /*}}}*/