apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <config.h>
  17
  18 #include <apt-pkg/acquire-item.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/aptconfiguration.h>
  21 #include <apt-pkg/sourcelist.h>
  22 #include <apt-pkg/error.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/sha1.h>
  26 #include <apt-pkg/tagfile.h>
  27 #include <apt-pkg/indexrecords.h>
  28 #include <apt-pkg/acquire.h>
  29 #include <apt-pkg/hashes.h>
  30 #include <apt-pkg/indexfile.h>
  31 #include <apt-pkg/pkgcache.h>
  32 #include <apt-pkg/cacheiterators.h>
  33 #include <apt-pkg/pkgrecords.h>
  34
  35 #include <stddef.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38 #include <iostream>
  39 #include <vector>
  40 #include <sys/stat.h>
  41 #include <unistd.h>
  42 #include <errno.h>
  43 #include <string>
  44 #include <sstream>
  45 #include <stdio.h>
  46 #include <ctime>
  47
  48 #include <apti18n.h>
  49                                                                         /*}}}*/
  50
  51 using namespace std;
  52
  53 // Acquire::Item::Item - Constructor                                    /*{{{*/
  54 // ---------------------------------------------------------------------
  55 /* */
  56 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  57                        PartialSize(0), Mode(0), ID(0), Complete(false),
  58                        Local(false), QueueCounter(0),
  59                        ExpectedAdditionalItems(0)
  60 {
  61    Owner->Add(this);
  62    Status = StatIdle;
  63 }
  64                                                                         /*}}}*/
  65 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  66 // ---------------------------------------------------------------------
  67 /* */
  68 pkgAcquire::Item::~Item()
  69 {
  70    Owner->Remove(this);
  71 }
  72                                                                         /*}}}*/
  73 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  74 // ---------------------------------------------------------------------
  75 /* We return to an idle state if there are still other queues that could
  76    fetch this object */
  77 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  78 {
  79    Status = StatIdle;
  80    ErrorText = LookupTag(Message,"Message");
  81    UsedMirror =  LookupTag(Message,"UsedMirror");
  82    if (QueueCounter <= 1)
  83    {
  84       /* This indicates that the file is not available right now but might
  85          be sometime later. If we do a retry cycle then this should be
  86          retried [CDROMs] */
  87       if (Cnf->LocalOnly == true &&
  88           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  89       {
  90          Status = StatIdle;
  91          Dequeue();
  92          return;
  93       }
  94
  95       Status = StatError;
  96       Dequeue();
  97    }
  98
  99    // report mirror failure back to LP if we actually use a mirror
 100    string FailReason = LookupTag(Message, "FailReason");
 101    if(FailReason.size() != 0)
 102       ReportMirrorFailure(FailReason);
 103    else
 104       ReportMirrorFailure(ErrorText);
 105 }
 106                                                                         /*}}}*/
 107 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
 108 // ---------------------------------------------------------------------
 109 /* Stash status and the file size. Note that setting Complete means
 110    sub-phases of the acquire process such as decompresion are operating */
 111 void pkgAcquire::Item::Start(string /*Message*/,unsigned long long Size)
 112 {
 113    Status = StatFetching;
 114    if (FileSize == 0 && Complete == false)
 115       FileSize = Size;
 116 }
 117                                                                         /*}}}*/
 118 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
 119 // ---------------------------------------------------------------------
 120 /* */
 121 void pkgAcquire::Item::Done(string Message,unsigned long long Size,string /*Hash*/,
 122                             pkgAcquire::MethodConfig * /*Cnf*/)
 123 {
 124    // We just downloaded something..
 125    string FileName = LookupTag(Message,"Filename");
 126    UsedMirror =  LookupTag(Message,"UsedMirror");
 127    if (Complete == false && !Local && FileName == DestFile)
 128    {
 129       if (Owner->Log != 0)
 130          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 131    }
 132
 133    if (FileSize == 0)
 134       FileSize= Size;
 135    Status = StatDone;
 136    ErrorText = string();
 137    Owner->Dequeue(this);
 138 }
 139                                                                         /*}}}*/
 140 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 141 // ---------------------------------------------------------------------
 142 /* This helper function is used by a lot of item methods as their final
 143    step */
 144 void pkgAcquire::Item::Rename(string From,string To)
 145 {
 146    if (rename(From.c_str(),To.c_str()) != 0)
 147    {
 148       char S[300];
 149       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 150               From.c_str(),To.c_str());
 151       Status = StatError;
 152       ErrorText = S;
 153    }
 154 }
 155                                                                         /*}}}*/
 156 bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const error)/*{{{*/
 157 {
 158    if(FileExists(DestFile))
 159       Rename(DestFile, DestFile + ".FAILED");
 160
 161    switch (error)
 162    {
 163       case HashSumMismatch:
 164          ErrorText = _("Hash Sum mismatch");
 165          Status = StatAuthError;
 166          ReportMirrorFailure("HashChecksumFailure");
 167          break;
 168       case SizeMismatch:
 169          ErrorText = _("Size mismatch");
 170          Status = StatAuthError;
 171          ReportMirrorFailure("SizeFailure");
 172          break;
 173       case InvalidFormat:
 174          ErrorText = _("Invalid file format");
 175          Status = StatError;
 176          // do not report as usually its not the mirrors fault, but Portal/Proxy
 177          break;
 178    }
 179    return false;
 180 }
 181                                                                         /*}}}*/
 182 // Acquire::Item::ReportMirrorFailure                                   /*{{{*/
 183 // ---------------------------------------------------------------------
 184 void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
 185 {
 186    // we only act if a mirror was used at all
 187    if(UsedMirror.empty())
 188       return;
 189 #if 0
 190    std::cerr << "\nReportMirrorFailure: "
 191              << UsedMirror
 192              << " Uri: " << DescURI()
 193              << " FailCode: "
 194              << FailCode << std::endl;
 195 #endif
 196    const char *Args[40];
 197    unsigned int i = 0;
 198    string report = _config->Find("Methods::Mirror::ProblemReporting",
 199                                  "/usr/lib/apt/apt-report-mirror-failure");
 200    if(!FileExists(report))
 201       return;
 202    Args[i++] = report.c_str();
 203    Args[i++] = UsedMirror.c_str();
 204    Args[i++] = DescURI().c_str();
 205    Args[i++] = FailCode.c_str();
 206    Args[i++] = NULL;
 207    pid_t pid = ExecFork();
 208    if(pid < 0)
 209    {
 210       _error->Error("ReportMirrorFailure Fork failed");
 211       return;
 212    }
 213    else if(pid == 0)
 214    {
 215       execvp(Args[0], (char**)Args);
 216       std::cerr << "Could not exec " << Args[0] << std::endl;
 217       _exit(100);
 218    }
 219    if(!ExecWait(pid, "report-mirror-failure"))
 220    {
 221       _error->Warning("Couldn't report problem to '%s'",
 222                       _config->Find("Methods::Mirror::ProblemReporting").c_str());
 223    }
 224 }
 225                                                                         /*}}}*/
 226 // AcqSubIndex::AcqSubIndex - Constructor                               /*{{{*/
 227 // ---------------------------------------------------------------------
 228 /* Get a sub-index file based on checksums from a 'master' file and
 229    possibly query additional files */
 230 pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, string const &URI,
 231                                  string const &URIDesc, string const &ShortDesc,
 232                                  HashString const &ExpectedHash)
 233    : Item(Owner), ExpectedHash(ExpectedHash)
 234 {
 235    /* XXX: Beware: Currently this class does nothing (of value) anymore ! */
 236    Debug = _config->FindB("Debug::pkgAcquire::SubIndex",false);
 237
 238    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 239    DestFile += URItoFileName(URI);
 240
 241    Desc.URI = URI;
 242    Desc.Description = URIDesc;
 243    Desc.Owner = this;
 244    Desc.ShortDesc = ShortDesc;
 245
 246    QueueURI(Desc);
 247
 248    if(Debug)
 249       std::clog << "pkgAcqSubIndex: " << Desc.URI << std::endl;
 250 }
 251                                                                         /*}}}*/
 252 // AcqSubIndex::Custom600Headers - Insert custom request headers        /*{{{*/
 253 // ---------------------------------------------------------------------
 254 /* The only header we use is the last-modified header. */
 255 string pkgAcqSubIndex::Custom600Headers()
 256 {
 257    string Final = _config->FindDir("Dir::State::lists");
 258    Final += URItoFileName(Desc.URI);
 259
 260    struct stat Buf;
 261    if (stat(Final.c_str(),&Buf) != 0)
 262       return "\nIndex-File: true\nFail-Ignore: true\n";
 263    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 264 }
 265                                                                         /*}}}*/
 266 void pkgAcqSubIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 267 {
 268    if(Debug)
 269       std::clog << "pkgAcqSubIndex failed: " << Desc.URI << " with " << Message << std::endl;
 270
 271    Complete = false;
 272    Status = StatDone;
 273    Dequeue();
 274
 275    // No good Index is provided
 276 }
 277                                                                         /*}}}*/
 278 void pkgAcqSubIndex::Done(string Message,unsigned long long Size,string Md5Hash,        /*{{{*/
 279                            pkgAcquire::MethodConfig *Cnf)
 280 {
 281    if(Debug)
 282       std::clog << "pkgAcqSubIndex::Done(): " << Desc.URI << std::endl;
 283
 284    string FileName = LookupTag(Message,"Filename");
 285    if (FileName.empty() == true)
 286    {
 287       Status = StatError;
 288       ErrorText = "Method gave a blank filename";
 289       return;
 290    }
 291
 292    if (FileName != DestFile)
 293    {
 294       Local = true;
 295       Desc.URI = "copy:" + FileName;
 296       QueueURI(Desc);
 297       return;
 298    }
 299
 300    Item::Done(Message,Size,Md5Hash,Cnf);
 301
 302    string FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(Desc.URI);
 303
 304    /* Downloaded invalid transindex => Error (LP: #346386) (Closes: #627642) */
 305    indexRecords SubIndexParser;
 306    if (FileExists(DestFile) == true && !SubIndexParser.Load(DestFile)) {
 307       Status = StatError;
 308       ErrorText = SubIndexParser.ErrorText;
 309       return;
 310    }
 311
 312    // success in downloading the index
 313    // rename the index
 314    if(Debug)
 315       std::clog << "Renaming: " << DestFile << " -> " << FinalFile << std::endl;
 316    Rename(DestFile,FinalFile);
 317    chmod(FinalFile.c_str(),0644);
 318    DestFile = FinalFile;
 319
 320    if(ParseIndex(DestFile) == false)
 321       return Failed("", NULL);
 322
 323    Complete = true;
 324    Status = StatDone;
 325    Dequeue();
 326    return;
 327 }
 328                                                                         /*}}}*/
 329 bool pkgAcqSubIndex::ParseIndex(string const &IndexFile)                /*{{{*/
 330 {
 331    indexRecords SubIndexParser;
 332    if (FileExists(IndexFile) == false || SubIndexParser.Load(IndexFile) == false)
 333       return false;
 334    // so something with the downloaded index
 335    return true;
 336 }
 337                                                                         /*}}}*/
 338 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 339 // ---------------------------------------------------------------------
 340 /* Get the DiffIndex file first and see if there are patches available
 341  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 342  * patches. If anything goes wrong in that process, it will fall back to
 343  * the original packages file
 344  */
 345 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 346                                  string URI,string URIDesc,string ShortDesc,
 347                                  HashString ExpectedHash)
 348    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 349      Description(URIDesc)
 350 {
 351
 352    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 353
 354    Desc.Description = URIDesc + "/DiffIndex";
 355    Desc.Owner = this;
 356    Desc.ShortDesc = ShortDesc;
 357    Desc.URI = URI + ".diff/Index";
 358
 359    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 360    DestFile += URItoFileName(URI) + string(".DiffIndex");
 361
 362    if(Debug)
 363       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 364
 365    // look for the current package file
 366    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 367    CurrentPackagesFile += URItoFileName(RealURI);
 368
 369    // FIXME: this file:/ check is a hack to prevent fetching
 370    //        from local sources. this is really silly, and
 371    //        should be fixed cleanly as soon as possible
 372    if(!FileExists(CurrentPackagesFile) ||
 373       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 374    {
 375       // we don't have a pkg file or we don't want to queue
 376       if(Debug)
 377          std::clog << "No index file, local or canceld by user" << std::endl;
 378       Failed("", NULL);
 379       return;
 380    }
 381
 382    if(Debug)
 383       std::clog << "pkgAcqDiffIndex::pkgAcqDiffIndex(): "
 384          << CurrentPackagesFile << std::endl;
 385
 386    QueueURI(Desc);
 387
 388 }
 389                                                                         /*}}}*/
 390 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 391 // ---------------------------------------------------------------------
 392 /* The only header we use is the last-modified header. */
 393 string pkgAcqDiffIndex::Custom600Headers()
 394 {
 395    string Final = _config->FindDir("Dir::State::lists");
 396    Final += URItoFileName(RealURI) + string(".IndexDiff");
 397
 398    if(Debug)
 399       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 400
 401    struct stat Buf;
 402    if (stat(Final.c_str(),&Buf) != 0)
 403       return "\nIndex-File: true";
 404
 405    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 406 }
 407                                                                         /*}}}*/
 408 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 409 {
 410    if(Debug)
 411       std::clog << "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile
 412          << std::endl;
 413
 414    pkgTagSection Tags;
 415    string ServerSha1;
 416    vector<DiffInfo> available_patches;
 417
 418    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 419    pkgTagFile TF(&Fd);
 420    if (_error->PendingError() == true)
 421       return false;
 422
 423    if(TF.Step(Tags) == true)
 424    {
 425       bool found = false;
 426       DiffInfo d;
 427       string size;
 428
 429       string const tmp = Tags.FindS("SHA1-Current");
 430       std::stringstream ss(tmp);
 431       ss >> ServerSha1 >> size;
 432       unsigned long const ServerSize = atol(size.c_str());
 433
 434       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 435       SHA1Summation SHA1;
 436       SHA1.AddFD(fd);
 437       string const local_sha1 = SHA1.Result();
 438
 439       if(local_sha1 == ServerSha1)
 440       {
 441          // we have the same sha1 as the server so we are done here
 442          if(Debug)
 443             std::clog << "Package file is up-to-date" << std::endl;
 444          // list cleanup needs to know that this file as well as the already
 445          // present index is ours, so we create an empty diff to save it for us
 446          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 447                ExpectedHash, ServerSha1, available_patches);
 448          return true;
 449       }
 450       else
 451       {
 452          if(Debug)
 453             std::clog << "SHA1-Current: " << ServerSha1 << " and we start at "<< fd.Name() << " " << fd.Size() << " " << local_sha1 << std::endl;
 454
 455          // check the historie and see what patches we need
 456          string const history = Tags.FindS("SHA1-History");
 457          std::stringstream hist(history);
 458          while(hist >> d.sha1 >> size >> d.file)
 459          {
 460             // read until the first match is found
 461             // from that point on, we probably need all diffs
 462             if(d.sha1 == local_sha1)
 463                found=true;
 464             else if (found == false)
 465                continue;
 466
 467             if(Debug)
 468                std::clog << "Need to get diff: " << d.file << std::endl;
 469             available_patches.push_back(d);
 470          }
 471
 472          if (available_patches.empty() == false)
 473          {
 474             // patching with too many files is rather slow compared to a fast download
 475             unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 476             if (fileLimit != 0 && fileLimit < available_patches.size())
 477             {
 478                if (Debug)
 479                   std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 480                         << ") so fallback to complete download" << std::endl;
 481                return false;
 482             }
 483
 484             // see if the patches are too big
 485             found = false; // it was true and it will be true again at the end
 486             d = *available_patches.begin();
 487             string const firstPatch = d.file;
 488             unsigned long patchesSize = 0;
 489             std::stringstream patches(Tags.FindS("SHA1-Patches"));
 490             while(patches >> d.sha1 >> size >> d.file)
 491             {
 492                if (firstPatch == d.file)
 493                   found = true;
 494                else if (found == false)
 495                   continue;
 496
 497                patchesSize += atol(size.c_str());
 498             }
 499             unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 500             if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 501             {
 502                if (Debug)
 503                   std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 504                         << ") so fallback to complete download" << std::endl;
 505                return false;
 506             }
 507          }
 508       }
 509
 510       // we have something, queue the next diff
 511       if(found)
 512       {
 513          // queue the diffs
 514          string::size_type const last_space = Description.rfind(" ");
 515          if(last_space != string::npos)
 516             Description.erase(last_space, Description.size()-last_space);
 517
 518          /* decide if we should download patches one by one or in one go:
 519             The first is good if the server merges patches, but many don't so client
 520             based merging can be attempt in which case the second is better.
 521             "bad things" will happen if patches are merged on the server,
 522             but client side merging is attempt as well */
 523          bool pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true);
 524          if (pdiff_merge == true)
 525          {
 526             // reprepro adds this flag if it has merged patches on the server
 527             std::string const precedence = Tags.FindS("X-Patch-Precedence");
 528             pdiff_merge = (precedence != "merged");
 529          }
 530
 531          if (pdiff_merge == false)
 532             new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 533                   ExpectedHash, ServerSha1, available_patches);
 534          else
 535          {
 536             std::vector<pkgAcqIndexMergeDiffs*> *diffs = new std::vector<pkgAcqIndexMergeDiffs*>(available_patches.size());
 537             for(size_t i = 0; i < available_patches.size(); ++i)
 538                (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, RealURI, Description, Desc.ShortDesc, ExpectedHash,
 539                      available_patches[i], diffs);
 540          }
 541
 542          Complete = false;
 543          Status = StatDone;
 544          Dequeue();
 545          return true;
 546       }
 547    }
 548
 549    // Nothing found, report and return false
 550    // Failing here is ok, if we return false later, the full
 551    // IndexFile is queued
 552    if(Debug)
 553       std::clog << "Can't find a patch in the index file" << std::endl;
 554    return false;
 555 }
 556                                                                         /*}}}*/
 557 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 558 {
 559    if(Debug)
 560       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl
 561                 << "Falling back to normal index file acquire" << std::endl;
 562
 563    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 564                    ExpectedHash);
 565
 566    Complete = false;
 567    Status = StatDone;
 568    Dequeue();
 569 }
 570                                                                         /*}}}*/
 571 void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,string Md5Hash,       /*{{{*/
 572                            pkgAcquire::MethodConfig *Cnf)
 573 {
 574    if(Debug)
 575       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 576
 577    Item::Done(Message,Size,Md5Hash,Cnf);
 578
 579    string FinalFile;
 580    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 581
 582    // success in downloading the index
 583    // rename the index
 584    FinalFile += string(".IndexDiff");
 585    if(Debug)
 586       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 587                 << std::endl;
 588    Rename(DestFile,FinalFile);
 589    chmod(FinalFile.c_str(),0644);
 590    DestFile = FinalFile;
 591
 592    if(!ParseDiffIndex(DestFile))
 593       return Failed("", NULL);
 594
 595    Complete = true;
 596    Status = StatDone;
 597    Dequeue();
 598    return;
 599 }
 600                                                                         /*}}}*/
 601 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 602 // ---------------------------------------------------------------------
 603 /* The package diff is added to the queue. one object is constructed
 604  * for each diff and the index
 605  */
 606 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 607                                    string URI,string URIDesc,string ShortDesc,
 608                                    HashString ExpectedHash,
 609                                    string ServerSha1,
 610                                    vector<DiffInfo> diffs)
 611    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 612      available_patches(diffs), ServerSha1(ServerSha1)
 613 {
 614
 615    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 616    DestFile += URItoFileName(URI);
 617
 618    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 619
 620    Description = URIDesc;
 621    Desc.Owner = this;
 622    Desc.ShortDesc = ShortDesc;
 623
 624    if(available_patches.empty() == true)
 625    {
 626       // we are done (yeah!)
 627       Finish(true);
 628    }
 629    else
 630    {
 631       // get the next diff
 632       State = StateFetchDiff;
 633       QueueNextDiff();
 634    }
 635 }
 636                                                                         /*}}}*/
 637 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 638 {
 639    if(Debug)
 640       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl
 641                 << "Falling back to normal index file acquire" << std::endl;
 642    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 643                    ExpectedHash);
 644    Finish();
 645 }
 646                                                                         /*}}}*/
 647 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 648 void pkgAcqIndexDiffs::Finish(bool allDone)
 649 {
 650    // we restore the original name, this is required, otherwise
 651    // the file will be cleaned
 652    if(allDone)
 653    {
 654       DestFile = _config->FindDir("Dir::State::lists");
 655       DestFile += URItoFileName(RealURI);
 656
 657       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 658       {
 659          RenameOnError(HashSumMismatch);
 660          Dequeue();
 661          return;
 662       }
 663
 664       // this is for the "real" finish
 665       Complete = true;
 666       Status = StatDone;
 667       Dequeue();
 668       if(Debug)
 669          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 670       return;
 671    }
 672
 673    if(Debug)
 674       std::clog << "Finishing: " << Desc.URI << std::endl;
 675    Complete = false;
 676    Status = StatDone;
 677    Dequeue();
 678    return;
 679 }
 680                                                                         /*}}}*/
 681 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 682 {
 683
 684    // calc sha1 of the just patched file
 685    string FinalFile = _config->FindDir("Dir::State::lists");
 686    FinalFile += URItoFileName(RealURI);
 687
 688    FileFd fd(FinalFile, FileFd::ReadOnly);
 689    SHA1Summation SHA1;
 690    SHA1.AddFD(fd);
 691    string local_sha1 = string(SHA1.Result());
 692    if(Debug)
 693       std::clog << "QueueNextDiff: "
 694                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 695
 696    // final file reached before all patches are applied
 697    if(local_sha1 == ServerSha1)
 698    {
 699       Finish(true);
 700       return true;
 701    }
 702
 703    // remove all patches until the next matching patch is found
 704    // this requires the Index file to be ordered
 705    for(vector<DiffInfo>::iterator I=available_patches.begin();
 706        available_patches.empty() == false &&
 707           I != available_patches.end() &&
 708           I->sha1 != local_sha1;
 709        ++I)
 710    {
 711       available_patches.erase(I);
 712    }
 713
 714    // error checking and falling back if no patch was found
 715    if(available_patches.empty() == true)
 716    {
 717       Failed("", NULL);
 718       return false;
 719    }
 720
 721    // queue the right diff
 722    Desc.URI = RealURI + ".diff/" + available_patches[0].file + ".gz";
 723    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 724    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 725    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 726
 727    if(Debug)
 728       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 729
 730    QueueURI(Desc);
 731
 732    return true;
 733 }
 734                                                                         /*}}}*/
 735 void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size,string Md5Hash,      /*{{{*/
 736                             pkgAcquire::MethodConfig *Cnf)
 737 {
 738    if(Debug)
 739       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 740
 741    Item::Done(Message,Size,Md5Hash,Cnf);
 742
 743    string FinalFile;
 744    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 745
 746    // success in downloading a diff, enter ApplyDiff state
 747    if(State == StateFetchDiff)
 748    {
 749
 750       // rred excepts the patch as $FinalFile.ed
 751       Rename(DestFile,FinalFile+".ed");
 752
 753       if(Debug)
 754          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 755
 756       State = StateApplyDiff;
 757       Local = true;
 758       Desc.URI = "rred:" + FinalFile;
 759       QueueURI(Desc);
 760       Mode = "rred";
 761       return;
 762    }
 763
 764
 765    // success in download/apply a diff, queue next (if needed)
 766    if(State == StateApplyDiff)
 767    {
 768       // remove the just applied patch
 769       available_patches.erase(available_patches.begin());
 770       unlink((FinalFile + ".ed").c_str());
 771
 772       // move into place
 773       if(Debug)
 774       {
 775          std::clog << "Moving patched file in place: " << std::endl
 776                    << DestFile << " -> " << FinalFile << std::endl;
 777       }
 778       Rename(DestFile,FinalFile);
 779       chmod(FinalFile.c_str(),0644);
 780
 781       // see if there is more to download
 782       if(available_patches.empty() == false) {
 783          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 784                               ExpectedHash, ServerSha1, available_patches);
 785          return Finish();
 786       } else
 787          return Finish(true);
 788    }
 789 }
 790                                                                         /*}}}*/
 791 // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor                 /*{{{*/
 792 pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner,
 793                                    string const &URI, string const &URIDesc,
 794                                    string const &ShortDesc, HashString const &ExpectedHash,
 795                                    DiffInfo const &patch,
 796                                    std::vector<pkgAcqIndexMergeDiffs*> const * const allPatches)
 797    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 798      patch(patch),allPatches(allPatches), State(StateFetchDiff)
 799 {
 800
 801    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 802    DestFile += URItoFileName(URI);
 803
 804    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 805
 806    Description = URIDesc;
 807    Desc.Owner = this;
 808    Desc.ShortDesc = ShortDesc;
 809
 810    Desc.URI = RealURI + ".diff/" + patch.file + ".gz";
 811    Desc.Description = Description + " " + patch.file + string(".pdiff");
 812    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 813    DestFile += URItoFileName(RealURI + ".diff/" + patch.file);
 814
 815    if(Debug)
 816       std::clog << "pkgAcqIndexMergeDiffs: " << Desc.URI << std::endl;
 817
 818    QueueURI(Desc);
 819 }
 820                                                                         /*}}}*/
 821 void pkgAcqIndexMergeDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 822 {
 823    if(Debug)
 824       std::clog << "pkgAcqIndexMergeDiffs failed: " << Desc.URI << " with " << Message << std::endl;
 825    Complete = false;
 826    Status = StatDone;
 827    Dequeue();
 828
 829    // check if we are the first to fail, otherwise we are done here
 830    State = StateDoneDiff;
 831    for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 832          I != allPatches->end(); ++I)
 833       if ((*I)->State == StateErrorDiff)
 834          return;
 835
 836    // first failure means we should fallback
 837    State = StateErrorDiff;
 838    std::clog << "Falling back to normal index file acquire" << std::endl;
 839    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 840                    ExpectedHash);
 841 }
 842                                                                         /*}}}*/
 843 void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,string Md5Hash, /*{{{*/
 844                             pkgAcquire::MethodConfig *Cnf)
 845 {
 846    if(Debug)
 847       std::clog << "pkgAcqIndexMergeDiffs::Done(): " << Desc.URI << std::endl;
 848
 849    Item::Done(Message,Size,Md5Hash,Cnf);
 850
 851    string const FinalFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 852
 853    if (State == StateFetchDiff)
 854    {
 855       // rred expects the patch as $FinalFile.ed.$patchname.gz
 856       Rename(DestFile, FinalFile + ".ed." + patch.file + ".gz");
 857
 858       // check if this is the last completed diff
 859       State = StateDoneDiff;
 860       for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 861             I != allPatches->end(); ++I)
 862          if ((*I)->State != StateDoneDiff)
 863          {
 864             if(Debug)
 865                std::clog << "Not the last done diff in the batch: " << Desc.URI << std::endl;
 866             return;
 867          }
 868
 869       // this is the last completed diff, so we are ready to apply now
 870       State = StateApplyDiff;
 871
 872       if(Debug)
 873          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 874
 875       Local = true;
 876       Desc.URI = "rred:" + FinalFile;
 877       QueueURI(Desc);
 878       Mode = "rred";
 879       return;
 880    }
 881    // success in download/apply all diffs, clean up
 882    else if (State == StateApplyDiff)
 883    {
 884       // see if we really got the expected file
 885       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 886       {
 887          RenameOnError(HashSumMismatch);
 888          return;
 889       }
 890
 891       // move the result into place
 892       if(Debug)
 893          std::clog << "Moving patched file in place: " << std::endl
 894                    << DestFile << " -> " << FinalFile << std::endl;
 895       Rename(DestFile, FinalFile);
 896       chmod(FinalFile.c_str(), 0644);
 897
 898       // otherwise lists cleanup will eat the file
 899       DestFile = FinalFile;
 900
 901       // ensure the ed's are gone regardless of list-cleanup
 902       for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 903             I != allPatches->end(); ++I)
 904       {
 905             std::string patch = FinalFile + ".ed." + (*I)->patch.file + ".gz";
 906             unlink(patch.c_str());
 907       }
 908
 909       // all set and done
 910       Complete = true;
 911       if(Debug)
 912          std::clog << "allDone: " << DestFile << "\n" << std::endl;
 913    }
 914 }
 915                                                                         /*}}}*/
 916 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 917 // ---------------------------------------------------------------------
 918 /* The package file is added to the queue and a second class is
 919    instantiated to fetch the revision file */
 920 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 921                          string URI,string URIDesc,string ShortDesc,
 922                          HashString ExpectedHash, string comprExt)
 923    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 924 {
 925    if(comprExt.empty() == true)
 926    {
 927       // autoselect the compression method
 928       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 929       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 930          comprExt.append(*t).append(" ");
 931       if (comprExt.empty() == false)
 932          comprExt.erase(comprExt.end()-1);
 933    }
 934    CompressionExtension = comprExt;
 935
 936    Init(URI, URIDesc, ShortDesc);
 937 }
 938 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target,
 939                          HashString const &ExpectedHash, indexRecords *MetaIndexParser)
 940    : Item(Owner), RealURI(Target->URI), ExpectedHash(ExpectedHash)
 941 {
 942    // autoselect the compression method
 943    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 944    CompressionExtension = "";
 945    if (ExpectedHash.empty() == false)
 946    {
 947       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 948          if (*t == "uncompressed" || MetaIndexParser->Exists(string(Target->MetaKey).append(".").append(*t)) == true)
 949             CompressionExtension.append(*t).append(" ");
 950    }
 951    else
 952    {
 953       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 954          CompressionExtension.append(*t).append(" ");
 955    }
 956    if (CompressionExtension.empty() == false)
 957       CompressionExtension.erase(CompressionExtension.end()-1);
 958
 959    // only verify non-optional targets, see acquire-item.h for a FIXME
 960    // to make this more flexible
 961    if (Target->IsOptional())
 962      Verify = false;
 963    else
 964      Verify = true;
 965
 966    // FIXME: use the appropriate compression Extension
 967    // load the filesize
 968    indexRecords::checkSum *Record = MetaIndexParser->Lookup(string(Target->MetaKey));
 969    if(Record)
 970       FileSize = Record->Size;
 971
 972    Init(Target->URI, Target->Description, Target->ShortDesc);
 973 }
 974                                                                         /*}}}*/
 975 // AcqIndex::Init - defered Constructor                                 /*{{{*/
 976 void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &ShortDesc) {
 977    Decompression = false;
 978    Erase = false;
 979
 980    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 981    DestFile += URItoFileName(URI);
 982
 983    std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
 984    if (comprExt == "uncompressed")
 985       Desc.URI = URI;
 986    else
 987       Desc.URI = URI + '.' + comprExt;
 988
 989    Desc.Description = URIDesc;
 990    Desc.Owner = this;
 991    Desc.ShortDesc = ShortDesc;
 992
 993    QueueURI(Desc);
 994 }
 995                                                                         /*}}}*/
 996 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 997 // ---------------------------------------------------------------------
 998 /* The only header we use is the last-modified header. */
 999 string pkgAcqIndex::Custom600Headers()
1000 {
1001    string Final = _config->FindDir("Dir::State::lists");
1002    Final += URItoFileName(RealURI);
1003    if (_config->FindB("Acquire::GzipIndexes",false))
1004       Final += ".gz";
1005
1006    string msg = "\nIndex-File: true";
1007    // FIXME: this really should use "IndexTarget::IsOptional()" but that
1008    //        seems to be difficult without breaking ABI
1009    if (ShortDesc().find("Translation") != 0)
1010       msg += "\nFail-Ignore: true";
1011    struct stat Buf;
1012    if (stat(Final.c_str(),&Buf) == 0)
1013       msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1014
1015    return msg;
1016 }
1017                                                                         /*}}}*/
1018 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
1019 {
1020    size_t const nextExt = CompressionExtension.find(' ');
1021    if (nextExt != std::string::npos)
1022    {
1023       CompressionExtension = CompressionExtension.substr(nextExt+1);
1024       Init(RealURI, Desc.Description, Desc.ShortDesc);
1025       return;
1026    }
1027
1028    // on decompression failure, remove bad versions in partial/
1029    if (Decompression && Erase) {
1030       string s = _config->FindDir("Dir::State::lists") + "partial/";
1031       s.append(URItoFileName(RealURI));
1032       unlink(s.c_str());
1033    }
1034
1035    Item::Failed(Message,Cnf);
1036 }
1037                                                                         /*}}}*/
1038 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
1039 // ---------------------------------------------------------------------
1040 /* This goes through a number of states.. On the initial fetch the
1041    method could possibly return an alternate filename which points
1042    to the uncompressed version of the file. If this is so the file
1043    is copied into the partial directory. In all other cases the file
1044    is decompressed with a gzip uri. */
1045 void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash,
1046                        pkgAcquire::MethodConfig *Cfg)
1047 {
1048    Item::Done(Message,Size,Hash,Cfg);
1049
1050    if (Decompression == true)
1051    {
1052       if (_config->FindB("Debug::pkgAcquire::Auth", false))
1053       {
1054          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
1055          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
1056       }
1057
1058       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
1059       {
1060          RenameOnError(HashSumMismatch);
1061          return;
1062       }
1063
1064       /* Verify the index file for correctness (all indexes must
1065        * have a Package field) (LP: #346386) (Closes: #627642) */
1066       if (Verify == true)
1067       {
1068          FileFd fd(DestFile, FileFd::ReadOnly);
1069          // Only test for correctness if the file is not empty (empty is ok)
1070          if (fd.FileSize() > 0)
1071          {
1072             pkgTagSection sec;
1073             pkgTagFile tag(&fd);
1074
1075             // all our current indexes have a field 'Package' in each section
1076             if (_error->PendingError() == true || tag.Step(sec) == false || sec.Exists("Package") == false)
1077             {
1078                RenameOnError(InvalidFormat);
1079                return;
1080             }
1081          }
1082       }
1083
1084       // Done, move it into position
1085       string FinalFile = _config->FindDir("Dir::State::lists");
1086       FinalFile += URItoFileName(RealURI);
1087       Rename(DestFile,FinalFile);
1088       chmod(FinalFile.c_str(),0644);
1089
1090       /* We restore the original name to DestFile so that the clean operation
1091          will work OK */
1092       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1093       DestFile += URItoFileName(RealURI);
1094
1095       // Remove the compressed version.
1096       if (Erase == true)
1097          unlink(DestFile.c_str());
1098       return;
1099    }
1100
1101    Erase = false;
1102    Complete = true;
1103
1104    // Handle the unzipd case
1105    string FileName = LookupTag(Message,"Alt-Filename");
1106    if (FileName.empty() == false)
1107    {
1108       // The files timestamp matches
1109       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
1110          return;
1111       Decompression = true;
1112       Local = true;
1113       DestFile += ".decomp";
1114       Desc.URI = "copy:" + FileName;
1115       QueueURI(Desc);
1116       Mode = "copy";
1117       return;
1118    }
1119
1120    FileName = LookupTag(Message,"Filename");
1121    if (FileName.empty() == true)
1122    {
1123       Status = StatError;
1124       ErrorText = "Method gave a blank filename";
1125    }
1126
1127    std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
1128
1129    // The files timestamp matches
1130    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) {
1131        if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
1132           // Update DestFile for .gz suffix so that the clean operation keeps it
1133           DestFile += ".gz";
1134       return;
1135     }
1136
1137    if (FileName == DestFile)
1138       Erase = true;
1139    else
1140       Local = true;
1141
1142    string decompProg;
1143
1144    // If we enable compressed indexes and already have gzip, keep it
1145    if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) {
1146       string FinalFile = _config->FindDir("Dir::State::lists");
1147       FinalFile += URItoFileName(RealURI) + ".gz";
1148       Rename(DestFile,FinalFile);
1149       chmod(FinalFile.c_str(),0644);
1150
1151       // Update DestFile for .gz suffix so that the clean operation keeps it
1152       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1153       DestFile += URItoFileName(RealURI) + ".gz";
1154       return;
1155     }
1156
1157    // get the binary name for your used compression type
1158    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
1159    if(decompProg.empty() == false);
1160    else if(compExt == "uncompressed")
1161       decompProg = "copy";
1162    else {
1163       _error->Error("Unsupported extension: %s", compExt.c_str());
1164       return;
1165    }
1166
1167    Decompression = true;
1168    DestFile += ".decomp";
1169    Desc.URI = decompProg + ":" + FileName;
1170    QueueURI(Desc);
1171
1172    // FIXME: this points to a c++ string that goes out of scope
1173    Mode = decompProg.c_str();
1174 }
1175                                                                         /*}}}*/
1176 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
1177 // ---------------------------------------------------------------------
1178 /* The Translation file is added to the queue */
1179 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
1180                             string URI,string URIDesc,string ShortDesc)
1181   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
1182 {
1183 }
1184 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const *Target,
1185                          HashString const &ExpectedHash, indexRecords *MetaIndexParser)
1186   : pkgAcqIndex(Owner, Target, ExpectedHash, MetaIndexParser)
1187 {
1188    // load the filesize
1189    indexRecords::checkSum *Record = MetaIndexParser->Lookup(string(Target->MetaKey));
1190    if(Record)
1191       FileSize = Record->Size;
1192 }
1193                                                                         /*}}}*/
1194 // AcqIndexTrans::Custom600Headers - Insert custom request headers      /*{{{*/
1195 // ---------------------------------------------------------------------
1196 string pkgAcqIndexTrans::Custom600Headers()
1197 {
1198    string Final = _config->FindDir("Dir::State::lists");
1199    Final += URItoFileName(RealURI);
1200
1201    struct stat Buf;
1202    if (stat(Final.c_str(),&Buf) != 0)
1203       return "\nFail-Ignore: true\nIndex-File: true";
1204    return "\nFail-Ignore: true\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1205 }
1206                                                                         /*}}}*/
1207 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
1208 // ---------------------------------------------------------------------
1209 /* */
1210 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1211 {
1212    size_t const nextExt = CompressionExtension.find(' ');
1213    if (nextExt != std::string::npos)
1214    {
1215       CompressionExtension = CompressionExtension.substr(nextExt+1);
1216       Init(RealURI, Desc.Description, Desc.ShortDesc);
1217       Status = StatIdle;
1218       return;
1219    }
1220
1221    if (Cnf->LocalOnly == true ||
1222        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1223    {
1224       // Ignore this
1225       Status = StatDone;
1226       Complete = false;
1227       Dequeue();
1228       return;
1229    }
1230
1231    Item::Failed(Message,Cnf);
1232 }
1233                                                                         /*}}}*/
1234 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
1235                              string URI,string URIDesc,string ShortDesc,
1236                              string MetaIndexURI, string MetaIndexURIDesc,
1237                              string MetaIndexShortDesc,
1238                              const vector<IndexTarget*>* IndexTargets,
1239                              indexRecords* MetaIndexParser) :
1240    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
1241    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1242    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
1243 {
1244    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1245    DestFile += URItoFileName(URI);
1246
1247    // remove any partial downloaded sig-file in partial/.
1248    // it may confuse proxies and is too small to warrant a
1249    // partial download anyway
1250    unlink(DestFile.c_str());
1251
1252    // Create the item
1253    Desc.Description = URIDesc;
1254    Desc.Owner = this;
1255    Desc.ShortDesc = ShortDesc;
1256    Desc.URI = URI;
1257
1258    string Final = _config->FindDir("Dir::State::lists");
1259    Final += URItoFileName(RealURI);
1260    if (RealFileExists(Final) == true)
1261    {
1262       // File was already in place.  It needs to be re-downloaded/verified
1263       // because Release might have changed, we do give it a different
1264       // name than DestFile because otherwise the http method will
1265       // send If-Range requests and there are too many broken servers
1266       // out there that do not understand them
1267       LastGoodSig = DestFile+".reverify";
1268       Rename(Final,LastGoodSig);
1269    }
1270
1271    // we expect the indextargets + one additional Release file
1272    ExpectedAdditionalItems = IndexTargets->size() + 1;
1273
1274    QueueURI(Desc);
1275 }
1276                                                                         /*}}}*/
1277 pkgAcqMetaSig::~pkgAcqMetaSig()                                         /*{{{*/
1278 {
1279    // if the file was never queued undo file-changes done in the constructor
1280    if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false &&
1281          LastGoodSig.empty() == false)
1282    {
1283       string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1284       if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true)
1285          Rename(LastGoodSig, Final);
1286    }
1287
1288 }
1289                                                                         /*}}}*/
1290 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
1291 // ---------------------------------------------------------------------
1292 /* The only header we use is the last-modified header. */
1293 string pkgAcqMetaSig::Custom600Headers()
1294 {
1295    struct stat Buf;
1296    if (stat(LastGoodSig.c_str(),&Buf) != 0)
1297       return "\nIndex-File: true";
1298
1299    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1300 }
1301
1302 void pkgAcqMetaSig::Done(string Message,unsigned long long Size,string MD5,
1303                          pkgAcquire::MethodConfig *Cfg)
1304 {
1305    Item::Done(Message,Size,MD5,Cfg);
1306
1307    string FileName = LookupTag(Message,"Filename");
1308    if (FileName.empty() == true)
1309    {
1310       Status = StatError;
1311       ErrorText = "Method gave a blank filename";
1312       return;
1313    }
1314
1315    if (FileName != DestFile)
1316    {
1317       // We have to copy it into place
1318       Local = true;
1319       Desc.URI = "copy:" + FileName;
1320       QueueURI(Desc);
1321       return;
1322    }
1323
1324    Complete = true;
1325
1326    // at this point pkgAcqMetaIndex takes over
1327    ExpectedAdditionalItems = 0;
1328
1329    // put the last known good file back on i-m-s hit (it will
1330    // be re-verified again)
1331    // Else do nothing, we have the new file in DestFile then
1332    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1333       Rename(LastGoodSig, DestFile);
1334
1335    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
1336    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc,
1337                        MetaIndexShortDesc,  DestFile, IndexTargets,
1338                        MetaIndexParser);
1339
1340 }
1341                                                                         /*}}}*/
1342 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
1343 {
1344    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1345
1346    // at this point pkgAcqMetaIndex takes over
1347    ExpectedAdditionalItems = 0;
1348
1349    // if we get a network error we fail gracefully
1350    if(Status == StatTransientNetworkError)
1351    {
1352       Item::Failed(Message,Cnf);
1353       // move the sigfile back on transient network failures
1354       if(FileExists(LastGoodSig))
1355          Rename(LastGoodSig,Final);
1356
1357       // set the status back to , Item::Failed likes to reset it
1358       Status = pkgAcquire::Item::StatTransientNetworkError;
1359       return;
1360    }
1361
1362    // Delete any existing sigfile when the acquire failed
1363    unlink(Final.c_str());
1364
1365    // queue a pkgAcqMetaIndex with no sigfile
1366    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1367                        "", IndexTargets, MetaIndexParser);
1368
1369    if (Cnf->LocalOnly == true ||
1370        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1371    {
1372       // Ignore this
1373       Status = StatDone;
1374       Complete = false;
1375       Dequeue();
1376       return;
1377    }
1378
1379    Item::Failed(Message,Cnf);
1380 }
1381                                                                         /*}}}*/
1382 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
1383                                  string URI,string URIDesc,string ShortDesc,
1384                                  string SigFile,
1385                                  const vector<struct IndexTarget*>* IndexTargets,
1386                                  indexRecords* MetaIndexParser) :
1387    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
1388    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
1389 {
1390    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1391    DestFile += URItoFileName(URI);
1392
1393    // Create the item
1394    Desc.Description = URIDesc;
1395    Desc.Owner = this;
1396    Desc.ShortDesc = ShortDesc;
1397    Desc.URI = URI;
1398
1399    // we expect more item
1400    ExpectedAdditionalItems = IndexTargets->size();
1401
1402    QueueURI(Desc);
1403 }
1404                                                                         /*}}}*/
1405 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
1406 // ---------------------------------------------------------------------
1407 /* The only header we use is the last-modified header. */
1408 string pkgAcqMetaIndex::Custom600Headers()
1409 {
1410    string Final = _config->FindDir("Dir::State::lists");
1411    Final += URItoFileName(RealURI);
1412
1413    struct stat Buf;
1414    if (stat(Final.c_str(),&Buf) != 0)
1415       return "\nIndex-File: true";
1416
1417    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1418 }
1419                                                                         /*}}}*/
1420 void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,string Hash,  /*{{{*/
1421                            pkgAcquire::MethodConfig *Cfg)
1422 {
1423    Item::Done(Message,Size,Hash,Cfg);
1424
1425    // MetaIndexes are done in two passes: one to download the
1426    // metaindex with an appropriate method, and a second to verify it
1427    // with the gpgv method
1428
1429    if (AuthPass == true)
1430    {
1431       AuthDone(Message);
1432
1433       // all cool, move Release file into place
1434       Complete = true;
1435    }
1436    else
1437    {
1438       RetrievalDone(Message);
1439       if (!Complete)
1440          // Still more retrieving to do
1441          return;
1442
1443       if (SigFile == "")
1444       {
1445          // There was no signature file, so we are finished.  Download
1446          // the indexes and do only hashsum verification if possible
1447          MetaIndexParser->Load(DestFile);
1448          QueueIndexes(false);
1449       }
1450       else
1451       {
1452          // There was a signature file, so pass it to gpgv for
1453          // verification
1454
1455          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1456             std::cerr << "Metaindex acquired, queueing gpg verification ("
1457                       << SigFile << "," << DestFile << ")\n";
1458          AuthPass = true;
1459          Desc.URI = "gpgv:" + SigFile;
1460          QueueURI(Desc);
1461          Mode = "gpgv";
1462          return;
1463       }
1464    }
1465
1466    if (Complete == true)
1467    {
1468       string FinalFile = _config->FindDir("Dir::State::lists");
1469       FinalFile += URItoFileName(RealURI);
1470       if (SigFile == DestFile)
1471          SigFile = FinalFile;
1472       Rename(DestFile,FinalFile);
1473       chmod(FinalFile.c_str(),0644);
1474       DestFile = FinalFile;
1475    }
1476 }
1477                                                                         /*}}}*/
1478 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
1479 {
1480    // We have just finished downloading a Release file (it is not
1481    // verified yet)
1482
1483    string FileName = LookupTag(Message,"Filename");
1484    if (FileName.empty() == true)
1485    {
1486       Status = StatError;
1487       ErrorText = "Method gave a blank filename";
1488       return;
1489    }
1490
1491    if (FileName != DestFile)
1492    {
1493       Local = true;
1494       Desc.URI = "copy:" + FileName;
1495       QueueURI(Desc);
1496       return;
1497    }
1498
1499    // make sure to verify against the right file on I-M-S hit
1500    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1501    if(IMSHit)
1502    {
1503       string FinalFile = _config->FindDir("Dir::State::lists");
1504       FinalFile += URItoFileName(RealURI);
1505       if (SigFile == DestFile)
1506       {
1507          SigFile = FinalFile;
1508          // constructor of pkgAcqMetaClearSig moved it out of the way,
1509          // now move it back in on IMS hit for the 'old' file
1510          string const OldClearSig = DestFile + ".reverify";
1511          if (RealFileExists(OldClearSig) == true)
1512             Rename(OldClearSig, FinalFile);
1513       }
1514       DestFile = FinalFile;
1515    }
1516    Complete = true;
1517 }
1518                                                                         /*}}}*/
1519 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1520 {
1521    // At this point, the gpgv method has succeeded, so there is a
1522    // valid signature from a key in the trusted keyring.  We
1523    // perform additional verification of its contents, and use them
1524    // to verify the indexes we are about to download
1525
1526    if (!MetaIndexParser->Load(DestFile))
1527    {
1528       Status = StatAuthError;
1529       ErrorText = MetaIndexParser->ErrorText;
1530       return;
1531    }
1532
1533    if (!VerifyVendor(Message))
1534    {
1535       return;
1536    }
1537
1538    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1539       std::cerr << "Signature verification succeeded: "
1540                 << DestFile << std::endl;
1541
1542    // Download further indexes with verification
1543    QueueIndexes(true);
1544
1545    // is it a clearsigned MetaIndex file?
1546    if (DestFile == SigFile)
1547       return;
1548
1549    // Done, move signature file into position
1550    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1551       URItoFileName(RealURI) + ".gpg";
1552    Rename(SigFile,VerifiedSigFile);
1553    chmod(VerifiedSigFile.c_str(),0644);
1554 }
1555                                                                         /*}}}*/
1556 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1557 {
1558 #if 0
1559    /* Reject invalid, existing Release files (LP: #346386) (Closes: #627642)
1560     * FIXME: Disabled; it breaks unsigned repositories without hashes */
1561    if (!verify && FileExists(DestFile) && !MetaIndexParser->Load(DestFile))
1562    {
1563       Status = StatError;
1564       ErrorText = MetaIndexParser->ErrorText;
1565       return;
1566    }
1567 #endif
1568    bool transInRelease = false;
1569    {
1570       std::vector<std::string> const keys = MetaIndexParser->MetaKeys();
1571       for (std::vector<std::string>::const_iterator k = keys.begin(); k != keys.end(); ++k)
1572          // FIXME: Feels wrong to check for hardcoded string here, but what should we do else…
1573          if (k->find("Translation-") != std::string::npos)
1574          {
1575             transInRelease = true;
1576             break;
1577          }
1578    }
1579
1580    // at this point the real Items are loaded in the fetcher
1581    ExpectedAdditionalItems = 0;
1582
1583    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1584         Target != IndexTargets->end();
1585         ++Target)
1586    {
1587       HashString ExpectedIndexHash;
1588       indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1589       bool compressedAvailable = false;
1590       if (Record == NULL)
1591       {
1592          if ((*Target)->IsOptional() == true)
1593          {
1594             std::vector<std::string> types = APT::Configuration::getCompressionTypes();
1595             for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
1596                if (MetaIndexParser->Exists((*Target)->MetaKey + "." + *t) == true)
1597                {
1598                   compressedAvailable = true;
1599                   break;
1600                }
1601          }
1602          else if (verify == true)
1603          {
1604             Status = StatAuthError;
1605             strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), (*Target)->MetaKey.c_str());
1606             return;
1607          }
1608       }
1609       else
1610       {
1611          ExpectedIndexHash = Record->Hash;
1612          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1613          {
1614             std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1615             std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1616             std::cerr << "For: " << Record->MetaKeyFilename << std::endl;
1617          }
1618          if (verify == true && ExpectedIndexHash.empty() == true && (*Target)->IsOptional() == false)
1619          {
1620             Status = StatAuthError;
1621             strprintf(ErrorText, _("Unable to find hash sum for '%s' in Release file"), (*Target)->MetaKey.c_str());
1622             return;
1623          }
1624       }
1625
1626       if ((*Target)->IsOptional() == true)
1627       {
1628          if ((*Target)->IsSubIndex() == true)
1629             new pkgAcqSubIndex(Owner, (*Target)->URI, (*Target)->Description,
1630                                 (*Target)->ShortDesc, ExpectedIndexHash);
1631          else if (transInRelease == false || Record != NULL || compressedAvailable == true)
1632          {
1633             if (_config->FindB("Acquire::PDiffs",true) == true && transInRelease == true &&
1634                 MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true)
1635                new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1636                                    (*Target)->ShortDesc, ExpectedIndexHash);
1637             else
1638                new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1639          }
1640          continue;
1641       }
1642
1643       /* Queue Packages file (either diff or full packages files, depending
1644          on the users option) - we also check if the PDiff Index file is listed
1645          in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
1646          instead, but passing the required info to it is to much hassle */
1647       if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
1648           MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true))
1649          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1650                              (*Target)->ShortDesc, ExpectedIndexHash);
1651       else
1652          new pkgAcqIndex(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1653    }
1654 }
1655                                                                         /*}}}*/
1656 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1657 {
1658    string::size_type pos;
1659
1660    // check for missing sigs (that where not fatal because otherwise we had
1661    // bombed earlier)
1662    string missingkeys;
1663    string msg = _("There is no public key available for the "
1664                   "following key IDs:\n");
1665    pos = Message.find("NO_PUBKEY ");
1666    if (pos != std::string::npos)
1667    {
1668       string::size_type start = pos+strlen("NO_PUBKEY ");
1669       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1670       missingkeys += (Fingerprint);
1671    }
1672    if(!missingkeys.empty())
1673       _error->Warning("%s", (msg + missingkeys).c_str());
1674
1675    string Transformed = MetaIndexParser->GetExpectedDist();
1676
1677    if (Transformed == "../project/experimental")
1678    {
1679       Transformed = "experimental";
1680    }
1681
1682    pos = Transformed.rfind('/');
1683    if (pos != string::npos)
1684    {
1685       Transformed = Transformed.substr(0, pos);
1686    }
1687
1688    if (Transformed == ".")
1689    {
1690       Transformed = "";
1691    }
1692
1693    if (_config->FindB("Acquire::Check-Valid-Until", true) == true &&
1694        MetaIndexParser->GetValidUntil() > 0) {
1695       time_t const invalid_since = time(NULL) - MetaIndexParser->GetValidUntil();
1696       if (invalid_since > 0)
1697          // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1698          // the time since then the file is invalid - formated in the same way as in
1699          // the download progress display (e.g. 7d 3h 42min 1s)
1700          return _error->Error(
1701             _("Release file for %s is expired (invalid since %s). "
1702               "Updates for this repository will not be applied."),
1703             RealURI.c_str(), TimeToStr(invalid_since).c_str());
1704    }
1705
1706    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1707    {
1708       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1709       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1710       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1711    }
1712
1713    if (MetaIndexParser->CheckDist(Transformed) == false)
1714    {
1715       // This might become fatal one day
1716 //       Status = StatAuthError;
1717 //       ErrorText = "Conflicting distribution; expected "
1718 //          + MetaIndexParser->GetExpectedDist() + " but got "
1719 //          + MetaIndexParser->GetDist();
1720 //       return false;
1721       if (!Transformed.empty())
1722       {
1723          _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1724                          Desc.Description.c_str(),
1725                          Transformed.c_str(),
1726                          MetaIndexParser->GetDist().c_str());
1727       }
1728    }
1729
1730    return true;
1731 }
1732                                                                         /*}}}*/
1733 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1734 // ---------------------------------------------------------------------
1735 /* */
1736 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)
1737 {
1738    if (AuthPass == true)
1739    {
1740       // gpgv method failed, if we have a good signature
1741       string LastGoodSigFile = _config->FindDir("Dir::State::lists").append("partial/").append(URItoFileName(RealURI));
1742       if (DestFile != SigFile)
1743          LastGoodSigFile.append(".gpg");
1744       LastGoodSigFile.append(".reverify");
1745
1746       if(FileExists(LastGoodSigFile))
1747       {
1748          string VerifiedSigFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1749          if (DestFile != SigFile)
1750             VerifiedSigFile.append(".gpg");
1751          Rename(LastGoodSigFile, VerifiedSigFile);
1752          Status = StatTransientNetworkError;
1753          _error->Warning(_("An error occurred during the signature "
1754                            "verification. The repository is not updated "
1755                            "and the previous index files will be used. "
1756                            "GPG error: %s: %s\n"),
1757                          Desc.Description.c_str(),
1758                          LookupTag(Message,"Message").c_str());
1759          RunScripts("APT::Update::Auth-Failure");
1760          return;
1761       } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) {
1762          /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */
1763          _error->Error(_("GPG error: %s: %s"),
1764                          Desc.Description.c_str(),
1765                          LookupTag(Message,"Message").c_str());
1766          return;
1767       } else {
1768          _error->Warning(_("GPG error: %s: %s"),
1769                          Desc.Description.c_str(),
1770                          LookupTag(Message,"Message").c_str());
1771       }
1772       // gpgv method failed
1773       ReportMirrorFailure("GPGFailure");
1774    }
1775
1776    /* Always move the meta index, even if gpgv failed. This ensures
1777     * that PackageFile objects are correctly filled in */
1778    if (FileExists(DestFile)) {
1779       string FinalFile = _config->FindDir("Dir::State::lists");
1780       FinalFile += URItoFileName(RealURI);
1781       /* InRelease files become Release files, otherwise
1782        * they would be considered as trusted later on */
1783       if (SigFile == DestFile) {
1784          RealURI = RealURI.replace(RealURI.rfind("InRelease"), 9,
1785                                        "Release");
1786          FinalFile = FinalFile.replace(FinalFile.rfind("InRelease"), 9,
1787                                        "Release");
1788          SigFile = FinalFile;
1789       }
1790       Rename(DestFile,FinalFile);
1791       chmod(FinalFile.c_str(),0644);
1792
1793       DestFile = FinalFile;
1794    }
1795
1796    // No Release file was present, or verification failed, so fall
1797    // back to queueing Packages files without verification
1798    QueueIndexes(false);
1799 }
1800                                                                         /*}}}*/
1801 pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner,               /*{{{*/
1802                 string const &URI, string const &URIDesc, string const &ShortDesc,
1803                 string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc,
1804                 string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc,
1805                 const vector<struct IndexTarget*>* IndexTargets,
1806                 indexRecords* MetaIndexParser) :
1807         pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, "", IndexTargets, MetaIndexParser),
1808         MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1809         MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc)
1810 {
1811    SigFile = DestFile;
1812
1813    // index targets + (worst case:) Release/Release.gpg
1814    ExpectedAdditionalItems = IndexTargets->size() + 2;
1815
1816
1817    // keep the old InRelease around in case of transistent network errors
1818    string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1819    if (RealFileExists(Final) == true)
1820    {
1821       string const LastGoodSig = DestFile + ".reverify";
1822       Rename(Final,LastGoodSig);
1823    }
1824 }
1825                                                                         /*}}}*/
1826 pkgAcqMetaClearSig::~pkgAcqMetaClearSig()                               /*{{{*/
1827 {
1828    // if the file was never queued undo file-changes done in the constructor
1829    if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false)
1830    {
1831       string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1832       string const LastGoodSig = DestFile + ".reverify";
1833       if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true)
1834          Rename(LastGoodSig, Final);
1835    }
1836 }
1837                                                                         /*}}}*/
1838 // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/
1839 // ---------------------------------------------------------------------
1840 // FIXME: this can go away once the InRelease file is used widely
1841 string pkgAcqMetaClearSig::Custom600Headers()
1842 {
1843    string Final = _config->FindDir("Dir::State::lists");
1844    Final += URItoFileName(RealURI);
1845
1846    struct stat Buf;
1847    if (stat(Final.c_str(),&Buf) != 0)
1848    {
1849       Final = DestFile + ".reverify";
1850       if (stat(Final.c_str(),&Buf) != 0)
1851          return "\nIndex-File: true\nFail-Ignore: true\n";
1852    }
1853
1854    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1855 }
1856                                                                         /*}}}*/
1857 void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/
1858 {
1859    // we failed, we will not get additional items from this method
1860    ExpectedAdditionalItems = 0;
1861
1862    if (AuthPass == false)
1863    {
1864       // Remove the 'old' InRelease file if we try Release.gpg now as otherwise
1865       // the file will stay around and gives a false-auth impression (CVE-2012-0214)
1866       string FinalFile = _config->FindDir("Dir::State::lists");
1867       FinalFile.append(URItoFileName(RealURI));
1868       if (FileExists(FinalFile))
1869          unlink(FinalFile.c_str());
1870
1871       new pkgAcqMetaSig(Owner,
1872                         MetaSigURI, MetaSigURIDesc, MetaSigShortDesc,
1873                         MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1874                         IndexTargets, MetaIndexParser);
1875       if (Cnf->LocalOnly == true ||
1876           StringToBool(LookupTag(Message, "Transient-Failure"), false) == false)
1877          Dequeue();
1878    }
1879    else
1880       pkgAcqMetaIndex::Failed(Message, Cnf);
1881 }
1882                                                                         /*}}}*/
1883 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1884 // ---------------------------------------------------------------------
1885 /* This just sets up the initial fetch environment and queues the first
1886    possibilitiy */
1887 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1888                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1889                              string &StoreFilename) :
1890                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1891                StoreFilename(StoreFilename), Vf(Version.FileList()),
1892                Trusted(false)
1893 {
1894    Retries = _config->FindI("Acquire::Retries",0);
1895
1896    if (Version.Arch() == 0)
1897    {
1898       _error->Error(_("I wasn't able to locate a file for the %s package. "
1899                       "This might mean you need to manually fix this package. "
1900                       "(due to missing arch)"),
1901                     Version.ParentPkg().FullName().c_str());
1902       return;
1903    }
1904
1905    /* We need to find a filename to determine the extension. We make the
1906       assumption here that all the available sources for this version share
1907       the same extension.. */
1908    // Skip not source sources, they do not have file fields.
1909    for (; Vf.end() == false; ++Vf)
1910    {
1911       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1912          continue;
1913       break;
1914    }
1915
1916    // Does not really matter here.. we are going to fail out below
1917    if (Vf.end() != true)
1918    {
1919       // If this fails to get a file name we will bomb out below.
1920       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1921       if (_error->PendingError() == true)
1922          return;
1923
1924       // Generate the final file name as: package_version_arch.foo
1925       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1926                       QuoteString(Version.VerStr(),"_:") + '_' +
1927                       QuoteString(Version.Arch(),"_:.") +
1928                       "." + flExtension(Parse.FileName());
1929    }
1930
1931    // check if we have one trusted source for the package. if so, switch
1932    // to "TrustedOnly" mode - but only if not in AllowUnauthenticated mode
1933    bool const allowUnauth = _config->FindB("APT::Get::AllowUnauthenticated", false);
1934    bool const debugAuth = _config->FindB("Debug::pkgAcquire::Auth", false);
1935    bool seenUntrusted = false;
1936    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; ++i)
1937    {
1938       pkgIndexFile *Index;
1939       if (Sources->FindIndex(i.File(),Index) == false)
1940          continue;
1941
1942       if (debugAuth == true)
1943          std::cerr << "Checking index: " << Index->Describe()
1944                    << "(Trusted=" << Index->IsTrusted() << ")" << std::endl;
1945
1946       if (Index->IsTrusted() == true)
1947       {
1948          Trusted = true;
1949          if (allowUnauth == false)
1950             break;
1951       }
1952       else
1953          seenUntrusted = true;
1954    }
1955
1956    // "allow-unauthenticated" restores apts old fetching behaviour
1957    // that means that e.g. unauthenticated file:// uris are higher
1958    // priority than authenticated http:// uris
1959    if (allowUnauth == true && seenUntrusted == true)
1960       Trusted = false;
1961
1962    // Select a source
1963    if (QueueNext() == false && _error->PendingError() == false)
1964       _error->Error(_("Can't find a source to download version '%s' of '%s'"),
1965                     Version.VerStr(), Version.ParentPkg().FullName(false).c_str());
1966 }
1967                                                                         /*}}}*/
1968 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1969 // ---------------------------------------------------------------------
1970 /* This queues the next available file version for download. It checks if
1971    the archive is already available in the cache and stashs the MD5 for
1972    checking later. */
1973 bool pkgAcqArchive::QueueNext()
1974 {
1975    string const ForceHash = _config->Find("Acquire::ForceHash");
1976    for (; Vf.end() == false; ++Vf)
1977    {
1978       // Ignore not source sources
1979       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1980          continue;
1981
1982       // Try to cross match against the source list
1983       pkgIndexFile *Index;
1984       if (Sources->FindIndex(Vf.File(),Index) == false)
1985             continue;
1986
1987       // only try to get a trusted package from another source if that source
1988       // is also trusted
1989       if(Trusted && !Index->IsTrusted())
1990          continue;
1991
1992       // Grab the text package record
1993       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1994       if (_error->PendingError() == true)
1995          return false;
1996
1997       string PkgFile = Parse.FileName();
1998       if (ForceHash.empty() == false)
1999       {
2000          if(stringcasecmp(ForceHash, "sha512") == 0)
2001             ExpectedHash = HashString("SHA512", Parse.SHA512Hash());
2002          else if(stringcasecmp(ForceHash, "sha256") == 0)
2003             ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
2004          else if (stringcasecmp(ForceHash, "sha1") == 0)
2005             ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
2006          else
2007             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
2008       }
2009       else
2010       {
2011          string Hash;
2012          if ((Hash = Parse.SHA512Hash()).empty() == false)
2013             ExpectedHash = HashString("SHA512", Hash);
2014          else if ((Hash = Parse.SHA256Hash()).empty() == false)
2015             ExpectedHash = HashString("SHA256", Hash);
2016          else if ((Hash = Parse.SHA1Hash()).empty() == false)
2017             ExpectedHash = HashString("SHA1", Hash);
2018          else
2019             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
2020       }
2021       if (PkgFile.empty() == true)
2022          return _error->Error(_("The package index files are corrupted. No Filename: "
2023                               "field for package %s."),
2024                               Version.ParentPkg().Name());
2025
2026       Desc.URI = Index->ArchiveURI(PkgFile);
2027       Desc.Description = Index->ArchiveInfo(Version);
2028       Desc.Owner = this;
2029       Desc.ShortDesc = Version.ParentPkg().FullName(true);
2030
2031       // See if we already have the file. (Legacy filenames)
2032       FileSize = Version->Size;
2033       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
2034       struct stat Buf;
2035       if (stat(FinalFile.c_str(),&Buf) == 0)
2036       {
2037          // Make sure the size matches
2038          if ((unsigned long long)Buf.st_size == Version->Size)
2039          {
2040             Complete = true;
2041             Local = true;
2042             Status = StatDone;
2043             StoreFilename = DestFile = FinalFile;
2044             return true;
2045          }
2046
2047          /* Hmm, we have a file and its size does not match, this means it is
2048             an old style mismatched arch */
2049          unlink(FinalFile.c_str());
2050       }
2051
2052       // Check it again using the new style output filenames
2053       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
2054       if (stat(FinalFile.c_str(),&Buf) == 0)
2055       {
2056          // Make sure the size matches
2057          if ((unsigned long long)Buf.st_size == Version->Size)
2058          {
2059             Complete = true;
2060             Local = true;
2061             Status = StatDone;
2062             StoreFilename = DestFile = FinalFile;
2063             return true;
2064          }
2065
2066          /* Hmm, we have a file and its size does not match, this shouldn't
2067             happen.. */
2068          unlink(FinalFile.c_str());
2069       }
2070
2071       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
2072
2073       // Check the destination file
2074       if (stat(DestFile.c_str(),&Buf) == 0)
2075       {
2076          // Hmm, the partial file is too big, erase it
2077          if ((unsigned long long)Buf.st_size > Version->Size)
2078             unlink(DestFile.c_str());
2079          else
2080             PartialSize = Buf.st_size;
2081       }
2082
2083       // Disables download of archives - useful if no real installation follows,
2084       // e.g. if we are just interested in proposed installation order
2085       if (_config->FindB("Debug::pkgAcqArchive::NoQueue", false) == true)
2086       {
2087          Complete = true;
2088          Local = true;
2089          Status = StatDone;
2090          StoreFilename = DestFile = FinalFile;
2091          return true;
2092       }
2093
2094       // Create the item
2095       Local = false;
2096       QueueURI(Desc);
2097
2098       ++Vf;
2099       return true;
2100    }
2101    return false;
2102 }
2103                                                                         /*}}}*/
2104 // AcqArchive::Done - Finished fetching                                 /*{{{*/
2105 // ---------------------------------------------------------------------
2106 /* */
2107 void pkgAcqArchive::Done(string Message,unsigned long long Size,string CalcHash,
2108                          pkgAcquire::MethodConfig *Cfg)
2109 {
2110    Item::Done(Message,Size,CalcHash,Cfg);
2111
2112    // Check the size
2113    if (Size != Version->Size)
2114    {
2115       RenameOnError(SizeMismatch);
2116       return;
2117    }
2118
2119    // Check the hash
2120    if(ExpectedHash.toStr() != CalcHash)
2121    {
2122       RenameOnError(HashSumMismatch);
2123       return;
2124    }
2125
2126    // Grab the output filename
2127    string FileName = LookupTag(Message,"Filename");
2128    if (FileName.empty() == true)
2129    {
2130       Status = StatError;
2131       ErrorText = "Method gave a blank filename";
2132       return;
2133    }
2134
2135    Complete = true;
2136
2137    // Reference filename
2138    if (FileName != DestFile)
2139    {
2140       StoreFilename = DestFile = FileName;
2141       Local = true;
2142       return;
2143    }
2144
2145    // Done, move it into position
2146    string FinalFile = _config->FindDir("Dir::Cache::Archives");
2147    FinalFile += flNotDir(StoreFilename);
2148    Rename(DestFile,FinalFile);
2149
2150    StoreFilename = DestFile = FinalFile;
2151    Complete = true;
2152 }
2153                                                                         /*}}}*/
2154 // AcqArchive::Failed - Failure handler                                 /*{{{*/
2155 // ---------------------------------------------------------------------
2156 /* Here we try other sources */
2157 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2158 {
2159    ErrorText = LookupTag(Message,"Message");
2160
2161    /* We don't really want to retry on failed media swaps, this prevents
2162       that. An interesting observation is that permanent failures are not
2163       recorded. */
2164    if (Cnf->Removable == true &&
2165        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2166    {
2167       // Vf = Version.FileList();
2168       while (Vf.end() == false) ++Vf;
2169       StoreFilename = string();
2170       Item::Failed(Message,Cnf);
2171       return;
2172    }
2173
2174    if (QueueNext() == false)
2175    {
2176       // This is the retry counter
2177       if (Retries != 0 &&
2178           Cnf->LocalOnly == false &&
2179           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2180       {
2181          Retries--;
2182          Vf = Version.FileList();
2183          if (QueueNext() == true)
2184             return;
2185       }
2186
2187       StoreFilename = string();
2188       Item::Failed(Message,Cnf);
2189    }
2190 }
2191                                                                         /*}}}*/
2192 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
2193 // ---------------------------------------------------------------------
2194 APT_PURE bool pkgAcqArchive::IsTrusted()
2195 {
2196    return Trusted;
2197 }
2198                                                                         /*}}}*/
2199 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
2200 // ---------------------------------------------------------------------
2201 /* */
2202 void pkgAcqArchive::Finished()
2203 {
2204    if (Status == pkgAcquire::Item::StatDone &&
2205        Complete == true)
2206       return;
2207    StoreFilename = string();
2208 }
2209                                                                         /*}}}*/
2210 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
2211 // ---------------------------------------------------------------------
2212 /* The file is added to the queue */
2213 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
2214                        unsigned long long Size,string Dsc,string ShortDesc,
2215                        const string &DestDir, const string &DestFilename,
2216                        bool IsIndexFile) :
2217                        Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile)
2218 {
2219    Retries = _config->FindI("Acquire::Retries",0);
2220
2221    if(!DestFilename.empty())
2222       DestFile = DestFilename;
2223    else if(!DestDir.empty())
2224       DestFile = DestDir + "/" + flNotDir(URI);
2225    else
2226       DestFile = flNotDir(URI);
2227
2228    // Create the item
2229    Desc.URI = URI;
2230    Desc.Description = Dsc;
2231    Desc.Owner = this;
2232
2233    // Set the short description to the archive component
2234    Desc.ShortDesc = ShortDesc;
2235
2236    // Get the transfer sizes
2237    FileSize = Size;
2238    struct stat Buf;
2239    if (stat(DestFile.c_str(),&Buf) == 0)
2240    {
2241       // Hmm, the partial file is too big, erase it
2242       if ((Size > 0) && (unsigned long long)Buf.st_size > Size)
2243          unlink(DestFile.c_str());
2244       else
2245          PartialSize = Buf.st_size;
2246    }
2247
2248    QueueURI(Desc);
2249 }
2250                                                                         /*}}}*/
2251 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
2252 // ---------------------------------------------------------------------
2253 /* */
2254 void pkgAcqFile::Done(string Message,unsigned long long Size,string CalcHash,
2255                       pkgAcquire::MethodConfig *Cnf)
2256 {
2257    Item::Done(Message,Size,CalcHash,Cnf);
2258
2259    // Check the hash
2260    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
2261    {
2262       RenameOnError(HashSumMismatch);
2263       return;
2264    }
2265
2266    string FileName = LookupTag(Message,"Filename");
2267    if (FileName.empty() == true)
2268    {
2269       Status = StatError;
2270       ErrorText = "Method gave a blank filename";
2271       return;
2272    }
2273
2274    Complete = true;
2275
2276    // The files timestamp matches
2277    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
2278       return;
2279
2280    // We have to copy it into place
2281    if (FileName != DestFile)
2282    {
2283       Local = true;
2284       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
2285           Cnf->Removable == true)
2286       {
2287          Desc.URI = "copy:" + FileName;
2288          QueueURI(Desc);
2289          return;
2290       }
2291
2292       // Erase the file if it is a symlink so we can overwrite it
2293       struct stat St;
2294       if (lstat(DestFile.c_str(),&St) == 0)
2295       {
2296          if (S_ISLNK(St.st_mode) != 0)
2297             unlink(DestFile.c_str());
2298       }
2299
2300       // Symlink the file
2301       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
2302       {
2303          ErrorText = "Link to " + DestFile + " failure ";
2304          Status = StatError;
2305          Complete = false;
2306       }
2307    }
2308 }
2309                                                                         /*}}}*/
2310 // AcqFile::Failed - Failure handler                                    /*{{{*/
2311 // ---------------------------------------------------------------------
2312 /* Here we try other sources */
2313 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2314 {
2315    ErrorText = LookupTag(Message,"Message");
2316
2317    // This is the retry counter
2318    if (Retries != 0 &&
2319        Cnf->LocalOnly == false &&
2320        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2321    {
2322       Retries--;
2323       QueueURI(Desc);
2324       return;
2325    }
2326
2327    Item::Failed(Message,Cnf);
2328 }
2329                                                                         /*}}}*/
2330 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
2331 // ---------------------------------------------------------------------
2332 /* The only header we use is the last-modified header. */
2333 string pkgAcqFile::Custom600Headers()
2334 {
2335    if (IsIndexFile)
2336       return "\nIndex-File: true";
2337    return "";
2338 }
2339                                                                         /*}}}*/