apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <config.h>
  17
  18 #include <apt-pkg/acquire-item.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/aptconfiguration.h>
  21 #include <apt-pkg/sourcelist.h>
  22 #include <apt-pkg/error.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/sha1.h>
  26 #include <apt-pkg/tagfile.h>
  27 #include <apt-pkg/indexrecords.h>
  28 #include <apt-pkg/acquire.h>
  29 #include <apt-pkg/hashes.h>
  30 #include <apt-pkg/indexfile.h>
  31 #include <apt-pkg/pkgcache.h>
  32 #include <apt-pkg/cacheiterators.h>
  33 #include <apt-pkg/pkgrecords.h>
  34
  35 #include <stddef.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38 #include <iostream>
  39 #include <vector>
  40 #include <sys/stat.h>
  41 #include <unistd.h>
  42 #include <errno.h>
  43 #include <string>
  44 #include <sstream>
  45 #include <stdio.h>
  46 #include <ctime>
  47
  48 #include <apti18n.h>
  49                                                                         /*}}}*/
  50
  51 using namespace std;
  52
  53 // Acquire::Item::Item - Constructor                                    /*{{{*/
  54 // ---------------------------------------------------------------------
  55 /* */
  56 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  57                        PartialSize(0), Mode(0), ID(0), Complete(false),
  58                        Local(false), QueueCounter(0)
  59 {
  60    Owner->Add(this);
  61    Status = StatIdle;
  62 }
  63                                                                         /*}}}*/
  64 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  65 // ---------------------------------------------------------------------
  66 /* */
  67 pkgAcquire::Item::~Item()
  68 {
  69    Owner->Remove(this);
  70 }
  71                                                                         /*}}}*/
  72 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  73 // ---------------------------------------------------------------------
  74 /* We return to an idle state if there are still other queues that could
  75    fetch this object */
  76 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  77 {
  78    Status = StatIdle;
  79    ErrorText = LookupTag(Message,"Message");
  80    UsedMirror =  LookupTag(Message,"UsedMirror");
  81    if (QueueCounter <= 1)
  82    {
  83       /* This indicates that the file is not available right now but might
  84          be sometime later. If we do a retry cycle then this should be
  85          retried [CDROMs] */
  86       if (Cnf->LocalOnly == true &&
  87           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  88       {
  89          Status = StatIdle;
  90          Dequeue();
  91          return;
  92       }
  93
  94       Status = StatError;
  95       Dequeue();
  96    }
  97
  98    // report mirror failure back to LP if we actually use a mirror
  99    string FailReason = LookupTag(Message, "FailReason");
 100    if(FailReason.size() != 0)
 101       ReportMirrorFailure(FailReason);
 102    else
 103       ReportMirrorFailure(ErrorText);
 104 }
 105                                                                         /*}}}*/
 106 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
 107 // ---------------------------------------------------------------------
 108 /* Stash status and the file size. Note that setting Complete means
 109    sub-phases of the acquire process such as decompresion are operating */
 110 void pkgAcquire::Item::Start(string /*Message*/,unsigned long long Size)
 111 {
 112    Status = StatFetching;
 113    if (FileSize == 0 && Complete == false)
 114       FileSize = Size;
 115 }
 116                                                                         /*}}}*/
 117 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
 118 // ---------------------------------------------------------------------
 119 /* */
 120 void pkgAcquire::Item::Done(string Message,unsigned long long Size,string /*Hash*/,
 121                             pkgAcquire::MethodConfig * /*Cnf*/)
 122 {
 123    // We just downloaded something..
 124    string FileName = LookupTag(Message,"Filename");
 125    UsedMirror =  LookupTag(Message,"UsedMirror");
 126    if (Complete == false && !Local && FileName == DestFile)
 127    {
 128       if (Owner->Log != 0)
 129          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 130    }
 131
 132    if (FileSize == 0)
 133       FileSize= Size;
 134    Status = StatDone;
 135    ErrorText = string();
 136    Owner->Dequeue(this);
 137 }
 138                                                                         /*}}}*/
 139 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 140 // ---------------------------------------------------------------------
 141 /* This helper function is used by a lot of item methods as their final
 142    step */
 143 void pkgAcquire::Item::Rename(string From,string To)
 144 {
 145    if (rename(From.c_str(),To.c_str()) != 0)
 146    {
 147       char S[300];
 148       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 149               From.c_str(),To.c_str());
 150       Status = StatError;
 151       ErrorText = S;
 152    }
 153 }
 154                                                                         /*}}}*/
 155 bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const error)/*{{{*/
 156 {
 157    if(FileExists(DestFile))
 158       Rename(DestFile, DestFile + ".FAILED");
 159
 160    switch (error)
 161    {
 162       case HashSumMismatch:
 163          ErrorText = _("Hash Sum mismatch");
 164          Status = StatAuthError;
 165          ReportMirrorFailure("HashChecksumFailure");
 166          break;
 167       case SizeMismatch:
 168          ErrorText = _("Size mismatch");
 169          Status = StatAuthError;
 170          ReportMirrorFailure("SizeFailure");
 171          break;
 172       case InvalidFormat:
 173          ErrorText = _("Invalid file format");
 174          Status = StatError;
 175          // do not report as usually its not the mirrors fault, but Portal/Proxy
 176          break;
 177    }
 178    return false;
 179 }
 180                                                                         /*}}}*/
 181 // Acquire::Item::ReportMirrorFailure                                   /*{{{*/
 182 // ---------------------------------------------------------------------
 183 void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
 184 {
 185    // we only act if a mirror was used at all
 186    if(UsedMirror.empty())
 187       return;
 188 #if 0
 189    std::cerr << "\nReportMirrorFailure: "
 190              << UsedMirror
 191              << " Uri: " << DescURI()
 192              << " FailCode: "
 193              << FailCode << std::endl;
 194 #endif
 195    const char *Args[40];
 196    unsigned int i = 0;
 197    string report = _config->Find("Methods::Mirror::ProblemReporting",
 198                                  "/usr/lib/apt/apt-report-mirror-failure");
 199    if(!FileExists(report))
 200       return;
 201    Args[i++] = report.c_str();
 202    Args[i++] = UsedMirror.c_str();
 203    Args[i++] = DescURI().c_str();
 204    Args[i++] = FailCode.c_str();
 205    Args[i++] = NULL;
 206    pid_t pid = ExecFork();
 207    if(pid < 0)
 208    {
 209       _error->Error("ReportMirrorFailure Fork failed");
 210       return;
 211    }
 212    else if(pid == 0)
 213    {
 214       execvp(Args[0], (char**)Args);
 215       std::cerr << "Could not exec " << Args[0] << std::endl;
 216       _exit(100);
 217    }
 218    if(!ExecWait(pid, "report-mirror-failure"))
 219    {
 220       _error->Warning("Couldn't report problem to '%s'",
 221                       _config->Find("Methods::Mirror::ProblemReporting").c_str());
 222    }
 223 }
 224                                                                         /*}}}*/
 225 // AcqSubIndex::AcqSubIndex - Constructor                               /*{{{*/
 226 // ---------------------------------------------------------------------
 227 /* Get a sub-index file based on checksums from a 'master' file and
 228    possibly query additional files */
 229 pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, string const &URI,
 230                                  string const &URIDesc, string const &ShortDesc,
 231                                  HashString const &ExpectedHash)
 232    : Item(Owner), ExpectedHash(ExpectedHash)
 233 {
 234    /* XXX: Beware: Currently this class does nothing (of value) anymore ! */
 235    Debug = _config->FindB("Debug::pkgAcquire::SubIndex",false);
 236
 237    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 238    DestFile += URItoFileName(URI);
 239
 240    Desc.URI = URI;
 241    Desc.Description = URIDesc;
 242    Desc.Owner = this;
 243    Desc.ShortDesc = ShortDesc;
 244
 245    QueueURI(Desc);
 246
 247    if(Debug)
 248       std::clog << "pkgAcqSubIndex: " << Desc.URI << std::endl;
 249 }
 250                                                                         /*}}}*/
 251 // AcqSubIndex::Custom600Headers - Insert custom request headers        /*{{{*/
 252 // ---------------------------------------------------------------------
 253 /* The only header we use is the last-modified header. */
 254 string pkgAcqSubIndex::Custom600Headers()
 255 {
 256    string Final = _config->FindDir("Dir::State::lists");
 257    Final += URItoFileName(Desc.URI);
 258
 259    struct stat Buf;
 260    if (stat(Final.c_str(),&Buf) != 0)
 261       return "\nIndex-File: true\nFail-Ignore: true\n";
 262    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 263 }
 264                                                                         /*}}}*/
 265 void pkgAcqSubIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 266 {
 267    if(Debug)
 268       std::clog << "pkgAcqSubIndex failed: " << Desc.URI << " with " << Message << std::endl;
 269
 270    Complete = false;
 271    Status = StatDone;
 272    Dequeue();
 273
 274    // No good Index is provided
 275 }
 276                                                                         /*}}}*/
 277 void pkgAcqSubIndex::Done(string Message,unsigned long long Size,string Md5Hash,        /*{{{*/
 278                            pkgAcquire::MethodConfig *Cnf)
 279 {
 280    if(Debug)
 281       std::clog << "pkgAcqSubIndex::Done(): " << Desc.URI << std::endl;
 282
 283    string FileName = LookupTag(Message,"Filename");
 284    if (FileName.empty() == true)
 285    {
 286       Status = StatError;
 287       ErrorText = "Method gave a blank filename";
 288       return;
 289    }
 290
 291    if (FileName != DestFile)
 292    {
 293       Local = true;
 294       Desc.URI = "copy:" + FileName;
 295       QueueURI(Desc);
 296       return;
 297    }
 298
 299    Item::Done(Message,Size,Md5Hash,Cnf);
 300
 301    string FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(Desc.URI);
 302
 303    /* Downloaded invalid transindex => Error (LP: #346386) (Closes: #627642) */
 304    indexRecords SubIndexParser;
 305    if (FileExists(DestFile) == true && !SubIndexParser.Load(DestFile)) {
 306       Status = StatError;
 307       ErrorText = SubIndexParser.ErrorText;
 308       return;
 309    }
 310
 311    // success in downloading the index
 312    // rename the index
 313    if(Debug)
 314       std::clog << "Renaming: " << DestFile << " -> " << FinalFile << std::endl;
 315    Rename(DestFile,FinalFile);
 316    chmod(FinalFile.c_str(),0644);
 317    DestFile = FinalFile;
 318
 319    if(ParseIndex(DestFile) == false)
 320       return Failed("", NULL);
 321
 322    Complete = true;
 323    Status = StatDone;
 324    Dequeue();
 325    return;
 326 }
 327                                                                         /*}}}*/
 328 bool pkgAcqSubIndex::ParseIndex(string const &IndexFile)                /*{{{*/
 329 {
 330    indexRecords SubIndexParser;
 331    if (FileExists(IndexFile) == false || SubIndexParser.Load(IndexFile) == false)
 332       return false;
 333    // so something with the downloaded index
 334    return true;
 335 }
 336                                                                         /*}}}*/
 337 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 338 // ---------------------------------------------------------------------
 339 /* Get the DiffIndex file first and see if there are patches available
 340  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 341  * patches. If anything goes wrong in that process, it will fall back to
 342  * the original packages file
 343  */
 344 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 345                                  string URI,string URIDesc,string ShortDesc,
 346                                  HashString ExpectedHash)
 347    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 348      Description(URIDesc)
 349 {
 350
 351    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 352
 353    Desc.Description = URIDesc + "/DiffIndex";
 354    Desc.Owner = this;
 355    Desc.ShortDesc = ShortDesc;
 356    Desc.URI = URI + ".diff/Index";
 357
 358    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 359    DestFile += URItoFileName(URI) + string(".DiffIndex");
 360
 361    if(Debug)
 362       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 363
 364    // look for the current package file
 365    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 366    CurrentPackagesFile += URItoFileName(RealURI);
 367
 368    // FIXME: this file:/ check is a hack to prevent fetching
 369    //        from local sources. this is really silly, and
 370    //        should be fixed cleanly as soon as possible
 371    if(!FileExists(CurrentPackagesFile) ||
 372       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 373    {
 374       // we don't have a pkg file or we don't want to queue
 375       if(Debug)
 376          std::clog << "No index file, local or canceld by user" << std::endl;
 377       Failed("", NULL);
 378       return;
 379    }
 380
 381    if(Debug)
 382       std::clog << "pkgAcqDiffIndex::pkgAcqDiffIndex(): "
 383          << CurrentPackagesFile << std::endl;
 384
 385    QueueURI(Desc);
 386
 387 }
 388                                                                         /*}}}*/
 389 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 390 // ---------------------------------------------------------------------
 391 /* The only header we use is the last-modified header. */
 392 string pkgAcqDiffIndex::Custom600Headers()
 393 {
 394    string Final = _config->FindDir("Dir::State::lists");
 395    Final += URItoFileName(RealURI) + string(".IndexDiff");
 396
 397    if(Debug)
 398       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 399
 400    struct stat Buf;
 401    if (stat(Final.c_str(),&Buf) != 0)
 402       return "\nIndex-File: true";
 403
 404    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 405 }
 406                                                                         /*}}}*/
 407 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 408 {
 409    if(Debug)
 410       std::clog << "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile
 411          << std::endl;
 412
 413    pkgTagSection Tags;
 414    string ServerSha1;
 415    vector<DiffInfo> available_patches;
 416
 417    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 418    pkgTagFile TF(&Fd);
 419    if (_error->PendingError() == true)
 420       return false;
 421
 422    if(TF.Step(Tags) == true)
 423    {
 424       bool found = false;
 425       DiffInfo d;
 426       string size;
 427
 428       string const tmp = Tags.FindS("SHA1-Current");
 429       std::stringstream ss(tmp);
 430       ss >> ServerSha1 >> size;
 431       unsigned long const ServerSize = atol(size.c_str());
 432
 433       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 434       SHA1Summation SHA1;
 435       SHA1.AddFD(fd);
 436       string const local_sha1 = SHA1.Result();
 437
 438       if(local_sha1 == ServerSha1)
 439       {
 440          // we have the same sha1 as the server so we are done here
 441          if(Debug)
 442             std::clog << "Package file is up-to-date" << std::endl;
 443          // list cleanup needs to know that this file as well as the already
 444          // present index is ours, so we create an empty diff to save it for us
 445          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 446                ExpectedHash, ServerSha1, available_patches);
 447          return true;
 448       }
 449       else
 450       {
 451          if(Debug)
 452             std::clog << "SHA1-Current: " << ServerSha1 << " and we start at "<< fd.Name() << " " << fd.Size() << " " << local_sha1 << std::endl;
 453
 454          // check the historie and see what patches we need
 455          string const history = Tags.FindS("SHA1-History");
 456          std::stringstream hist(history);
 457          while(hist >> d.sha1 >> size >> d.file)
 458          {
 459             // read until the first match is found
 460             // from that point on, we probably need all diffs
 461             if(d.sha1 == local_sha1)
 462                found=true;
 463             else if (found == false)
 464                continue;
 465
 466             if(Debug)
 467                std::clog << "Need to get diff: " << d.file << std::endl;
 468             available_patches.push_back(d);
 469          }
 470
 471          if (available_patches.empty() == false)
 472          {
 473             // patching with too many files is rather slow compared to a fast download
 474             unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 475             if (fileLimit != 0 && fileLimit < available_patches.size())
 476             {
 477                if (Debug)
 478                   std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 479                         << ") so fallback to complete download" << std::endl;
 480                return false;
 481             }
 482
 483             // see if the patches are too big
 484             found = false; // it was true and it will be true again at the end
 485             d = *available_patches.begin();
 486             string const firstPatch = d.file;
 487             unsigned long patchesSize = 0;
 488             std::stringstream patches(Tags.FindS("SHA1-Patches"));
 489             while(patches >> d.sha1 >> size >> d.file)
 490             {
 491                if (firstPatch == d.file)
 492                   found = true;
 493                else if (found == false)
 494                   continue;
 495
 496                patchesSize += atol(size.c_str());
 497             }
 498             unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 499             if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 500             {
 501                if (Debug)
 502                   std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 503                         << ") so fallback to complete download" << std::endl;
 504                return false;
 505             }
 506          }
 507       }
 508
 509       // we have something, queue the next diff
 510       if(found)
 511       {
 512          // queue the diffs
 513          string::size_type const last_space = Description.rfind(" ");
 514          if(last_space != string::npos)
 515             Description.erase(last_space, Description.size()-last_space);
 516
 517          /* decide if we should download patches one by one or in one go:
 518             The first is good if the server merges patches, but many don't so client
 519             based merging can be attempt in which case the second is better.
 520             "bad things" will happen if patches are merged on the server,
 521             but client side merging is attempt as well */
 522          bool pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true);
 523          if (pdiff_merge == true)
 524          {
 525             // reprepro adds this flag if it has merged patches on the server
 526             std::string const precedence = Tags.FindS("X-Patch-Precedence");
 527             pdiff_merge = (precedence != "merged");
 528          }
 529
 530          if (pdiff_merge == false)
 531             new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 532                   ExpectedHash, ServerSha1, available_patches);
 533          else
 534          {
 535             std::vector<pkgAcqIndexMergeDiffs*> *diffs = new std::vector<pkgAcqIndexMergeDiffs*>(available_patches.size());
 536             for(size_t i = 0; i < available_patches.size(); ++i)
 537                (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, RealURI, Description, Desc.ShortDesc, ExpectedHash,
 538                      available_patches[i], diffs);
 539          }
 540
 541          Complete = false;
 542          Status = StatDone;
 543          Dequeue();
 544          return true;
 545       }
 546    }
 547
 548    // Nothing found, report and return false
 549    // Failing here is ok, if we return false later, the full
 550    // IndexFile is queued
 551    if(Debug)
 552       std::clog << "Can't find a patch in the index file" << std::endl;
 553    return false;
 554 }
 555                                                                         /*}}}*/
 556 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 557 {
 558    if(Debug)
 559       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl
 560                 << "Falling back to normal index file acquire" << std::endl;
 561
 562    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 563                    ExpectedHash);
 564
 565    Complete = false;
 566    Status = StatDone;
 567    Dequeue();
 568 }
 569                                                                         /*}}}*/
 570 void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,string Md5Hash,       /*{{{*/
 571                            pkgAcquire::MethodConfig *Cnf)
 572 {
 573    if(Debug)
 574       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 575
 576    Item::Done(Message,Size,Md5Hash,Cnf);
 577
 578    string FinalFile;
 579    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 580
 581    // success in downloading the index
 582    // rename the index
 583    FinalFile += string(".IndexDiff");
 584    if(Debug)
 585       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 586                 << std::endl;
 587    Rename(DestFile,FinalFile);
 588    chmod(FinalFile.c_str(),0644);
 589    DestFile = FinalFile;
 590
 591    if(!ParseDiffIndex(DestFile))
 592       return Failed("", NULL);
 593
 594    Complete = true;
 595    Status = StatDone;
 596    Dequeue();
 597    return;
 598 }
 599                                                                         /*}}}*/
 600 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 601 // ---------------------------------------------------------------------
 602 /* The package diff is added to the queue. one object is constructed
 603  * for each diff and the index
 604  */
 605 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 606                                    string URI,string URIDesc,string ShortDesc,
 607                                    HashString ExpectedHash,
 608                                    string ServerSha1,
 609                                    vector<DiffInfo> diffs)
 610    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 611      available_patches(diffs), ServerSha1(ServerSha1)
 612 {
 613
 614    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 615    DestFile += URItoFileName(URI);
 616
 617    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 618
 619    Description = URIDesc;
 620    Desc.Owner = this;
 621    Desc.ShortDesc = ShortDesc;
 622
 623    if(available_patches.empty() == true)
 624    {
 625       // we are done (yeah!)
 626       Finish(true);
 627    }
 628    else
 629    {
 630       // get the next diff
 631       State = StateFetchDiff;
 632       QueueNextDiff();
 633    }
 634 }
 635                                                                         /*}}}*/
 636 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 637 {
 638    if(Debug)
 639       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl
 640                 << "Falling back to normal index file acquire" << std::endl;
 641    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 642                    ExpectedHash);
 643    Finish();
 644 }
 645                                                                         /*}}}*/
 646 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 647 void pkgAcqIndexDiffs::Finish(bool allDone)
 648 {
 649    // we restore the original name, this is required, otherwise
 650    // the file will be cleaned
 651    if(allDone)
 652    {
 653       DestFile = _config->FindDir("Dir::State::lists");
 654       DestFile += URItoFileName(RealURI);
 655
 656       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 657       {
 658          RenameOnError(HashSumMismatch);
 659          Dequeue();
 660          return;
 661       }
 662
 663       // this is for the "real" finish
 664       Complete = true;
 665       Status = StatDone;
 666       Dequeue();
 667       if(Debug)
 668          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 669       return;
 670    }
 671
 672    if(Debug)
 673       std::clog << "Finishing: " << Desc.URI << std::endl;
 674    Complete = false;
 675    Status = StatDone;
 676    Dequeue();
 677    return;
 678 }
 679                                                                         /*}}}*/
 680 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 681 {
 682
 683    // calc sha1 of the just patched file
 684    string FinalFile = _config->FindDir("Dir::State::lists");
 685    FinalFile += URItoFileName(RealURI);
 686
 687    FileFd fd(FinalFile, FileFd::ReadOnly);
 688    SHA1Summation SHA1;
 689    SHA1.AddFD(fd);
 690    string local_sha1 = string(SHA1.Result());
 691    if(Debug)
 692       std::clog << "QueueNextDiff: "
 693                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 694
 695    // final file reached before all patches are applied
 696    if(local_sha1 == ServerSha1)
 697    {
 698       Finish(true);
 699       return true;
 700    }
 701
 702    // remove all patches until the next matching patch is found
 703    // this requires the Index file to be ordered
 704    for(vector<DiffInfo>::iterator I=available_patches.begin();
 705        available_patches.empty() == false &&
 706           I != available_patches.end() &&
 707           I->sha1 != local_sha1;
 708        ++I)
 709    {
 710       available_patches.erase(I);
 711    }
 712
 713    // error checking and falling back if no patch was found
 714    if(available_patches.empty() == true)
 715    {
 716       Failed("", NULL);
 717       return false;
 718    }
 719
 720    // queue the right diff
 721    Desc.URI = RealURI + ".diff/" + available_patches[0].file + ".gz";
 722    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 723    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 724    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 725
 726    if(Debug)
 727       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 728
 729    QueueURI(Desc);
 730
 731    return true;
 732 }
 733                                                                         /*}}}*/
 734 void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size,string Md5Hash,      /*{{{*/
 735                             pkgAcquire::MethodConfig *Cnf)
 736 {
 737    if(Debug)
 738       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 739
 740    Item::Done(Message,Size,Md5Hash,Cnf);
 741
 742    string FinalFile;
 743    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 744
 745    // success in downloading a diff, enter ApplyDiff state
 746    if(State == StateFetchDiff)
 747    {
 748
 749       // rred excepts the patch as $FinalFile.ed
 750       Rename(DestFile,FinalFile+".ed");
 751
 752       if(Debug)
 753          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 754
 755       State = StateApplyDiff;
 756       Local = true;
 757       Desc.URI = "rred:" + FinalFile;
 758       QueueURI(Desc);
 759       Mode = "rred";
 760       return;
 761    }
 762
 763
 764    // success in download/apply a diff, queue next (if needed)
 765    if(State == StateApplyDiff)
 766    {
 767       // remove the just applied patch
 768       available_patches.erase(available_patches.begin());
 769       unlink((FinalFile + ".ed").c_str());
 770
 771       // move into place
 772       if(Debug)
 773       {
 774          std::clog << "Moving patched file in place: " << std::endl
 775                    << DestFile << " -> " << FinalFile << std::endl;
 776       }
 777       Rename(DestFile,FinalFile);
 778       chmod(FinalFile.c_str(),0644);
 779
 780       // see if there is more to download
 781       if(available_patches.empty() == false) {
 782          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 783                               ExpectedHash, ServerSha1, available_patches);
 784          return Finish();
 785       } else
 786          return Finish(true);
 787    }
 788 }
 789                                                                         /*}}}*/
 790 // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor                 /*{{{*/
 791 pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner,
 792                                    string const &URI, string const &URIDesc,
 793                                    string const &ShortDesc, HashString const &ExpectedHash,
 794                                    DiffInfo const &patch,
 795                                    std::vector<pkgAcqIndexMergeDiffs*> const * const allPatches)
 796    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 797      patch(patch),allPatches(allPatches), State(StateFetchDiff)
 798 {
 799
 800    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 801    DestFile += URItoFileName(URI);
 802
 803    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 804
 805    Description = URIDesc;
 806    Desc.Owner = this;
 807    Desc.ShortDesc = ShortDesc;
 808
 809    Desc.URI = RealURI + ".diff/" + patch.file + ".gz";
 810    Desc.Description = Description + " " + patch.file + string(".pdiff");
 811    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 812    DestFile += URItoFileName(RealURI + ".diff/" + patch.file);
 813
 814    if(Debug)
 815       std::clog << "pkgAcqIndexMergeDiffs: " << Desc.URI << std::endl;
 816
 817    QueueURI(Desc);
 818 }
 819                                                                         /*}}}*/
 820 void pkgAcqIndexMergeDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 821 {
 822    if(Debug)
 823       std::clog << "pkgAcqIndexMergeDiffs failed: " << Desc.URI << " with " << Message << std::endl;
 824    Complete = false;
 825    Status = StatDone;
 826    Dequeue();
 827
 828    // check if we are the first to fail, otherwise we are done here
 829    State = StateDoneDiff;
 830    for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 831          I != allPatches->end(); ++I)
 832       if ((*I)->State == StateErrorDiff)
 833          return;
 834
 835    // first failure means we should fallback
 836    State = StateErrorDiff;
 837    std::clog << "Falling back to normal index file acquire" << std::endl;
 838    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 839                    ExpectedHash);
 840 }
 841                                                                         /*}}}*/
 842 void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,string Md5Hash, /*{{{*/
 843                             pkgAcquire::MethodConfig *Cnf)
 844 {
 845    if(Debug)
 846       std::clog << "pkgAcqIndexMergeDiffs::Done(): " << Desc.URI << std::endl;
 847
 848    Item::Done(Message,Size,Md5Hash,Cnf);
 849
 850    string const FinalFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 851
 852    if (State == StateFetchDiff)
 853    {
 854       // rred expects the patch as $FinalFile.ed.$patchname.gz
 855       Rename(DestFile, FinalFile + ".ed." + patch.file + ".gz");
 856
 857       // check if this is the last completed diff
 858       State = StateDoneDiff;
 859       for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 860             I != allPatches->end(); ++I)
 861          if ((*I)->State != StateDoneDiff)
 862          {
 863             if(Debug)
 864                std::clog << "Not the last done diff in the batch: " << Desc.URI << std::endl;
 865             return;
 866          }
 867
 868       // this is the last completed diff, so we are ready to apply now
 869       State = StateApplyDiff;
 870
 871       if(Debug)
 872          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 873
 874       Local = true;
 875       Desc.URI = "rred:" + FinalFile;
 876       QueueURI(Desc);
 877       Mode = "rred";
 878       return;
 879    }
 880    // success in download/apply all diffs, clean up
 881    else if (State == StateApplyDiff)
 882    {
 883       // see if we really got the expected file
 884       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 885       {
 886          RenameOnError(HashSumMismatch);
 887          return;
 888       }
 889
 890       // move the result into place
 891       if(Debug)
 892          std::clog << "Moving patched file in place: " << std::endl
 893                    << DestFile << " -> " << FinalFile << std::endl;
 894       Rename(DestFile, FinalFile);
 895       chmod(FinalFile.c_str(), 0644);
 896
 897       // otherwise lists cleanup will eat the file
 898       DestFile = FinalFile;
 899
 900       // ensure the ed's are gone regardless of list-cleanup
 901       for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 902             I != allPatches->end(); ++I)
 903       {
 904             std::string patch = FinalFile + ".ed." + (*I)->patch.file + ".gz";
 905             unlink(patch.c_str());
 906       }
 907
 908       // all set and done
 909       Complete = true;
 910       if(Debug)
 911          std::clog << "allDone: " << DestFile << "\n" << std::endl;
 912    }
 913 }
 914                                                                         /*}}}*/
 915 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 916 // ---------------------------------------------------------------------
 917 /* The package file is added to the queue and a second class is
 918    instantiated to fetch the revision file */
 919 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 920                          string URI,string URIDesc,string ShortDesc,
 921                          HashString ExpectedHash, string comprExt)
 922    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 923 {
 924    if(comprExt.empty() == true)
 925    {
 926       // autoselect the compression method
 927       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 928       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 929          comprExt.append(*t).append(" ");
 930       if (comprExt.empty() == false)
 931          comprExt.erase(comprExt.end()-1);
 932    }
 933    CompressionExtension = comprExt;
 934
 935    Init(URI, URIDesc, ShortDesc);
 936 }
 937 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target,
 938                          HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
 939    : Item(Owner), RealURI(Target->URI), ExpectedHash(ExpectedHash)
 940 {
 941    // autoselect the compression method
 942    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 943    CompressionExtension = "";
 944    if (ExpectedHash.empty() == false)
 945    {
 946       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 947          if (*t == "uncompressed" || MetaIndexParser->Exists(string(Target->MetaKey).append(".").append(*t)) == true)
 948             CompressionExtension.append(*t).append(" ");
 949    }
 950    else
 951    {
 952       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 953          CompressionExtension.append(*t).append(" ");
 954    }
 955    if (CompressionExtension.empty() == false)
 956       CompressionExtension.erase(CompressionExtension.end()-1);
 957
 958    // only verify non-optional targets, see acquire-item.h for a FIXME
 959    // to make this more flexible
 960    if (Target->IsOptional())
 961      Verify = false;
 962    else
 963      Verify = true;
 964
 965    Init(Target->URI, Target->Description, Target->ShortDesc);
 966 }
 967                                                                         /*}}}*/
 968 // AcqIndex::Init - defered Constructor                                 /*{{{*/
 969 void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &ShortDesc) {
 970    Decompression = false;
 971    Erase = false;
 972
 973    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 974    DestFile += URItoFileName(URI);
 975
 976    std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
 977    if (comprExt == "uncompressed")
 978       Desc.URI = URI;
 979    else
 980       Desc.URI = URI + '.' + comprExt;
 981
 982    Desc.Description = URIDesc;
 983    Desc.Owner = this;
 984    Desc.ShortDesc = ShortDesc;
 985
 986    QueueURI(Desc);
 987 }
 988                                                                         /*}}}*/
 989 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 990 // ---------------------------------------------------------------------
 991 /* The only header we use is the last-modified header. */
 992 string pkgAcqIndex::Custom600Headers()
 993 {
 994    string Final = _config->FindDir("Dir::State::lists");
 995    Final += URItoFileName(RealURI);
 996    if (_config->FindB("Acquire::GzipIndexes",false))
 997       Final += ".gz";
 998
 999    string msg = "\nIndex-File: true";
1000    // FIXME: this really should use "IndexTarget::IsOptional()" but that
1001    //        seems to be difficult without breaking ABI
1002    if (ShortDesc().find("Translation") != 0)
1003       msg += "\nFail-Ignore: true";
1004    struct stat Buf;
1005    if (stat(Final.c_str(),&Buf) == 0)
1006       msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1007
1008    return msg;
1009 }
1010                                                                         /*}}}*/
1011 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
1012 {
1013    size_t const nextExt = CompressionExtension.find(' ');
1014    if (nextExt != std::string::npos)
1015    {
1016       CompressionExtension = CompressionExtension.substr(nextExt+1);
1017       Init(RealURI, Desc.Description, Desc.ShortDesc);
1018       return;
1019    }
1020
1021    // on decompression failure, remove bad versions in partial/
1022    if (Decompression && Erase) {
1023       string s = _config->FindDir("Dir::State::lists") + "partial/";
1024       s.append(URItoFileName(RealURI));
1025       unlink(s.c_str());
1026    }
1027
1028    Item::Failed(Message,Cnf);
1029 }
1030                                                                         /*}}}*/
1031 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
1032 // ---------------------------------------------------------------------
1033 /* This goes through a number of states.. On the initial fetch the
1034    method could possibly return an alternate filename which points
1035    to the uncompressed version of the file. If this is so the file
1036    is copied into the partial directory. In all other cases the file
1037    is decompressed with a gzip uri. */
1038 void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash,
1039                        pkgAcquire::MethodConfig *Cfg)
1040 {
1041    Item::Done(Message,Size,Hash,Cfg);
1042
1043    if (Decompression == true)
1044    {
1045       if (_config->FindB("Debug::pkgAcquire::Auth", false))
1046       {
1047          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
1048          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
1049       }
1050
1051       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
1052       {
1053          RenameOnError(HashSumMismatch);
1054          return;
1055       }
1056
1057       /* Verify the index file for correctness (all indexes must
1058        * have a Package field) (LP: #346386) (Closes: #627642) */
1059       if (Verify == true)
1060       {
1061          FileFd fd(DestFile, FileFd::ReadOnly);
1062          // Only test for correctness if the file is not empty (empty is ok)
1063          if (fd.FileSize() > 0)
1064          {
1065             pkgTagSection sec;
1066             pkgTagFile tag(&fd);
1067
1068             // all our current indexes have a field 'Package' in each section
1069             if (_error->PendingError() == true || tag.Step(sec) == false || sec.Exists("Package") == false)
1070             {
1071                RenameOnError(InvalidFormat);
1072                return;
1073             }
1074          }
1075       }
1076
1077       // Done, move it into position
1078       string FinalFile = _config->FindDir("Dir::State::lists");
1079       FinalFile += URItoFileName(RealURI);
1080       Rename(DestFile,FinalFile);
1081       chmod(FinalFile.c_str(),0644);
1082
1083       /* We restore the original name to DestFile so that the clean operation
1084          will work OK */
1085       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1086       DestFile += URItoFileName(RealURI);
1087
1088       // Remove the compressed version.
1089       if (Erase == true)
1090          unlink(DestFile.c_str());
1091       return;
1092    }
1093
1094    Erase = false;
1095    Complete = true;
1096
1097    // Handle the unzipd case
1098    string FileName = LookupTag(Message,"Alt-Filename");
1099    if (FileName.empty() == false)
1100    {
1101       // The files timestamp matches
1102       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
1103          return;
1104       Decompression = true;
1105       Local = true;
1106       DestFile += ".decomp";
1107       Desc.URI = "copy:" + FileName;
1108       QueueURI(Desc);
1109       Mode = "copy";
1110       return;
1111    }
1112
1113    FileName = LookupTag(Message,"Filename");
1114    if (FileName.empty() == true)
1115    {
1116       Status = StatError;
1117       ErrorText = "Method gave a blank filename";
1118    }
1119
1120    std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
1121
1122    // The files timestamp matches
1123    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) {
1124        if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
1125           // Update DestFile for .gz suffix so that the clean operation keeps it
1126           DestFile += ".gz";
1127       return;
1128     }
1129
1130    if (FileName == DestFile)
1131       Erase = true;
1132    else
1133       Local = true;
1134
1135    string decompProg;
1136
1137    // If we enable compressed indexes and already have gzip, keep it
1138    if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) {
1139       string FinalFile = _config->FindDir("Dir::State::lists");
1140       FinalFile += URItoFileName(RealURI) + ".gz";
1141       Rename(DestFile,FinalFile);
1142       chmod(FinalFile.c_str(),0644);
1143
1144       // Update DestFile for .gz suffix so that the clean operation keeps it
1145       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1146       DestFile += URItoFileName(RealURI) + ".gz";
1147       return;
1148     }
1149
1150    // get the binary name for your used compression type
1151    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
1152    if(decompProg.empty() == false);
1153    else if(compExt == "uncompressed")
1154       decompProg = "copy";
1155    else {
1156       _error->Error("Unsupported extension: %s", compExt.c_str());
1157       return;
1158    }
1159
1160    Decompression = true;
1161    DestFile += ".decomp";
1162    Desc.URI = decompProg + ":" + FileName;
1163    QueueURI(Desc);
1164
1165    // FIXME: this points to a c++ string that goes out of scope
1166    Mode = decompProg.c_str();
1167 }
1168                                                                         /*}}}*/
1169 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
1170 // ---------------------------------------------------------------------
1171 /* The Translation file is added to the queue */
1172 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
1173                             string URI,string URIDesc,string ShortDesc)
1174   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
1175 {
1176 }
1177 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const *Target,
1178                          HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
1179   : pkgAcqIndex(Owner, Target, ExpectedHash, MetaIndexParser)
1180 {
1181 }
1182                                                                         /*}}}*/
1183 // AcqIndexTrans::Custom600Headers - Insert custom request headers      /*{{{*/
1184 // ---------------------------------------------------------------------
1185 string pkgAcqIndexTrans::Custom600Headers()
1186 {
1187    string Final = _config->FindDir("Dir::State::lists");
1188    Final += URItoFileName(RealURI);
1189
1190    struct stat Buf;
1191    if (stat(Final.c_str(),&Buf) != 0)
1192       return "\nFail-Ignore: true\nIndex-File: true";
1193    return "\nFail-Ignore: true\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1194 }
1195                                                                         /*}}}*/
1196 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
1197 // ---------------------------------------------------------------------
1198 /* */
1199 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1200 {
1201    size_t const nextExt = CompressionExtension.find(' ');
1202    if (nextExt != std::string::npos)
1203    {
1204       CompressionExtension = CompressionExtension.substr(nextExt+1);
1205       Init(RealURI, Desc.Description, Desc.ShortDesc);
1206       Status = StatIdle;
1207       return;
1208    }
1209
1210    if (Cnf->LocalOnly == true ||
1211        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1212    {
1213       // Ignore this
1214       Status = StatDone;
1215       Complete = false;
1216       Dequeue();
1217       return;
1218    }
1219
1220    Item::Failed(Message,Cnf);
1221 }
1222                                                                         /*}}}*/
1223 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
1224                              string URI,string URIDesc,string ShortDesc,
1225                              string MetaIndexURI, string MetaIndexURIDesc,
1226                              string MetaIndexShortDesc,
1227                              const vector<IndexTarget*>* IndexTargets,
1228                              indexRecords* MetaIndexParser) :
1229    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
1230    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1231    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
1232 {
1233    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1234    DestFile += URItoFileName(URI);
1235
1236    // remove any partial downloaded sig-file in partial/.
1237    // it may confuse proxies and is too small to warrant a
1238    // partial download anyway
1239    unlink(DestFile.c_str());
1240
1241    // Create the item
1242    Desc.Description = URIDesc;
1243    Desc.Owner = this;
1244    Desc.ShortDesc = ShortDesc;
1245    Desc.URI = URI;
1246
1247    string Final = _config->FindDir("Dir::State::lists");
1248    Final += URItoFileName(RealURI);
1249    if (RealFileExists(Final) == true)
1250    {
1251       // File was already in place.  It needs to be re-downloaded/verified
1252       // because Release might have changed, we do give it a different
1253       // name than DestFile because otherwise the http method will
1254       // send If-Range requests and there are too many broken servers
1255       // out there that do not understand them
1256       LastGoodSig = DestFile+".reverify";
1257       Rename(Final,LastGoodSig);
1258    }
1259
1260    QueueURI(Desc);
1261 }
1262                                                                         /*}}}*/
1263 pkgAcqMetaSig::~pkgAcqMetaSig()                                         /*{{{*/
1264 {
1265    // if the file was never queued undo file-changes done in the constructor
1266    if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false &&
1267          LastGoodSig.empty() == false)
1268    {
1269       string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1270       if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true)
1271          Rename(LastGoodSig, Final);
1272    }
1273
1274 }
1275                                                                         /*}}}*/
1276 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
1277 // ---------------------------------------------------------------------
1278 /* The only header we use is the last-modified header. */
1279 string pkgAcqMetaSig::Custom600Headers()
1280 {
1281    struct stat Buf;
1282    if (stat(LastGoodSig.c_str(),&Buf) != 0)
1283       return "\nIndex-File: true";
1284
1285    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1286 }
1287
1288 void pkgAcqMetaSig::Done(string Message,unsigned long long Size,string MD5,
1289                          pkgAcquire::MethodConfig *Cfg)
1290 {
1291    Item::Done(Message,Size,MD5,Cfg);
1292
1293    string FileName = LookupTag(Message,"Filename");
1294    if (FileName.empty() == true)
1295    {
1296       Status = StatError;
1297       ErrorText = "Method gave a blank filename";
1298       return;
1299    }
1300
1301    if (FileName != DestFile)
1302    {
1303       // We have to copy it into place
1304       Local = true;
1305       Desc.URI = "copy:" + FileName;
1306       QueueURI(Desc);
1307       return;
1308    }
1309
1310    Complete = true;
1311
1312    // put the last known good file back on i-m-s hit (it will
1313    // be re-verified again)
1314    // Else do nothing, we have the new file in DestFile then
1315    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1316       Rename(LastGoodSig, DestFile);
1317
1318    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
1319    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc,
1320                        MetaIndexShortDesc,  DestFile, IndexTargets,
1321                        MetaIndexParser);
1322
1323 }
1324                                                                         /*}}}*/
1325 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
1326 {
1327    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1328
1329    // if we get a network error we fail gracefully
1330    if(Status == StatTransientNetworkError)
1331    {
1332       Item::Failed(Message,Cnf);
1333       // move the sigfile back on transient network failures
1334       if(FileExists(LastGoodSig))
1335          Rename(LastGoodSig,Final);
1336
1337       // set the status back to , Item::Failed likes to reset it
1338       Status = pkgAcquire::Item::StatTransientNetworkError;
1339       return;
1340    }
1341
1342    // Delete any existing sigfile when the acquire failed
1343    unlink(Final.c_str());
1344
1345    // queue a pkgAcqMetaIndex with no sigfile
1346    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1347                        "", IndexTargets, MetaIndexParser);
1348
1349    if (Cnf->LocalOnly == true ||
1350        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1351    {
1352       // Ignore this
1353       Status = StatDone;
1354       Complete = false;
1355       Dequeue();
1356       return;
1357    }
1358
1359    Item::Failed(Message,Cnf);
1360 }
1361                                                                         /*}}}*/
1362 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
1363                                  string URI,string URIDesc,string ShortDesc,
1364                                  string SigFile,
1365                                  const vector<struct IndexTarget*>* IndexTargets,
1366                                  indexRecords* MetaIndexParser) :
1367    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
1368    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
1369 {
1370    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1371    DestFile += URItoFileName(URI);
1372
1373    // Create the item
1374    Desc.Description = URIDesc;
1375    Desc.Owner = this;
1376    Desc.ShortDesc = ShortDesc;
1377    Desc.URI = URI;
1378
1379    QueueURI(Desc);
1380 }
1381                                                                         /*}}}*/
1382 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
1383 // ---------------------------------------------------------------------
1384 /* The only header we use is the last-modified header. */
1385 string pkgAcqMetaIndex::Custom600Headers()
1386 {
1387    string Final = _config->FindDir("Dir::State::lists");
1388    Final += URItoFileName(RealURI);
1389
1390    struct stat Buf;
1391    if (stat(Final.c_str(),&Buf) != 0)
1392       return "\nIndex-File: true";
1393
1394    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1395 }
1396                                                                         /*}}}*/
1397 void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,string Hash,  /*{{{*/
1398                            pkgAcquire::MethodConfig *Cfg)
1399 {
1400    Item::Done(Message,Size,Hash,Cfg);
1401
1402    // MetaIndexes are done in two passes: one to download the
1403    // metaindex with an appropriate method, and a second to verify it
1404    // with the gpgv method
1405
1406    if (AuthPass == true)
1407    {
1408       AuthDone(Message);
1409
1410       // all cool, move Release file into place
1411       Complete = true;
1412    }
1413    else
1414    {
1415       RetrievalDone(Message);
1416       if (!Complete)
1417          // Still more retrieving to do
1418          return;
1419
1420       if (SigFile == "")
1421       {
1422          // There was no signature file, so we are finished.  Download
1423          // the indexes and do only hashsum verification if possible
1424          MetaIndexParser->Load(DestFile);
1425          QueueIndexes(false);
1426       }
1427       else
1428       {
1429          // FIXME: move this into pkgAcqMetaClearSig::Done on the next
1430          //        ABI break
1431
1432          // if we expect a ClearTextSignature (InRelase), ensure that
1433          // this is what we get and if not fail to queue a
1434          // Release/Release.gpg, see #346386
1435          if (SigFile == DestFile && !StartsWithGPGClearTextSignature(DestFile))
1436          {
1437             Failed(Message, Cfg);
1438             return;
1439          }
1440
1441          // There was a signature file, so pass it to gpgv for
1442          // verification
1443          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1444             std::cerr << "Metaindex acquired, queueing gpg verification ("
1445                       << SigFile << "," << DestFile << ")\n";
1446          AuthPass = true;
1447          Desc.URI = "gpgv:" + SigFile;
1448          QueueURI(Desc);
1449          Mode = "gpgv";
1450          return;
1451       }
1452    }
1453
1454    if (Complete == true)
1455    {
1456       string FinalFile = _config->FindDir("Dir::State::lists");
1457       FinalFile += URItoFileName(RealURI);
1458       if (SigFile == DestFile)
1459          SigFile = FinalFile;
1460       Rename(DestFile,FinalFile);
1461       chmod(FinalFile.c_str(),0644);
1462       DestFile = FinalFile;
1463    }
1464 }
1465                                                                         /*}}}*/
1466 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
1467 {
1468    // We have just finished downloading a Release file (it is not
1469    // verified yet)
1470
1471    string FileName = LookupTag(Message,"Filename");
1472    if (FileName.empty() == true)
1473    {
1474       Status = StatError;
1475       ErrorText = "Method gave a blank filename";
1476       return;
1477    }
1478
1479    if (FileName != DestFile)
1480    {
1481       Local = true;
1482       Desc.URI = "copy:" + FileName;
1483       QueueURI(Desc);
1484       return;
1485    }
1486
1487    // make sure to verify against the right file on I-M-S hit
1488    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1489    if(IMSHit)
1490    {
1491       string FinalFile = _config->FindDir("Dir::State::lists");
1492       FinalFile += URItoFileName(RealURI);
1493       if (SigFile == DestFile)
1494       {
1495          SigFile = FinalFile;
1496          // constructor of pkgAcqMetaClearSig moved it out of the way,
1497          // now move it back in on IMS hit for the 'old' file
1498          string const OldClearSig = DestFile + ".reverify";
1499          if (RealFileExists(OldClearSig) == true)
1500             Rename(OldClearSig, FinalFile);
1501       }
1502       DestFile = FinalFile;
1503    }
1504    Complete = true;
1505 }
1506                                                                         /*}}}*/
1507 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1508 {
1509    // At this point, the gpgv method has succeeded, so there is a
1510    // valid signature from a key in the trusted keyring.  We
1511    // perform additional verification of its contents, and use them
1512    // to verify the indexes we are about to download
1513
1514    if (!MetaIndexParser->Load(DestFile))
1515    {
1516       Status = StatAuthError;
1517       ErrorText = MetaIndexParser->ErrorText;
1518       return;
1519    }
1520
1521    if (!VerifyVendor(Message))
1522    {
1523       return;
1524    }
1525
1526    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1527       std::cerr << "Signature verification succeeded: "
1528                 << DestFile << std::endl;
1529
1530    // Download further indexes with verification
1531    QueueIndexes(true);
1532
1533    // is it a clearsigned MetaIndex file?
1534    if (DestFile == SigFile)
1535       return;
1536
1537    // Done, move signature file into position
1538    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1539       URItoFileName(RealURI) + ".gpg";
1540    Rename(SigFile,VerifiedSigFile);
1541    chmod(VerifiedSigFile.c_str(),0644);
1542 }
1543                                                                         /*}}}*/
1544 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1545 {
1546 #if 0
1547    /* Reject invalid, existing Release files (LP: #346386) (Closes: #627642)
1548     * FIXME: Disabled; it breaks unsigned repositories without hashes */
1549    if (!verify && FileExists(DestFile) && !MetaIndexParser->Load(DestFile))
1550    {
1551       Status = StatError;
1552       ErrorText = MetaIndexParser->ErrorText;
1553       return;
1554    }
1555 #endif
1556    bool transInRelease = false;
1557    {
1558       std::vector<std::string> const keys = MetaIndexParser->MetaKeys();
1559       for (std::vector<std::string>::const_iterator k = keys.begin(); k != keys.end(); ++k)
1560          // FIXME: Feels wrong to check for hardcoded string here, but what should we do else…
1561          if (k->find("Translation-") != std::string::npos)
1562          {
1563             transInRelease = true;
1564             break;
1565          }
1566    }
1567
1568    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1569         Target != IndexTargets->end();
1570         ++Target)
1571    {
1572       HashString ExpectedIndexHash;
1573       const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1574       bool compressedAvailable = false;
1575       if (Record == NULL)
1576       {
1577          if ((*Target)->IsOptional() == true)
1578          {
1579             std::vector<std::string> types = APT::Configuration::getCompressionTypes();
1580             for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
1581                if (MetaIndexParser->Exists((*Target)->MetaKey + "." + *t) == true)
1582                {
1583                   compressedAvailable = true;
1584                   break;
1585                }
1586          }
1587          else if (verify == true)
1588          {
1589             Status = StatAuthError;
1590             strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), (*Target)->MetaKey.c_str());
1591             return;
1592          }
1593       }
1594       else
1595       {
1596          ExpectedIndexHash = Record->Hash;
1597          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1598          {
1599             std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1600             std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1601             std::cerr << "For: " << Record->MetaKeyFilename << std::endl;
1602          }
1603          if (verify == true && ExpectedIndexHash.empty() == true && (*Target)->IsOptional() == false)
1604          {
1605             Status = StatAuthError;
1606             strprintf(ErrorText, _("Unable to find hash sum for '%s' in Release file"), (*Target)->MetaKey.c_str());
1607             return;
1608          }
1609       }
1610
1611       if ((*Target)->IsOptional() == true)
1612       {
1613          if ((*Target)->IsSubIndex() == true)
1614             new pkgAcqSubIndex(Owner, (*Target)->URI, (*Target)->Description,
1615                                 (*Target)->ShortDesc, ExpectedIndexHash);
1616          else if (transInRelease == false || Record != NULL || compressedAvailable == true)
1617          {
1618             if (_config->FindB("Acquire::PDiffs",true) == true && transInRelease == true &&
1619                 MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true)
1620                new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1621                                    (*Target)->ShortDesc, ExpectedIndexHash);
1622             else
1623                new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1624          }
1625          continue;
1626       }
1627
1628       /* Queue Packages file (either diff or full packages files, depending
1629          on the users option) - we also check if the PDiff Index file is listed
1630          in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
1631          instead, but passing the required info to it is to much hassle */
1632       if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
1633           MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true))
1634          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1635                              (*Target)->ShortDesc, ExpectedIndexHash);
1636       else
1637          new pkgAcqIndex(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1638    }
1639 }
1640                                                                         /*}}}*/
1641 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1642 {
1643    string::size_type pos;
1644
1645    // check for missing sigs (that where not fatal because otherwise we had
1646    // bombed earlier)
1647    string missingkeys;
1648    string msg = _("There is no public key available for the "
1649                   "following key IDs:\n");
1650    pos = Message.find("NO_PUBKEY ");
1651    if (pos != std::string::npos)
1652    {
1653       string::size_type start = pos+strlen("NO_PUBKEY ");
1654       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1655       missingkeys += (Fingerprint);
1656    }
1657    if(!missingkeys.empty())
1658       _error->Warning("%s", (msg + missingkeys).c_str());
1659
1660    string Transformed = MetaIndexParser->GetExpectedDist();
1661
1662    if (Transformed == "../project/experimental")
1663    {
1664       Transformed = "experimental";
1665    }
1666
1667    pos = Transformed.rfind('/');
1668    if (pos != string::npos)
1669    {
1670       Transformed = Transformed.substr(0, pos);
1671    }
1672
1673    if (Transformed == ".")
1674    {
1675       Transformed = "";
1676    }
1677
1678    if (_config->FindB("Acquire::Check-Valid-Until", true) == true &&
1679        MetaIndexParser->GetValidUntil() > 0) {
1680       time_t const invalid_since = time(NULL) - MetaIndexParser->GetValidUntil();
1681       if (invalid_since > 0)
1682          // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1683          // the time since then the file is invalid - formated in the same way as in
1684          // the download progress display (e.g. 7d 3h 42min 1s)
1685          return _error->Error(
1686             _("Release file for %s is expired (invalid since %s). "
1687               "Updates for this repository will not be applied."),
1688             RealURI.c_str(), TimeToStr(invalid_since).c_str());
1689    }
1690
1691    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1692    {
1693       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1694       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1695       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1696    }
1697
1698    if (MetaIndexParser->CheckDist(Transformed) == false)
1699    {
1700       // This might become fatal one day
1701 //       Status = StatAuthError;
1702 //       ErrorText = "Conflicting distribution; expected "
1703 //          + MetaIndexParser->GetExpectedDist() + " but got "
1704 //          + MetaIndexParser->GetDist();
1705 //       return false;
1706       if (!Transformed.empty())
1707       {
1708          _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1709                          Desc.Description.c_str(),
1710                          Transformed.c_str(),
1711                          MetaIndexParser->GetDist().c_str());
1712       }
1713    }
1714
1715    return true;
1716 }
1717                                                                         /*}}}*/
1718 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1719 // ---------------------------------------------------------------------
1720 /* */
1721 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)
1722 {
1723    if (AuthPass == true)
1724    {
1725       // gpgv method failed, if we have a good signature
1726       string LastGoodSigFile = _config->FindDir("Dir::State::lists").append("partial/").append(URItoFileName(RealURI));
1727       if (DestFile != SigFile)
1728          LastGoodSigFile.append(".gpg");
1729       LastGoodSigFile.append(".reverify");
1730
1731       if(FileExists(LastGoodSigFile))
1732       {
1733          string VerifiedSigFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1734          if (DestFile != SigFile)
1735             VerifiedSigFile.append(".gpg");
1736          Rename(LastGoodSigFile, VerifiedSigFile);
1737          Status = StatTransientNetworkError;
1738          _error->Warning(_("An error occurred during the signature "
1739                            "verification. The repository is not updated "
1740                            "and the previous index files will be used. "
1741                            "GPG error: %s: %s\n"),
1742                          Desc.Description.c_str(),
1743                          LookupTag(Message,"Message").c_str());
1744          RunScripts("APT::Update::Auth-Failure");
1745          return;
1746       } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) {
1747          /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */
1748          _error->Error(_("GPG error: %s: %s"),
1749                          Desc.Description.c_str(),
1750                          LookupTag(Message,"Message").c_str());
1751          return;
1752       } else {
1753          _error->Warning(_("GPG error: %s: %s"),
1754                          Desc.Description.c_str(),
1755                          LookupTag(Message,"Message").c_str());
1756       }
1757       // gpgv method failed
1758       ReportMirrorFailure("GPGFailure");
1759    }
1760
1761    /* Always move the meta index, even if gpgv failed. This ensures
1762     * that PackageFile objects are correctly filled in */
1763    if (FileExists(DestFile)) {
1764       string FinalFile = _config->FindDir("Dir::State::lists");
1765       FinalFile += URItoFileName(RealURI);
1766       /* InRelease files become Release files, otherwise
1767        * they would be considered as trusted later on */
1768       if (SigFile == DestFile) {
1769          RealURI = RealURI.replace(RealURI.rfind("InRelease"), 9,
1770                                        "Release");
1771          FinalFile = FinalFile.replace(FinalFile.rfind("InRelease"), 9,
1772                                        "Release");
1773          SigFile = FinalFile;
1774       }
1775       Rename(DestFile,FinalFile);
1776       chmod(FinalFile.c_str(),0644);
1777
1778       DestFile = FinalFile;
1779    }
1780
1781    // No Release file was present, or verification failed, so fall
1782    // back to queueing Packages files without verification
1783    QueueIndexes(false);
1784 }
1785                                                                         /*}}}*/
1786 pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner,               /*{{{*/
1787                 string const &URI, string const &URIDesc, string const &ShortDesc,
1788                 string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc,
1789                 string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc,
1790                 const vector<struct IndexTarget*>* IndexTargets,
1791                 indexRecords* MetaIndexParser) :
1792         pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, "", IndexTargets, MetaIndexParser),
1793         MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1794         MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc)
1795 {
1796    SigFile = DestFile;
1797
1798    // keep the old InRelease around in case of transistent network errors
1799    string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1800    if (RealFileExists(Final) == true)
1801    {
1802       string const LastGoodSig = DestFile + ".reverify";
1803       Rename(Final,LastGoodSig);
1804    }
1805 }
1806                                                                         /*}}}*/
1807 pkgAcqMetaClearSig::~pkgAcqMetaClearSig()                               /*{{{*/
1808 {
1809    // if the file was never queued undo file-changes done in the constructor
1810    if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false)
1811    {
1812       string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1813       string const LastGoodSig = DestFile + ".reverify";
1814       if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true)
1815          Rename(LastGoodSig, Final);
1816    }
1817 }
1818                                                                         /*}}}*/
1819 // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/
1820 // ---------------------------------------------------------------------
1821 // FIXME: this can go away once the InRelease file is used widely
1822 string pkgAcqMetaClearSig::Custom600Headers()
1823 {
1824    string Final = _config->FindDir("Dir::State::lists");
1825    Final += URItoFileName(RealURI);
1826
1827    struct stat Buf;
1828    if (stat(Final.c_str(),&Buf) != 0)
1829    {
1830       Final = DestFile + ".reverify";
1831       if (stat(Final.c_str(),&Buf) != 0)
1832          return "\nIndex-File: true\nFail-Ignore: true\n";
1833    }
1834
1835    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1836 }
1837                                                                         /*}}}*/
1838 void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/
1839 {
1840    if (AuthPass == false)
1841    {
1842       // Remove the 'old' InRelease file if we try Release.gpg now as otherwise
1843       // the file will stay around and gives a false-auth impression (CVE-2012-0214)
1844       string FinalFile = _config->FindDir("Dir::State::lists");
1845       FinalFile.append(URItoFileName(RealURI));
1846       if (FileExists(FinalFile))
1847          unlink(FinalFile.c_str());
1848
1849       new pkgAcqMetaSig(Owner,
1850                         MetaSigURI, MetaSigURIDesc, MetaSigShortDesc,
1851                         MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1852                         IndexTargets, MetaIndexParser);
1853       if (Cnf->LocalOnly == true ||
1854           StringToBool(LookupTag(Message, "Transient-Failure"), false) == false)
1855          Dequeue();
1856    }
1857    else
1858       pkgAcqMetaIndex::Failed(Message, Cnf);
1859 }
1860                                                                         /*}}}*/
1861 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1862 // ---------------------------------------------------------------------
1863 /* This just sets up the initial fetch environment and queues the first
1864    possibilitiy */
1865 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1866                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1867                              string &StoreFilename) :
1868                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1869                StoreFilename(StoreFilename), Vf(Version.FileList()),
1870                Trusted(false)
1871 {
1872    Retries = _config->FindI("Acquire::Retries",0);
1873
1874    if (Version.Arch() == 0)
1875    {
1876       _error->Error(_("I wasn't able to locate a file for the %s package. "
1877                       "This might mean you need to manually fix this package. "
1878                       "(due to missing arch)"),
1879                     Version.ParentPkg().FullName().c_str());
1880       return;
1881    }
1882
1883    /* We need to find a filename to determine the extension. We make the
1884       assumption here that all the available sources for this version share
1885       the same extension.. */
1886    // Skip not source sources, they do not have file fields.
1887    for (; Vf.end() == false; ++Vf)
1888    {
1889       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1890          continue;
1891       break;
1892    }
1893
1894    // Does not really matter here.. we are going to fail out below
1895    if (Vf.end() != true)
1896    {
1897       // If this fails to get a file name we will bomb out below.
1898       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1899       if (_error->PendingError() == true)
1900          return;
1901
1902       // Generate the final file name as: package_version_arch.foo
1903       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1904                       QuoteString(Version.VerStr(),"_:") + '_' +
1905                       QuoteString(Version.Arch(),"_:.") +
1906                       "." + flExtension(Parse.FileName());
1907    }
1908
1909    // check if we have one trusted source for the package. if so, switch
1910    // to "TrustedOnly" mode - but only if not in AllowUnauthenticated mode
1911    bool const allowUnauth = _config->FindB("APT::Get::AllowUnauthenticated", false);
1912    bool const debugAuth = _config->FindB("Debug::pkgAcquire::Auth", false);
1913    bool seenUntrusted = false;
1914    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; ++i)
1915    {
1916       pkgIndexFile *Index;
1917       if (Sources->FindIndex(i.File(),Index) == false)
1918          continue;
1919
1920       if (debugAuth == true)
1921          std::cerr << "Checking index: " << Index->Describe()
1922                    << "(Trusted=" << Index->IsTrusted() << ")" << std::endl;
1923
1924       if (Index->IsTrusted() == true)
1925       {
1926          Trusted = true;
1927          if (allowUnauth == false)
1928             break;
1929       }
1930       else
1931          seenUntrusted = true;
1932    }
1933
1934    // "allow-unauthenticated" restores apts old fetching behaviour
1935    // that means that e.g. unauthenticated file:// uris are higher
1936    // priority than authenticated http:// uris
1937    if (allowUnauth == true && seenUntrusted == true)
1938       Trusted = false;
1939
1940    // Select a source
1941    if (QueueNext() == false && _error->PendingError() == false)
1942       _error->Error(_("Can't find a source to download version '%s' of '%s'"),
1943                     Version.VerStr(), Version.ParentPkg().FullName(false).c_str());
1944 }
1945                                                                         /*}}}*/
1946 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1947 // ---------------------------------------------------------------------
1948 /* This queues the next available file version for download. It checks if
1949    the archive is already available in the cache and stashs the MD5 for
1950    checking later. */
1951 bool pkgAcqArchive::QueueNext()
1952 {
1953    string const ForceHash = _config->Find("Acquire::ForceHash");
1954    for (; Vf.end() == false; ++Vf)
1955    {
1956       // Ignore not source sources
1957       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1958          continue;
1959
1960       // Try to cross match against the source list
1961       pkgIndexFile *Index;
1962       if (Sources->FindIndex(Vf.File(),Index) == false)
1963             continue;
1964
1965       // only try to get a trusted package from another source if that source
1966       // is also trusted
1967       if(Trusted && !Index->IsTrusted())
1968          continue;
1969
1970       // Grab the text package record
1971       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1972       if (_error->PendingError() == true)
1973          return false;
1974
1975       string PkgFile = Parse.FileName();
1976       if (ForceHash.empty() == false)
1977       {
1978          if(stringcasecmp(ForceHash, "sha512") == 0)
1979             ExpectedHash = HashString("SHA512", Parse.SHA512Hash());
1980          else if(stringcasecmp(ForceHash, "sha256") == 0)
1981             ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
1982          else if (stringcasecmp(ForceHash, "sha1") == 0)
1983             ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
1984          else
1985             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1986       }
1987       else
1988       {
1989          string Hash;
1990          if ((Hash = Parse.SHA512Hash()).empty() == false)
1991             ExpectedHash = HashString("SHA512", Hash);
1992          else if ((Hash = Parse.SHA256Hash()).empty() == false)
1993             ExpectedHash = HashString("SHA256", Hash);
1994          else if ((Hash = Parse.SHA1Hash()).empty() == false)
1995             ExpectedHash = HashString("SHA1", Hash);
1996          else
1997             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1998       }
1999       if (PkgFile.empty() == true)
2000          return _error->Error(_("The package index files are corrupted. No Filename: "
2001                               "field for package %s."),
2002                               Version.ParentPkg().Name());
2003
2004       Desc.URI = Index->ArchiveURI(PkgFile);
2005       Desc.Description = Index->ArchiveInfo(Version);
2006       Desc.Owner = this;
2007       Desc.ShortDesc = Version.ParentPkg().FullName(true);
2008
2009       // See if we already have the file. (Legacy filenames)
2010       FileSize = Version->Size;
2011       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
2012       struct stat Buf;
2013       if (stat(FinalFile.c_str(),&Buf) == 0)
2014       {
2015          // Make sure the size matches
2016          if ((unsigned long long)Buf.st_size == Version->Size)
2017          {
2018             Complete = true;
2019             Local = true;
2020             Status = StatDone;
2021             StoreFilename = DestFile = FinalFile;
2022             return true;
2023          }
2024
2025          /* Hmm, we have a file and its size does not match, this means it is
2026             an old style mismatched arch */
2027          unlink(FinalFile.c_str());
2028       }
2029
2030       // Check it again using the new style output filenames
2031       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
2032       if (stat(FinalFile.c_str(),&Buf) == 0)
2033       {
2034          // Make sure the size matches
2035          if ((unsigned long long)Buf.st_size == Version->Size)
2036          {
2037             Complete = true;
2038             Local = true;
2039             Status = StatDone;
2040             StoreFilename = DestFile = FinalFile;
2041             return true;
2042          }
2043
2044          /* Hmm, we have a file and its size does not match, this shouldn't
2045             happen.. */
2046          unlink(FinalFile.c_str());
2047       }
2048
2049       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
2050
2051       // Check the destination file
2052       if (stat(DestFile.c_str(),&Buf) == 0)
2053       {
2054          // Hmm, the partial file is too big, erase it
2055          if ((unsigned long long)Buf.st_size > Version->Size)
2056             unlink(DestFile.c_str());
2057          else
2058             PartialSize = Buf.st_size;
2059       }
2060
2061       // Disables download of archives - useful if no real installation follows,
2062       // e.g. if we are just interested in proposed installation order
2063       if (_config->FindB("Debug::pkgAcqArchive::NoQueue", false) == true)
2064       {
2065          Complete = true;
2066          Local = true;
2067          Status = StatDone;
2068          StoreFilename = DestFile = FinalFile;
2069          return true;
2070       }
2071
2072       // Create the item
2073       Local = false;
2074       QueueURI(Desc);
2075
2076       ++Vf;
2077       return true;
2078    }
2079    return false;
2080 }
2081                                                                         /*}}}*/
2082 // AcqArchive::Done - Finished fetching                                 /*{{{*/
2083 // ---------------------------------------------------------------------
2084 /* */
2085 void pkgAcqArchive::Done(string Message,unsigned long long Size,string CalcHash,
2086                          pkgAcquire::MethodConfig *Cfg)
2087 {
2088    Item::Done(Message,Size,CalcHash,Cfg);
2089
2090    // Check the size
2091    if (Size != Version->Size)
2092    {
2093       RenameOnError(SizeMismatch);
2094       return;
2095    }
2096
2097    // Check the hash
2098    if(ExpectedHash.toStr() != CalcHash)
2099    {
2100       RenameOnError(HashSumMismatch);
2101       return;
2102    }
2103
2104    // Grab the output filename
2105    string FileName = LookupTag(Message,"Filename");
2106    if (FileName.empty() == true)
2107    {
2108       Status = StatError;
2109       ErrorText = "Method gave a blank filename";
2110       return;
2111    }
2112
2113    Complete = true;
2114
2115    // Reference filename
2116    if (FileName != DestFile)
2117    {
2118       StoreFilename = DestFile = FileName;
2119       Local = true;
2120       return;
2121    }
2122
2123    // Done, move it into position
2124    string FinalFile = _config->FindDir("Dir::Cache::Archives");
2125    FinalFile += flNotDir(StoreFilename);
2126    Rename(DestFile,FinalFile);
2127
2128    StoreFilename = DestFile = FinalFile;
2129    Complete = true;
2130 }
2131                                                                         /*}}}*/
2132 // AcqArchive::Failed - Failure handler                                 /*{{{*/
2133 // ---------------------------------------------------------------------
2134 /* Here we try other sources */
2135 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2136 {
2137    ErrorText = LookupTag(Message,"Message");
2138
2139    /* We don't really want to retry on failed media swaps, this prevents
2140       that. An interesting observation is that permanent failures are not
2141       recorded. */
2142    if (Cnf->Removable == true &&
2143        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2144    {
2145       // Vf = Version.FileList();
2146       while (Vf.end() == false) ++Vf;
2147       StoreFilename = string();
2148       Item::Failed(Message,Cnf);
2149       return;
2150    }
2151
2152    if (QueueNext() == false)
2153    {
2154       // This is the retry counter
2155       if (Retries != 0 &&
2156           Cnf->LocalOnly == false &&
2157           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2158       {
2159          Retries--;
2160          Vf = Version.FileList();
2161          if (QueueNext() == true)
2162             return;
2163       }
2164
2165       StoreFilename = string();
2166       Item::Failed(Message,Cnf);
2167    }
2168 }
2169                                                                         /*}}}*/
2170 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
2171 // ---------------------------------------------------------------------
2172 APT_PURE bool pkgAcqArchive::IsTrusted()
2173 {
2174    return Trusted;
2175 }
2176                                                                         /*}}}*/
2177 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
2178 // ---------------------------------------------------------------------
2179 /* */
2180 void pkgAcqArchive::Finished()
2181 {
2182    if (Status == pkgAcquire::Item::StatDone &&
2183        Complete == true)
2184       return;
2185    StoreFilename = string();
2186 }
2187                                                                         /*}}}*/
2188 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
2189 // ---------------------------------------------------------------------
2190 /* The file is added to the queue */
2191 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
2192                        unsigned long long Size,string Dsc,string ShortDesc,
2193                        const string &DestDir, const string &DestFilename,
2194                        bool IsIndexFile) :
2195                        Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile)
2196 {
2197    Retries = _config->FindI("Acquire::Retries",0);
2198
2199    if(!DestFilename.empty())
2200       DestFile = DestFilename;
2201    else if(!DestDir.empty())
2202       DestFile = DestDir + "/" + flNotDir(URI);
2203    else
2204       DestFile = flNotDir(URI);
2205
2206    // Create the item
2207    Desc.URI = URI;
2208    Desc.Description = Dsc;
2209    Desc.Owner = this;
2210
2211    // Set the short description to the archive component
2212    Desc.ShortDesc = ShortDesc;
2213
2214    // Get the transfer sizes
2215    FileSize = Size;
2216    struct stat Buf;
2217    if (stat(DestFile.c_str(),&Buf) == 0)
2218    {
2219       // Hmm, the partial file is too big, erase it
2220       if ((Size > 0) && (unsigned long long)Buf.st_size > Size)
2221          unlink(DestFile.c_str());
2222       else
2223          PartialSize = Buf.st_size;
2224    }
2225
2226    QueueURI(Desc);
2227 }
2228                                                                         /*}}}*/
2229 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
2230 // ---------------------------------------------------------------------
2231 /* */
2232 void pkgAcqFile::Done(string Message,unsigned long long Size,string CalcHash,
2233                       pkgAcquire::MethodConfig *Cnf)
2234 {
2235    Item::Done(Message,Size,CalcHash,Cnf);
2236
2237    // Check the hash
2238    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
2239    {
2240       RenameOnError(HashSumMismatch);
2241       return;
2242    }
2243
2244    string FileName = LookupTag(Message,"Filename");
2245    if (FileName.empty() == true)
2246    {
2247       Status = StatError;
2248       ErrorText = "Method gave a blank filename";
2249       return;
2250    }
2251
2252    Complete = true;
2253
2254    // The files timestamp matches
2255    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
2256       return;
2257
2258    // We have to copy it into place
2259    if (FileName != DestFile)
2260    {
2261       Local = true;
2262       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
2263           Cnf->Removable == true)
2264       {
2265          Desc.URI = "copy:" + FileName;
2266          QueueURI(Desc);
2267          return;
2268       }
2269
2270       // Erase the file if it is a symlink so we can overwrite it
2271       struct stat St;
2272       if (lstat(DestFile.c_str(),&St) == 0)
2273       {
2274          if (S_ISLNK(St.st_mode) != 0)
2275             unlink(DestFile.c_str());
2276       }
2277
2278       // Symlink the file
2279       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
2280       {
2281          ErrorText = "Link to " + DestFile + " failure ";
2282          Status = StatError;
2283          Complete = false;
2284       }
2285    }
2286 }
2287                                                                         /*}}}*/
2288 // AcqFile::Failed - Failure handler                                    /*{{{*/
2289 // ---------------------------------------------------------------------
2290 /* Here we try other sources */
2291 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2292 {
2293    ErrorText = LookupTag(Message,"Message");
2294
2295    // This is the retry counter
2296    if (Retries != 0 &&
2297        Cnf->LocalOnly == false &&
2298        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2299    {
2300       Retries--;
2301       QueueURI(Desc);
2302       return;
2303    }
2304
2305    Item::Failed(Message,Cnf);
2306 }
2307                                                                         /*}}}*/
2308 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
2309 // ---------------------------------------------------------------------
2310 /* The only header we use is the last-modified header. */
2311 string pkgAcqFile::Custom600Headers()
2312 {
2313    if (IsIndexFile)
2314       return "\nIndex-File: true";
2315    return "";
2316 }
2317                                                                         /*}}}*/